1 # -*- coding: utf-8 -*-
4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
5 # This file is part of openmano
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
25 utils is a module that implements functions that are used by all openmano modules,
26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
27 from dictionaries to appropriate database dictionaries, etc.
29 __author__
="Alfonso Tierno, Gerardo Garcia"
30 __date__
="$08-sep-2014 12:21:22$"
35 from functools
import reduce, partial
, wraps
36 from itertools
import tee
39 from six
.moves
import filter, filterfalse
41 from jsonschema
import exceptions
as js_e
42 from jsonschema
import validate
as js_v
45 from inspect
import getfullargspec
as getspec
47 from inspect
import getargspec
as getspec
49 #from bs4 import BeautifulSoup
51 def read_file(file_to_read
):
52 """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
54 f
= open(file_to_read
, 'r')
57 except Exception as e
:
58 return (False, str(e
))
60 return (True, read_data
)
62 def write_file(file_to_write
, text
):
63 """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
65 f
= open(file_to_write
, 'w')
68 except Exception as e
:
69 return (False, str(e
))
73 def format_in(http_response
, schema
):
75 client_data
= http_response
.json()
76 js_v(client_data
, schema
)
77 #print "Input data: ", str(client_data)
78 return True, client_data
79 except js_e
.ValidationError
as exc
:
80 print "validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
81 return False, ("validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
)
83 def remove_extra_items(data
, schema
):
85 if type(data
) is tuple or type(data
) is list:
87 a
= remove_extra_items(d
, schema
['items'])
90 elif type(data
) is dict:
91 # TODO deal with patternProperties
92 if 'properties' not in schema
:
95 if k
in schema
['properties'].keys():
96 a
= remove_extra_items(data
[k
], schema
['properties'][k
])
98 deleted
.append({k
: a
})
99 elif not schema
.get('additionalProperties'):
102 if len(deleted
) == 0:
104 elif len(deleted
) == 1:
109 #def format_html2text(http_content):
110 # soup=BeautifulSoup(http_content)
111 # text = soup.p.get_text() + " " + soup.pre.get_text()
115 def delete_nulls(var
):
116 if type(var
) is dict:
118 if var
[k
] is None: del var
[k
]
119 elif type(var
[k
]) is dict or type(var
[k
]) is list or type(var
[k
]) is tuple:
120 if delete_nulls(var
[k
]): del var
[k
]
121 if len(var
) == 0: return True
122 elif type(var
) is list or type(var
) is tuple:
124 if type(k
) is dict: delete_nulls(k
)
125 if len(var
) == 0: return True
129 def convert_bandwidth(data
, reverse
=False):
130 '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
131 It assumes that bandwidth is well formed
133 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
134 'reverse': by default convert form str to int (Mbps), if True it convert from number to units
138 if type(data
) is dict:
139 for k
in data
.keys():
140 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
141 convert_bandwidth(data
[k
], reverse
)
142 if "bandwidth" in data
:
144 value
=str(data
["bandwidth"])
146 pos
= value
.find("bps")
148 if value
[pos
-1]=="G": data
["bandwidth"] = int(data
["bandwidth"][:pos
-1]) * 1000
149 elif value
[pos
-1]=="k": data
["bandwidth"]= int(data
["bandwidth"][:pos
-1]) / 1000
150 else: data
["bandwidth"]= int(data
["bandwidth"][:pos
-1])
152 value
= int(data
["bandwidth"])
153 if value
% 1000 == 0: data
["bandwidth"]=str(value
/1000) + " Gbps"
154 else: data
["bandwidth"]=str(value
) + " Mbps"
156 print "convert_bandwidth exception for type", type(data
["bandwidth"]), " data", data
["bandwidth"]
158 if type(data
) is tuple or type(data
) is list:
160 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
161 convert_bandwidth(k
, reverse
)
163 def convert_float_timestamp2str(var
):
164 '''Converts timestamps (created_at, modified_at fields) represented as float
165 to a string with the format '%Y-%m-%dT%H:%i:%s'
166 It enters recursively in the dict var finding this kind of variables
168 if type(var
) is dict:
169 for k
,v
in var
.items():
170 if type(v
) is float and k
in ("created_at", "modified_at"):
171 var
[k
] = time
.strftime("%Y-%m-%dT%H:%M:%S", time
.localtime(v
) )
172 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
173 convert_float_timestamp2str(v
)
174 if len(var
) == 0: return True
175 elif type(var
) is list or type(var
) is tuple:
177 convert_float_timestamp2str(v
)
179 def convert_datetime2str(var
):
180 '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
181 It enters recursively in the dict var finding this kind of variables
183 if type(var
) is dict:
184 for k
,v
in var
.items():
185 if type(v
) is datetime
.datetime
:
186 var
[k
]= v
.strftime('%Y-%m-%dT%H:%M:%S')
187 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
188 convert_datetime2str(v
)
189 if len(var
) == 0: return True
190 elif type(var
) is list or type(var
) is tuple:
192 convert_datetime2str(v
)
194 def convert_str2boolean(data
, items
):
195 '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
198 'data': dictionary variable to be checked. None or empty is considered valid
199 'items': tuple of keys to convert
203 if type(data
) is dict:
204 for k
in data
.keys():
205 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
206 convert_str2boolean(data
[k
], items
)
208 if type(data
[k
]) is str:
209 if data
[k
]=="false" or data
[k
]=="False": data
[k
]=False
210 elif data
[k
]=="true" or data
[k
]=="True": data
[k
]=True
211 if type(data
) is tuple or type(data
) is list:
213 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
214 convert_str2boolean(k
, items
)
216 def check_valid_uuid(uuid
):
217 id_schema
= {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
218 id_schema2
= {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
220 js_v(uuid
, id_schema
)
222 except js_e
.ValidationError
:
224 js_v(uuid
, id_schema2
)
226 except js_e
.ValidationError
:
231 def expand_brackets(text
):
233 Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
234 if no bracket is used it just return the a list with the single text
235 It uses recursivity to allow several [] in the text
241 start
= text
.find("[")
243 if start
< 0 or end
< 0:
246 for char
in text
[start
+1:end
]:
247 text_list
+= expand_brackets(text
[:start
] + char
+ text
[end
+1:])
250 def deprecated(message
):
251 def deprecated_decorator(func
):
252 def deprecated_func(*args
, **kwargs
):
253 warnings
.warn("{} is a deprecated function. {}".format(func
.__name
__, message
),
254 category
=DeprecationWarning,
256 warnings
.simplefilter('default', DeprecationWarning)
257 return func(*args
, **kwargs
)
258 return deprecated_func
259 return deprecated_decorator
262 def truncate(text
, max_length
=1024):
263 """Limit huge texts in number of characters"""
265 if text
and len(text
) >= max_length
:
266 return text
[:max_length
//2-3] + " ... " + text
[-max_length
//2+3:]
270 def merge_dicts(*dicts
, **kwargs
):
271 """Creates a new dict merging N others and keyword arguments.
272 Right-most dicts take precedence.
273 Keyword args take precedence.
276 lambda acc
, x
: acc
.update(x
) or acc
,
277 list(dicts
) + [kwargs
], {})
280 def remove_none_items(adict
):
281 """Return a similar dict without keys associated to None values"""
282 return {k
: v
for k
, v
in adict
.items() if v
is not None}
285 def filter_dict_keys(adict
, allow
):
286 """Return a similar dict, but just containing the explicitly allowed keys
289 adict (dict): Simple python dict data struct
290 allow (list): Explicits allowed keys
292 return {k
: v
for k
, v
in adict
.items() if k
in allow
}
295 def filter_out_dict_keys(adict
, deny
):
296 """Return a similar dict, but not containing the explicitly denied keys
299 adict (dict): Simple python dict data struct
300 deny (list): Explicits denied keys
302 return {k
: v
for k
, v
in adict
.items() if k
not in deny
}
305 def expand_joined_fields(record
):
306 """Given a db query result, explode the fields that contains `.` (join
310 >> expand_joined_fiels({'wim.id': 2})
314 for field
, value
in record
.items():
315 keys
= field
.split('.')
317 target
= reduce(lambda target
, key
: target
.setdefault(key
, {}),
319 target
[keys
[-1]] = value
324 def ensure(condition
, exception
):
325 """Raise an exception if condition is not met"""
330 def partition(predicate
, iterable
):
331 """Create two derived iterators from a single one
332 The first iterator created will loop thought the values where the function
333 predicate is True, the second one will iterate over the values where it is
336 iterable1
, iterable2
= tee(iterable
)
337 return filter(predicate
, iterable2
), filterfalse(predicate
, iterable1
)
340 def pipe(*functions
):
341 """Compose functions of one argument in the opposite order,
342 So pipe(f, g)(x) = g(f(x))
344 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
, x
)
347 def compose(*functions
):
348 """Compose functions of one argument,
349 So compose(f, g)(x) = f(g(x))
351 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
[::-1], x
)
354 def safe_get(target
, key_path
, default
=None):
355 """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
356 a nested dict if present, or the default value
358 keys
= key_path
.split('.')
359 target
= reduce(lambda acc
, key
: acc
.get(key
) or {}, keys
[:-1], target
)
360 return target
.get(keys
[-1], default
)
363 class Attempt(object):
364 """Auxiliary class to be used in an attempt to retry executing a failing
368 count (int): 0-based "retries" counter
369 max_attempts (int): maximum number of "retries" allowed
370 info (dict): extra information about the specific attempt
371 (can be used to produce more meaningful error messages)
373 __slots__
= ('count', 'max', 'info')
377 def __init__(self
, count
=0, max_attempts
=MAX
, info
=None):
379 self
.max = max_attempts
380 self
.info
= info
or {}
384 """Like count, but in the opposite direction"""
385 return self
.max - self
.count
389 """1-based counter"""
390 return self
.count
+ 1
393 def inject_args(fn
=None, **args
):
394 """Partially apply keyword arguments in a function, but only if the function
395 define them in the first place
397 if fn
is None: # Allows calling the decorator directly or with parameters
398 return partial(inject_args
, **args
)
401 return wraps(fn
)(partial(fn
, **filter_dict_keys(args
, spec
.args
)))
404 def get_arg(name
, fn
, args
, kwargs
):
405 """Find the value of an argument for a function, given its argument list.
407 This function can be used to display more meaningful errors for debugging
413 if name
in spec
.args
:
414 i
= spec
.args
.index(name
)
415 return args
[i
] if i
< len(args
) else None