1 # -*- coding: utf-8 -*-
4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
5 # This file is part of openmano
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
25 utils is a module that implements functions that are used by all openmano modules,
26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
27 from dictionaries to appropriate database dictionaries, etc.
29 __author__
="Alfonso Tierno, Gerardo Garcia"
30 __date__
="$08-sep-2014 12:21:22$"
35 from functools
import reduce, partial
, wraps
36 from itertools
import tee
38 from itertools
import filterfalse
40 from jsonschema
import exceptions
as js_e
41 from jsonschema
import validate
as js_v
43 from inspect
import getfullargspec
as getspec
45 #from bs4 import BeautifulSoup
47 def read_file(file_to_read
):
48 """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
50 f
= open(file_to_read
, 'r')
53 except Exception as e
:
54 return (False, str(e
))
56 return (True, read_data
)
58 def write_file(file_to_write
, text
):
59 """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
61 f
= open(file_to_write
, 'w')
64 except Exception as e
:
65 return (False, str(e
))
69 def format_in(http_response
, schema
):
71 client_data
= http_response
.json()
72 js_v(client_data
, schema
)
73 #print "Input data: ", str(client_data)
74 return True, client_data
75 except js_e
.ValidationError
as exc
:
76 print("validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
)
77 return False, ("validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
)
79 def remove_extra_items(data
, schema
):
81 if isinstance(data
, (tuple, list)):
83 a
= remove_extra_items(d
, schema
['items'])
86 elif isinstance(data
, dict):
87 # TODO deal with patternProperties
88 if 'properties' not in schema
:
92 if k
in schema
['properties']:
93 a
= remove_extra_items(data
[k
], schema
['properties'][k
])
95 deleted
.append({k
: a
})
96 elif not schema
.get('additionalProperties'):
101 if len(deleted
) == 0:
103 elif len(deleted
) == 1:
108 #def format_html2text(http_content):
109 # soup=BeautifulSoup(http_content)
110 # text = soup.p.get_text() + " " + soup.pre.get_text()
114 def delete_nulls(var
):
115 if isinstance(var
, dict):
119 to_delete
.append([k
])
120 elif isinstance(var
[k
], (dict, list, tuple)):
121 if delete_nulls(var
[k
]):
127 elif isinstance(var
, (list, tuple)):
129 if isinstance(k
, dict):
136 def convert_bandwidth(data
, reverse
=False):
137 '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
138 It assumes that bandwidth is well formed
140 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
141 'reverse': by default convert form str to int (Mbps), if True it convert from number to units
145 if isinstance(data
, dict):
146 for k
in data
.keys():
147 if isinstance(data
[k
], (dict, tuple, list)):
148 convert_bandwidth(data
[k
], reverse
)
149 if "bandwidth" in data
:
151 value
=str(data
["bandwidth"])
153 pos
= value
.find("bps")
155 if value
[pos
-1]=="G":
156 data
["bandwidth"] = int(data
["bandwidth"][:pos
-1]) * 1000
157 elif value
[pos
-1]=="k":
158 data
["bandwidth"]= int(data
["bandwidth"][:pos
-1]) // 1000
160 data
["bandwidth"]= int(data
["bandwidth"][:pos
])
162 value
= int(data
["bandwidth"])
163 if value
% 1000 == 0 and value
> 1000:
164 data
["bandwidth"] = str(value
// 1000) + " Gbps"
166 data
["bandwidth"] = str(value
) + " Mbps"
168 print("convert_bandwidth exception for type", type(data
["bandwidth"]), " data", data
["bandwidth"])
170 if isinstance(data
, (tuple, list)):
172 if isinstance(k
, (dict, tuple, list)):
173 convert_bandwidth(k
, reverse
)
175 def convert_float_timestamp2str(var
):
176 '''Converts timestamps (created_at, modified_at fields) represented as float
177 to a string with the format '%Y-%m-%dT%H:%i:%s'
178 It enters recursively in the dict var finding this kind of variables
180 if type(var
) is dict:
181 for k
,v
in var
.items():
182 if type(v
) is float and k
in ("created_at", "modified_at"):
183 var
[k
] = time
.strftime("%Y-%m-%dT%H:%M:%S", time
.localtime(v
) )
184 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
185 convert_float_timestamp2str(v
)
186 if len(var
) == 0: return True
187 elif type(var
) is list or type(var
) is tuple:
189 convert_float_timestamp2str(v
)
191 def convert_datetime2str(var
):
192 '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
193 It enters recursively in the dict var finding this kind of variables
195 if type(var
) is dict:
196 for k
,v
in var
.items():
197 if type(v
) is datetime
.datetime
:
198 var
[k
]= v
.strftime('%Y-%m-%dT%H:%M:%S')
199 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
200 convert_datetime2str(v
)
201 if len(var
) == 0: return True
202 elif type(var
) is list or type(var
) is tuple:
204 convert_datetime2str(v
)
206 def convert_str2boolean(data
, items
):
207 '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
210 'data': dictionary variable to be checked. None or empty is considered valid
211 'items': tuple of keys to convert
215 if type(data
) is dict:
216 for k
in data
.keys():
217 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
218 convert_str2boolean(data
[k
], items
)
220 if type(data
[k
]) is str:
221 if data
[k
]=="false" or data
[k
]=="False": data
[k
]=False
222 elif data
[k
]=="true" or data
[k
]=="True": data
[k
]=True
223 if type(data
) is tuple or type(data
) is list:
225 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
226 convert_str2boolean(k
, items
)
228 def check_valid_uuid(uuid
):
229 id_schema
= {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
230 id_schema2
= {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
232 js_v(uuid
, id_schema
)
234 except js_e
.ValidationError
:
236 js_v(uuid
, id_schema2
)
238 except js_e
.ValidationError
:
243 def expand_brackets(text
):
245 Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
246 if no bracket is used it just return the a list with the single text
247 It uses recursivity to allow several [] in the text
253 start
= text
.find("[")
255 if start
< 0 or end
< 0:
258 for char
in text
[start
+1:end
]:
259 text_list
+= expand_brackets(text
[:start
] + char
+ text
[end
+1:])
262 def deprecated(message
):
263 def deprecated_decorator(func
):
264 def deprecated_func(*args
, **kwargs
):
265 warnings
.warn("{} is a deprecated function. {}".format(func
.__name
__, message
),
266 category
=DeprecationWarning,
268 warnings
.simplefilter('default', DeprecationWarning)
269 return func(*args
, **kwargs
)
270 return deprecated_func
271 return deprecated_decorator
274 def truncate(text
, max_length
=1024):
275 """Limit huge texts in number of characters"""
277 if text
and len(text
) >= max_length
:
278 return text
[:max_length
//2-3] + " ... " + text
[-max_length
//2+3:]
282 def merge_dicts(*dicts
, **kwargs
):
283 """Creates a new dict merging N others and keyword arguments.
284 Right-most dicts take precedence.
285 Keyword args take precedence.
288 lambda acc
, x
: acc
.update(x
) or acc
,
289 list(dicts
) + [kwargs
], {})
292 def remove_none_items(adict
):
293 """Return a similar dict without keys associated to None values"""
294 return {k
: v
for k
, v
in adict
.items() if v
is not None}
297 def filter_dict_keys(adict
, allow
):
298 """Return a similar dict, but just containing the explicitly allowed keys
301 adict (dict): Simple python dict data struct
302 allow (list): Explicits allowed keys
304 return {k
: v
for k
, v
in adict
.items() if k
in allow
}
307 def filter_out_dict_keys(adict
, deny
):
308 """Return a similar dict, but not containing the explicitly denied keys
311 adict (dict): Simple python dict data struct
312 deny (list): Explicits denied keys
314 return {k
: v
for k
, v
in adict
.items() if k
not in deny
}
317 def expand_joined_fields(record
):
318 """Given a db query result, explode the fields that contains `.` (join
322 >> expand_joined_fiels({'wim.id': 2})
326 for field
, value
in record
.items():
327 keys
= field
.split('.')
329 target
= reduce(lambda target
, key
: target
.setdefault(key
, {}),
331 target
[keys
[-1]] = value
336 def ensure(condition
, exception
):
337 """Raise an exception if condition is not met"""
342 def partition(predicate
, iterable
):
343 """Create two derived iterators from a single one
344 The first iterator created will loop thought the values where the function
345 predicate is True, the second one will iterate over the values where it is
348 iterable1
, iterable2
= tee(iterable
)
349 return filter(predicate
, iterable2
), filterfalse(predicate
, iterable1
)
352 def pipe(*functions
):
353 """Compose functions of one argument in the opposite order,
354 So pipe(f, g)(x) = g(f(x))
356 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
, x
)
359 def compose(*functions
):
360 """Compose functions of one argument,
361 So compose(f, g)(x) = f(g(x))
363 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
[::-1], x
)
366 def safe_get(target
, key_path
, default
=None):
367 """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
368 a nested dict if present, or the default value
370 keys
= key_path
.split('.')
371 target
= reduce(lambda acc
, key
: acc
.get(key
) or {}, keys
[:-1], target
)
372 return target
.get(keys
[-1], default
)
375 class Attempt(object):
376 """Auxiliary class to be used in an attempt to retry executing a failing
380 count (int): 0-based "retries" counter
381 max_attempts (int): maximum number of "retries" allowed
382 info (dict): extra information about the specific attempt
383 (can be used to produce more meaningful error messages)
385 __slots__
= ('count', 'max', 'info')
389 def __init__(self
, count
=0, max_attempts
=MAX
, info
=None):
391 self
.max = max_attempts
392 self
.info
= info
or {}
396 """Like count, but in the opposite direction"""
397 return self
.max - self
.count
401 """1-based counter"""
402 return self
.count
+ 1
405 def inject_args(fn
=None, **args
):
406 """Partially apply keyword arguments in a function, but only if the function
407 define them in the first place
409 if fn
is None: # Allows calling the decorator directly or with parameters
410 return partial(inject_args
, **args
)
413 return wraps(fn
)(partial(fn
, **filter_dict_keys(args
, spec
.args
)))
416 def get_arg(name
, fn
, args
, kwargs
):
417 """Find the value of an argument for a function, given its argument list.
419 This function can be used to display more meaningful errors for debugging
425 if name
in spec
.args
:
426 i
= spec
.args
.index(name
)
427 return args
[i
] if i
< len(args
) else None