1 # -*- coding: utf-8 -*-
4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
5 # This file is part of openmano
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
25 utils is a module that implements functions that are used by all openmano modules,
26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
27 from dictionaries to appropriate database dictionaries, etc.
29 __author__
="Alfonso Tierno, Gerardo Garcia"
30 __date__
="$08-sep-2014 12:21:22$"
35 from functools
import reduce, partial
, wraps
36 from itertools
import tee
39 from six
.moves
import filter, filterfalse
41 from jsonschema
import exceptions
as js_e
42 from jsonschema
import validate
as js_v
45 from inspect
import getfullargspec
as getspec
47 from inspect
import getargspec
as getspec
49 #from bs4 import BeautifulSoup
51 def read_file(file_to_read
):
52 """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
54 f
= open(file_to_read
, 'r')
57 except Exception as e
:
58 return (False, str(e
))
60 return (True, read_data
)
62 def write_file(file_to_write
, text
):
63 """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
65 f
= open(file_to_write
, 'w')
68 except Exception as e
:
69 return (False, str(e
))
73 def format_in(http_response
, schema
):
75 client_data
= http_response
.json()
76 js_v(client_data
, schema
)
77 #print "Input data: ", str(client_data)
78 return True, client_data
79 except js_e
.ValidationError
as exc
:
80 print "validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
81 return False, ("validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
)
83 def remove_extra_items(data
, schema
):
85 if type(data
) is tuple or type(data
) is list:
87 a
= remove_extra_items(d
, schema
['items'])
88 if a
is not None: deleted
.append(a
)
89 elif type(data
) is dict:
90 #TODO deal with patternProperties
91 if 'properties' not in schema
:
94 if k
not in schema
['properties'].keys():
98 a
= remove_extra_items(data
[k
], schema
['properties'][k
])
99 if a
is not None: deleted
.append({k
:a
})
100 if len(deleted
) == 0: return None
101 elif len(deleted
) == 1: return deleted
[0]
104 #def format_html2text(http_content):
105 # soup=BeautifulSoup(http_content)
106 # text = soup.p.get_text() + " " + soup.pre.get_text()
110 def delete_nulls(var
):
111 if type(var
) is dict:
113 if var
[k
] is None: del var
[k
]
114 elif type(var
[k
]) is dict or type(var
[k
]) is list or type(var
[k
]) is tuple:
115 if delete_nulls(var
[k
]): del var
[k
]
116 if len(var
) == 0: return True
117 elif type(var
) is list or type(var
) is tuple:
119 if type(k
) is dict: delete_nulls(k
)
120 if len(var
) == 0: return True
124 def convert_bandwidth(data
, reverse
=False):
125 '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
126 It assumes that bandwidth is well formed
128 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
129 'reverse': by default convert form str to int (Mbps), if True it convert from number to units
133 if type(data
) is dict:
134 for k
in data
.keys():
135 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
136 convert_bandwidth(data
[k
], reverse
)
137 if "bandwidth" in data
:
139 value
=str(data
["bandwidth"])
141 pos
= value
.find("bps")
143 if value
[pos
-1]=="G": data
["bandwidth"] = int(data
["bandwidth"][:pos
-1]) * 1000
144 elif value
[pos
-1]=="k": data
["bandwidth"]= int(data
["bandwidth"][:pos
-1]) / 1000
145 else: data
["bandwidth"]= int(data
["bandwidth"][:pos
-1])
147 value
= int(data
["bandwidth"])
148 if value
% 1000 == 0: data
["bandwidth"]=str(value
/1000) + " Gbps"
149 else: data
["bandwidth"]=str(value
) + " Mbps"
151 print "convert_bandwidth exception for type", type(data
["bandwidth"]), " data", data
["bandwidth"]
153 if type(data
) is tuple or type(data
) is list:
155 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
156 convert_bandwidth(k
, reverse
)
158 def convert_float_timestamp2str(var
):
159 '''Converts timestamps (created_at, modified_at fields) represented as float
160 to a string with the format '%Y-%m-%dT%H:%i:%s'
161 It enters recursively in the dict var finding this kind of variables
163 if type(var
) is dict:
164 for k
,v
in var
.items():
165 if type(v
) is float and k
in ("created_at", "modified_at"):
166 var
[k
] = time
.strftime("%Y-%m-%dT%H:%M:%S", time
.localtime(v
) )
167 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
168 convert_float_timestamp2str(v
)
169 if len(var
) == 0: return True
170 elif type(var
) is list or type(var
) is tuple:
172 convert_float_timestamp2str(v
)
174 def convert_datetime2str(var
):
175 '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
176 It enters recursively in the dict var finding this kind of variables
178 if type(var
) is dict:
179 for k
,v
in var
.items():
180 if type(v
) is datetime
.datetime
:
181 var
[k
]= v
.strftime('%Y-%m-%dT%H:%M:%S')
182 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
183 convert_datetime2str(v
)
184 if len(var
) == 0: return True
185 elif type(var
) is list or type(var
) is tuple:
187 convert_datetime2str(v
)
189 def convert_str2boolean(data
, items
):
190 '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
193 'data': dictionary variable to be checked. None or empty is considered valid
194 'items': tuple of keys to convert
198 if type(data
) is dict:
199 for k
in data
.keys():
200 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
201 convert_str2boolean(data
[k
], items
)
203 if type(data
[k
]) is str:
204 if data
[k
]=="false" or data
[k
]=="False": data
[k
]=False
205 elif data
[k
]=="true" or data
[k
]=="True": data
[k
]=True
206 if type(data
) is tuple or type(data
) is list:
208 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
209 convert_str2boolean(k
, items
)
211 def check_valid_uuid(uuid
):
212 id_schema
= {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
213 id_schema2
= {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
215 js_v(uuid
, id_schema
)
217 except js_e
.ValidationError
:
219 js_v(uuid
, id_schema2
)
221 except js_e
.ValidationError
:
226 def expand_brackets(text
):
228 Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
229 if no bracket is used it just return the a list with the single text
230 It uses recursivity to allow several [] in the text
236 start
= text
.find("[")
238 if start
< 0 or end
< 0:
241 for char
in text
[start
+1:end
]:
242 text_list
+= expand_brackets(text
[:start
] + char
+ text
[end
+1:])
245 def deprecated(message
):
246 def deprecated_decorator(func
):
247 def deprecated_func(*args
, **kwargs
):
248 warnings
.warn("{} is a deprecated function. {}".format(func
.__name
__, message
),
249 category
=DeprecationWarning,
251 warnings
.simplefilter('default', DeprecationWarning)
252 return func(*args
, **kwargs
)
253 return deprecated_func
254 return deprecated_decorator
257 def truncate(text
, max_length
=1024):
258 """Limit huge texts in number of characters"""
260 if text
and len(text
) >= max_length
:
261 return text
[:max_length
//2-3] + " ... " + text
[-max_length
//2+3:]
265 def merge_dicts(*dicts
, **kwargs
):
266 """Creates a new dict merging N others and keyword arguments.
267 Right-most dicts take precedence.
268 Keyword args take precedence.
271 lambda acc
, x
: acc
.update(x
) or acc
,
272 list(dicts
) + [kwargs
], {})
275 def remove_none_items(adict
):
276 """Return a similar dict without keys associated to None values"""
277 return {k
: v
for k
, v
in adict
.items() if v
is not None}
280 def filter_dict_keys(adict
, allow
):
281 """Return a similar dict, but just containing the explicitly allowed keys
284 adict (dict): Simple python dict data struct
285 allow (list): Explicits allowed keys
287 return {k
: v
for k
, v
in adict
.items() if k
in allow
}
290 def filter_out_dict_keys(adict
, deny
):
291 """Return a similar dict, but not containing the explicitly denied keys
294 adict (dict): Simple python dict data struct
295 deny (list): Explicits denied keys
297 return {k
: v
for k
, v
in adict
.items() if k
not in deny
}
300 def expand_joined_fields(record
):
301 """Given a db query result, explode the fields that contains `.` (join
305 >> expand_joined_fiels({'wim.id': 2})
309 for field
, value
in record
.items():
310 keys
= field
.split('.')
312 target
= reduce(lambda target
, key
: target
.setdefault(key
, {}),
314 target
[keys
[-1]] = value
319 def ensure(condition
, exception
):
320 """Raise an exception if condition is not met"""
325 def partition(predicate
, iterable
):
326 """Create two derived iterators from a single one
327 The first iterator created will loop thought the values where the function
328 predicate is True, the second one will iterate over the values where it is
331 iterable1
, iterable2
= tee(iterable
)
332 return filter(predicate
, iterable2
), filterfalse(predicate
, iterable1
)
335 def pipe(*functions
):
336 """Compose functions of one argument in the opposite order,
337 So pipe(f, g)(x) = g(f(x))
339 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
, x
)
342 def compose(*functions
):
343 """Compose functions of one argument,
344 So compose(f, g)(x) = f(g(x))
346 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
[::-1], x
)
349 def safe_get(target
, key_path
, default
=None):
350 """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
351 a nested dict if present, or the default value
353 keys
= key_path
.split('.')
354 target
= reduce(lambda acc
, key
: acc
.get(key
) or {}, keys
[:-1], target
)
355 return target
.get(keys
[-1], default
)
358 class Attempt(object):
359 """Auxiliary class to be used in an attempt to retry executing a failing
363 count (int): 0-based "retries" counter
364 max_attempts (int): maximum number of "retries" allowed
365 info (dict): extra information about the specific attempt
366 (can be used to produce more meaningful error messages)
368 __slots__
= ('count', 'max', 'info')
372 def __init__(self
, count
=0, max_attempts
=MAX
, info
=None):
374 self
.max = max_attempts
375 self
.info
= info
or {}
379 """Like count, but in the opposite direction"""
380 return self
.max - self
.count
384 """1-based counter"""
385 return self
.count
+ 1
388 def inject_args(fn
=None, **args
):
389 """Partially apply keyword arguments in a function, but only if the function
390 define them in the first place
392 if fn
is None: # Allows calling the decorator directly or with parameters
393 return partial(inject_args
, **args
)
396 return wraps(fn
)(partial(fn
, **filter_dict_keys(args
, spec
.args
)))
399 def get_arg(name
, fn
, args
, kwargs
):
400 """Find the value of an argument for a function, given its argument list.
402 This function can be used to display more meaningful errors for debugging
408 if name
in spec
.args
:
409 i
= spec
.args
.index(name
)
410 return args
[i
] if i
< len(args
) else None