1 # -*- coding: utf-8 -*-
4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
5 # This file is part of openmano
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
25 utils is a module that implements functions that are used by all openmano modules,
26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
27 from dictionaries to appropriate database dictionaries, etc.
29 __author__
="Alfonso Tierno, Gerardo Garcia"
30 __date__
="$08-sep-2014 12:21:22$"
35 from functools
import reduce
36 from itertools
import tee
38 from six
.moves
import filter, filterfalse
40 from jsonschema
import exceptions
as js_e
41 from jsonschema
import validate
as js_v
43 #from bs4 import BeautifulSoup
45 def read_file(file_to_read
):
46 """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
48 f
= open(file_to_read
, 'r')
51 except Exception as e
:
52 return (False, str(e
))
54 return (True, read_data
)
56 def write_file(file_to_write
, text
):
57 """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
59 f
= open(file_to_write
, 'w')
62 except Exception as e
:
63 return (False, str(e
))
67 def format_in(http_response
, schema
):
69 client_data
= http_response
.json()
70 js_v(client_data
, schema
)
71 #print "Input data: ", str(client_data)
72 return True, client_data
73 except js_e
.ValidationError
as exc
:
74 print "validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
75 return False, ("validate_in error, jsonschema exception ", exc
.message
, "at", exc
.path
)
77 def remove_extra_items(data
, schema
):
79 if type(data
) is tuple or type(data
) is list:
81 a
= remove_extra_items(d
, schema
['items'])
82 if a
is not None: deleted
.append(a
)
83 elif type(data
) is dict:
84 #TODO deal with patternProperties
85 if 'properties' not in schema
:
88 if k
not in schema
['properties'].keys():
92 a
= remove_extra_items(data
[k
], schema
['properties'][k
])
93 if a
is not None: deleted
.append({k
:a
})
94 if len(deleted
) == 0: return None
95 elif len(deleted
) == 1: return deleted
[0]
98 #def format_html2text(http_content):
99 # soup=BeautifulSoup(http_content)
100 # text = soup.p.get_text() + " " + soup.pre.get_text()
104 def delete_nulls(var
):
105 if type(var
) is dict:
107 if var
[k
] is None: del var
[k
]
108 elif type(var
[k
]) is dict or type(var
[k
]) is list or type(var
[k
]) is tuple:
109 if delete_nulls(var
[k
]): del var
[k
]
110 if len(var
) == 0: return True
111 elif type(var
) is list or type(var
) is tuple:
113 if type(k
) is dict: delete_nulls(k
)
114 if len(var
) == 0: return True
118 def convert_bandwidth(data
, reverse
=False):
119 '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
120 It assumes that bandwidth is well formed
122 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
123 'reverse': by default convert form str to int (Mbps), if True it convert from number to units
127 if type(data
) is dict:
128 for k
in data
.keys():
129 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
130 convert_bandwidth(data
[k
], reverse
)
131 if "bandwidth" in data
:
133 value
=str(data
["bandwidth"])
135 pos
= value
.find("bps")
137 if value
[pos
-1]=="G": data
["bandwidth"] = int(data
["bandwidth"][:pos
-1]) * 1000
138 elif value
[pos
-1]=="k": data
["bandwidth"]= int(data
["bandwidth"][:pos
-1]) / 1000
139 else: data
["bandwidth"]= int(data
["bandwidth"][:pos
-1])
141 value
= int(data
["bandwidth"])
142 if value
% 1000 == 0: data
["bandwidth"]=str(value
/1000) + " Gbps"
143 else: data
["bandwidth"]=str(value
) + " Mbps"
145 print "convert_bandwidth exception for type", type(data
["bandwidth"]), " data", data
["bandwidth"]
147 if type(data
) is tuple or type(data
) is list:
149 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
150 convert_bandwidth(k
, reverse
)
152 def convert_float_timestamp2str(var
):
153 '''Converts timestamps (created_at, modified_at fields) represented as float
154 to a string with the format '%Y-%m-%dT%H:%i:%s'
155 It enters recursively in the dict var finding this kind of variables
157 if type(var
) is dict:
158 for k
,v
in var
.items():
159 if type(v
) is float and k
in ("created_at", "modified_at"):
160 var
[k
] = time
.strftime("%Y-%m-%dT%H:%M:%S", time
.localtime(v
) )
161 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
162 convert_float_timestamp2str(v
)
163 if len(var
) == 0: return True
164 elif type(var
) is list or type(var
) is tuple:
166 convert_float_timestamp2str(v
)
168 def convert_datetime2str(var
):
169 '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
170 It enters recursively in the dict var finding this kind of variables
172 if type(var
) is dict:
173 for k
,v
in var
.items():
174 if type(v
) is datetime
.datetime
:
175 var
[k
]= v
.strftime('%Y-%m-%dT%H:%M:%S')
176 elif type(v
) is dict or type(v
) is list or type(v
) is tuple:
177 convert_datetime2str(v
)
178 if len(var
) == 0: return True
179 elif type(var
) is list or type(var
) is tuple:
181 convert_datetime2str(v
)
183 def convert_str2boolean(data
, items
):
184 '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
187 'data': dictionary variable to be checked. None or empty is considered valid
188 'items': tuple of keys to convert
192 if type(data
) is dict:
193 for k
in data
.keys():
194 if type(data
[k
]) is dict or type(data
[k
]) is tuple or type(data
[k
]) is list:
195 convert_str2boolean(data
[k
], items
)
197 if type(data
[k
]) is str:
198 if data
[k
]=="false" or data
[k
]=="False": data
[k
]=False
199 elif data
[k
]=="true" or data
[k
]=="True": data
[k
]=True
200 if type(data
) is tuple or type(data
) is list:
202 if type(k
) is dict or type(k
) is tuple or type(k
) is list:
203 convert_str2boolean(k
, items
)
205 def check_valid_uuid(uuid
):
206 id_schema
= {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
207 id_schema2
= {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
209 js_v(uuid
, id_schema
)
211 except js_e
.ValidationError
:
213 js_v(uuid
, id_schema2
)
215 except js_e
.ValidationError
:
220 def expand_brackets(text
):
222 Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
223 if no bracket is used it just return the a list with the single text
224 It uses recursivity to allow several [] in the text
228 start
= text
.find("[")
230 if start
< 0 or end
< 0:
233 for char
in text
[start
+1:end
]:
234 text_list
+= expand_brackets(text
[:start
] + char
+ text
[end
+1:])
237 def deprecated(message
):
238 def deprecated_decorator(func
):
239 def deprecated_func(*args
, **kwargs
):
240 warnings
.warn("{} is a deprecated function. {}".format(func
.__name
__, message
),
241 category
=DeprecationWarning,
243 warnings
.simplefilter('default', DeprecationWarning)
244 return func(*args
, **kwargs
)
245 return deprecated_func
246 return deprecated_decorator
249 def truncate(text
, max_length
=1024):
250 """Limit huge texts in number of characters"""
252 if text
and len(text
) >= max_length
:
253 return text
[:max_length
//2-3] + " ... " + text
[-max_length
//2+3:]
257 def merge_dicts(*dicts
, **kwargs
):
258 """Creates a new dict merging N others and keyword arguments.
259 Right-most dicts take precedence.
260 Keyword args take precedence.
263 lambda acc
, x
: acc
.update(x
) or acc
,
264 list(dicts
) + [kwargs
], {})
267 def remove_none_items(adict
):
268 """Return a similar dict without keys associated to None values"""
269 return {k
: v
for k
, v
in adict
.items() if v
is not None}
272 def filter_dict_keys(adict
, allow
):
273 """Return a similar dict, but just containing the explicitly allowed keys
276 adict (dict): Simple python dict data struct
277 allow (list): Explicits allowed keys
279 return {k
: v
for k
, v
in adict
.items() if k
in allow
}
282 def filter_out_dict_keys(adict
, deny
):
283 """Return a similar dict, but not containing the explicitly denied keys
286 adict (dict): Simple python dict data struct
287 deny (list): Explicits denied keys
289 return {k
: v
for k
, v
in adict
.items() if k
not in deny
}
292 def expand_joined_fields(record
):
293 """Given a db query result, explode the fields that contains `.` (join
297 >> expand_joined_fiels({'wim.id': 2})
301 for field
, value
in record
.items():
302 keys
= field
.split('.')
304 target
= reduce(lambda target
, key
: target
.setdefault(key
, {}),
306 target
[keys
[-1]] = value
311 def ensure(condition
, exception
):
312 """Raise an exception if condition is not met"""
317 def partition(predicate
, iterable
):
318 """Create two derived iterators from a single one
319 The first iterator created will loop thought the values where the function
320 predicate is True, the second one will iterate over the values where it is
323 iterable1
, iterable2
= tee(iterable
)
324 return filter(predicate
, iterable2
), filterfalse(predicate
, iterable1
)
327 def pipe(*functions
):
328 """Compose functions of one argument in the opposite order,
329 So pipe(f, g)(x) = g(f(x))
331 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
, x
)
334 def compose(*functions
):
335 """Compose functions of one argument,
336 So compose(f, g)(x) = f(g(x))
338 return lambda x
: reduce(lambda acc
, f
: f(acc
), functions
[::-1], x
)
341 def safe_get(target
, key_path
, default
=None):
342 """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
343 a nested dict if present, or the default value
345 keys
= key_path
.split('.')
346 target
= reduce(lambda acc
, key
: acc
.get(key
) or {}, keys
[:-1], target
)
347 return target
.get(keys
[-1], default
)