osm_ro/utils.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 utils is a module that implements functions that are used by all openmano modules,
  26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
  27 from dictionaries to appropriate database dictionaries, etc.
  28 '''
  29 __author__="Alfonso Tierno, Gerardo Garcia"
  30 __date__ ="$08-sep-2014 12:21:22$"
  31
  32 import datetime
  33 import time
  34 import warnings
  35 from functools import reduce, partial, wraps
  36 from itertools import tee
  37
  38 import six
  39 from six.moves import filter, filterfalse
  40
  41 from jsonschema import exceptions as js_e
  42 from jsonschema import validate as js_v
  43
  44 if six.PY3:
  45     from inspect import getfullargspec as getspec
  46 else:
  47     from inspect import getargspec as getspec
  48
  49 #from bs4 import BeautifulSoup
  50
  51 def read_file(file_to_read):
  52     """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
  53     try:
  54         f = open(file_to_read, 'r')
  55         read_data = f.read()
  56         f.close()
  57     except Exception as e:
  58         return (False, str(e))
  59
  60     return (True, read_data)
  61
  62 def write_file(file_to_write, text):
  63     """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
  64     try:
  65         f = open(file_to_write, 'w')
  66         f.write(text)
  67         f.close()
  68     except Exception as e:
  69         return (False, str(e))
  70
  71     return (True, None)
  72
  73 def format_in(http_response, schema):
  74     try:
  75         client_data = http_response.json()
  76         js_v(client_data, schema)
  77         #print "Input data: ", str(client_data)
  78         return True, client_data
  79     except js_e.ValidationError as exc:
  80         print "validate_in error, jsonschema exception ", exc.message, "at", exc.path
  81         return False, ("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  82
  83 def remove_extra_items(data, schema):
  84     deleted = []
  85     if type(data) is tuple or type(data) is list:
  86         for d in data:
  87             a = remove_extra_items(d, schema['items'])
  88             if a is not None:
  89                 deleted.append(a)
  90     elif type(data) is dict:
  91         # TODO deal with patternProperties
  92         if 'properties' not in schema:
  93             return None
  94         for k in data.keys():
  95             if k in schema['properties'].keys():
  96                 a = remove_extra_items(data[k], schema['properties'][k])
  97                 if a is not None:
  98                     deleted.append({k: a})
  99             elif not schema.get('additionalProperties'):
 100                 del data[k]
 101                 deleted.append(k)
 102     if len(deleted) == 0:
 103         return None
 104     elif len(deleted) == 1:
 105         return deleted[0]
 106
 107     return deleted
 108
 109 #def format_html2text(http_content):
 110 #    soup=BeautifulSoup(http_content)
 111 #    text = soup.p.get_text() + " " + soup.pre.get_text()
 112 #    return text
 113
 114
 115 def delete_nulls(var):
 116     if type(var) is dict:
 117         for k in var.keys():
 118             if var[k] is None: del var[k]
 119             elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple:
 120                 if delete_nulls(var[k]): del var[k]
 121         if len(var) == 0: return True
 122     elif type(var) is list or type(var) is tuple:
 123         for k in var:
 124             if type(k) is dict: delete_nulls(k)
 125         if len(var) == 0: return True
 126     return False
 127
 128
 129 def convert_bandwidth(data, reverse=False):
 130     '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
 131     It assumes that bandwidth is well formed
 132     Attributes:
 133         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
 134         'reverse': by default convert form str to int (Mbps), if True it convert from number to units
 135     Return:
 136         None
 137     '''
 138     if type(data) is dict:
 139         for k in data.keys():
 140             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 141                 convert_bandwidth(data[k], reverse)
 142         if "bandwidth" in data:
 143             try:
 144                 value=str(data["bandwidth"])
 145                 if not reverse:
 146                     pos = value.find("bps")
 147                     if pos>0:
 148                         if value[pos-1]=="G": data["bandwidth"] =  int(data["bandwidth"][:pos-1]) * 1000
 149                         elif value[pos-1]=="k": data["bandwidth"]= int(data["bandwidth"][:pos-1]) / 1000
 150                         else: data["bandwidth"]= int(data["bandwidth"][:pos-1])
 151                 else:
 152                     value = int(data["bandwidth"])
 153                     if value % 1000 == 0: data["bandwidth"]=str(value/1000) + " Gbps"
 154                     else: data["bandwidth"]=str(value) + " Mbps"
 155             except:
 156                 print "convert_bandwidth exception for type", type(data["bandwidth"]), " data", data["bandwidth"]
 157                 return
 158     if type(data) is tuple or type(data) is list:
 159         for k in data:
 160             if type(k) is dict or type(k) is tuple or type(k) is list:
 161                 convert_bandwidth(k, reverse)
 162
 163 def convert_float_timestamp2str(var):
 164     '''Converts timestamps (created_at, modified_at fields) represented as float
 165     to a string with the format '%Y-%m-%dT%H:%i:%s'
 166     It enters recursively in the dict var finding this kind of variables
 167     '''
 168     if type(var) is dict:
 169         for k,v in var.items():
 170             if type(v) is float and k in ("created_at", "modified_at"):
 171                 var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) )
 172             elif type(v) is dict or type(v) is list or type(v) is tuple:
 173                 convert_float_timestamp2str(v)
 174         if len(var) == 0: return True
 175     elif type(var) is list or type(var) is tuple:
 176         for v in var:
 177             convert_float_timestamp2str(v)
 178
 179 def convert_datetime2str(var):
 180     '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
 181     It enters recursively in the dict var finding this kind of variables
 182     '''
 183     if type(var) is dict:
 184         for k,v in var.items():
 185             if type(v) is datetime.datetime:
 186                 var[k]= v.strftime('%Y-%m-%dT%H:%M:%S')
 187             elif type(v) is dict or type(v) is list or type(v) is tuple:
 188                 convert_datetime2str(v)
 189         if len(var) == 0: return True
 190     elif type(var) is list or type(var) is tuple:
 191         for v in var:
 192             convert_datetime2str(v)
 193
 194 def convert_str2boolean(data, items):
 195     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
 196     Done recursively
 197     Attributes:
 198         'data': dictionary variable to be checked. None or empty is considered valid
 199         'items': tuple of keys to convert
 200     Return:
 201         None
 202     '''
 203     if type(data) is dict:
 204         for k in data.keys():
 205             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 206                 convert_str2boolean(data[k], items)
 207             if k in items:
 208                 if type(data[k]) is str:
 209                     if   data[k]=="false" or data[k]=="False": data[k]=False
 210                     elif data[k]=="true"  or data[k]=="True":  data[k]=True
 211     if type(data) is tuple or type(data) is list:
 212         for k in data:
 213             if type(k) is dict or type(k) is tuple or type(k) is list:
 214                 convert_str2boolean(k, items)
 215
 216 def check_valid_uuid(uuid):
 217     id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
 218     id_schema2 = {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
 219     try:
 220         js_v(uuid, id_schema)
 221         return True
 222     except js_e.ValidationError:
 223         try:
 224             js_v(uuid, id_schema2)
 225             return True
 226         except js_e.ValidationError:
 227             return False
 228     return False
 229
 230
 231 def expand_brackets(text):
 232     """
 233     Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
 234     if no bracket is used it just return the a list with the single text
 235     It uses recursivity to allow several [] in the text
 236     :param text:
 237     :return:
 238     """
 239     if text is None:
 240         return (None, )
 241     start = text.find("[")
 242     end = text.find("]")
 243     if start < 0 or end < 0:
 244         return [text]
 245     text_list = []
 246     for char in text[start+1:end]:
 247         text_list += expand_brackets(text[:start] + char + text[end+1:])
 248     return text_list
 249
 250 def deprecated(message):
 251   def deprecated_decorator(func):
 252       def deprecated_func(*args, **kwargs):
 253           warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
 254                         category=DeprecationWarning,
 255                         stacklevel=2)
 256           warnings.simplefilter('default', DeprecationWarning)
 257           return func(*args, **kwargs)
 258       return deprecated_func
 259   return deprecated_decorator
 260
 261
 262 def truncate(text, max_length=1024):
 263     """Limit huge texts in number of characters"""
 264     text = str(text)
 265     if text and len(text) >= max_length:
 266         return text[:max_length//2-3] + " ... " + text[-max_length//2+3:]
 267     return text
 268
 269
 270 def merge_dicts(*dicts, **kwargs):
 271     """Creates a new dict merging N others and keyword arguments.
 272     Right-most dicts take precedence.
 273     Keyword args take precedence.
 274     """
 275     return reduce(
 276         lambda acc, x: acc.update(x) or acc,
 277         list(dicts) + [kwargs], {})
 278
 279
 280 def remove_none_items(adict):
 281     """Return a similar dict without keys associated to None values"""
 282     return {k: v for k, v in adict.items() if v is not None}
 283
 284
 285 def filter_dict_keys(adict, allow):
 286     """Return a similar dict, but just containing the explicitly allowed keys
 287
 288     Arguments:
 289         adict (dict): Simple python dict data struct
 290         allow (list): Explicits allowed keys
 291     """
 292     return {k: v for k, v in adict.items() if k in allow}
 293
 294
 295 def filter_out_dict_keys(adict, deny):
 296     """Return a similar dict, but not containing the explicitly denied keys
 297
 298     Arguments:
 299         adict (dict): Simple python dict data struct
 300         deny (list): Explicits denied keys
 301     """
 302     return {k: v for k, v in adict.items() if k not in deny}
 303
 304
 305 def expand_joined_fields(record):
 306     """Given a db query result, explode the fields that contains `.` (join
 307     operations).
 308
 309     Example
 310         >> expand_joined_fiels({'wim.id': 2})
 311         # {'wim': {'id': 2}}
 312     """
 313     result = {}
 314     for field, value in record.items():
 315         keys = field.split('.')
 316         target = result
 317         target = reduce(lambda target, key: target.setdefault(key, {}),
 318                         keys[:-1], result)
 319         target[keys[-1]] = value
 320
 321     return result
 322
 323
 324 def ensure(condition, exception):
 325     """Raise an exception if condition is not met"""
 326     if not condition:
 327         raise exception
 328
 329
 330 def partition(predicate, iterable):
 331     """Create two derived iterators from a single one
 332     The first iterator created will loop thought the values where the function
 333     predicate is True, the second one will iterate over the values where it is
 334     false.
 335     """
 336     iterable1, iterable2 = tee(iterable)
 337     return filter(predicate, iterable2), filterfalse(predicate, iterable1)
 338
 339
 340 def pipe(*functions):
 341     """Compose functions of one argument in the opposite order,
 342     So pipe(f, g)(x) = g(f(x))
 343     """
 344     return lambda x: reduce(lambda acc, f: f(acc), functions, x)
 345
 346
 347 def compose(*functions):
 348     """Compose functions of one argument,
 349     So compose(f, g)(x) = f(g(x))
 350     """
 351     return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x)
 352
 353
 354 def safe_get(target, key_path, default=None):
 355     """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
 356     a nested dict if present, or the default value
 357     """
 358     keys = key_path.split('.')
 359     target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target)
 360     return target.get(keys[-1], default)
 361
 362
 363 class Attempt(object):
 364     """Auxiliary class to be used in an attempt to retry executing a failing
 365     procedure
 366
 367     Attributes:
 368         count (int): 0-based "retries" counter
 369         max_attempts (int): maximum number of "retries" allowed
 370         info (dict): extra information about the specific attempt
 371             (can be used to produce more meaningful error messages)
 372     """
 373     __slots__ = ('count', 'max', 'info')
 374
 375     MAX = 3
 376
 377     def __init__(self, count=0, max_attempts=MAX, info=None):
 378         self.count = count
 379         self.max = max_attempts
 380         self.info = info or {}
 381
 382     @property
 383     def countdown(self):
 384         """Like count, but in the opposite direction"""
 385         return self.max - self.count
 386
 387     @property
 388     def number(self):
 389         """1-based counter"""
 390         return self.count + 1
 391
 392
 393 def inject_args(fn=None, **args):
 394     """Partially apply keyword arguments in a function, but only if the function
 395     define them in the first place
 396     """
 397     if fn is None:  # Allows calling the decorator directly or with parameters
 398         return partial(inject_args, **args)
 399
 400     spec = getspec(fn)
 401     return wraps(fn)(partial(fn, **filter_dict_keys(args, spec.args)))
 402
 403
 404 def get_arg(name, fn, args, kwargs):
 405     """Find the value of an argument for a function, given its argument list.
 406
 407     This function can be used to display more meaningful errors for debugging
 408     """
 409     if name in kwargs:
 410         return kwargs[name]
 411
 412     spec = getspec(fn)
 413     if name in spec.args:
 414         i = spec.args.index(name)
 415         return args[i] if i < len(args) else None
 416
 417     return None