RO/osm_ro/utils.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 utils is a module that implements functions that are used by all openmano modules,
  26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
  27 from dictionaries to appropriate database dictionaries, etc.
  28 '''
  29 __author__="Alfonso Tierno, Gerardo Garcia"
  30 __date__ ="$08-sep-2014 12:21:22$"
  31
  32 import datetime
  33 import time
  34 import warnings
  35 from functools import reduce, partial, wraps
  36 from itertools import tee
  37
  38 from itertools import filterfalse
  39
  40 from jsonschema import exceptions as js_e
  41 from jsonschema import validate as js_v
  42
  43 from inspect import getfullargspec as getspec
  44
  45 #from bs4 import BeautifulSoup
  46
  47 def read_file(file_to_read):
  48     """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
  49     try:
  50         f = open(file_to_read, 'r')
  51         read_data = f.read()
  52         f.close()
  53     except Exception as e:
  54         return (False, str(e))
  55
  56     return (True, read_data)
  57
  58 def write_file(file_to_write, text):
  59     """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
  60     try:
  61         f = open(file_to_write, 'w')
  62         f.write(text)
  63         f.close()
  64     except Exception as e:
  65         return (False, str(e))
  66
  67     return (True, None)
  68
  69 def format_in(http_response, schema):
  70     try:
  71         client_data = http_response.json()
  72         js_v(client_data, schema)
  73         #print "Input data: ", str(client_data)
  74         return True, client_data
  75     except js_e.ValidationError as exc:
  76         print("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  77         return False, ("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  78
  79 def remove_extra_items(data, schema):
  80     deleted = []
  81     if isinstance(data, (tuple, list)):
  82         for d in data:
  83             a = remove_extra_items(d, schema['items'])
  84             if a:
  85                 deleted.append(a)
  86     elif isinstance(data, dict):
  87         # TODO deal with patternProperties
  88         if 'properties' not in schema:
  89             return None
  90         to_delete = []
  91         for k in data.keys():
  92             if k in schema['properties']:
  93                 a = remove_extra_items(data[k], schema['properties'][k])
  94                 if a:
  95                     deleted.append({k: a})
  96             elif not schema.get('additionalProperties'):
  97                 to_delete.append(k)
  98                 deleted.append(k)
  99         for k in to_delete:
 100             del data[k]
 101     if len(deleted) == 0:
 102         return None
 103     elif len(deleted) == 1:
 104         return deleted[0]
 105
 106     return deleted
 107
 108 #def format_html2text(http_content):
 109 #    soup=BeautifulSoup(http_content)
 110 #    text = soup.p.get_text() + " " + soup.pre.get_text()
 111 #    return text
 112
 113
 114 def delete_nulls(var):
 115     if isinstance(var, dict):
 116         to_delete = []
 117         for k in var.keys():
 118             if var[k] is None:
 119                 to_delete.append([k])
 120             elif isinstance(var[k], (dict, list, tuple)):
 121                 if delete_nulls(var[k]):
 122                     to_delete.append(k)
 123         for k in to_delete:
 124             del var[k]
 125         if len(var) == 0:
 126             return True
 127     elif isinstance(var, (list, tuple)):
 128         for k in var:
 129             if isinstance(k, dict):
 130                 delete_nulls(k)
 131         if len(var) == 0:
 132             return True
 133     return False
 134
 135
 136 def convert_bandwidth(data, reverse=False):
 137     '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
 138     It assumes that bandwidth is well formed
 139     Attributes:
 140         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
 141         'reverse': by default convert form str to int (Mbps), if True it convert from number to units
 142     Return:
 143         None
 144     '''
 145     if isinstance(data, dict):
 146         for k in data.keys():
 147             if isinstance(data[k], (dict, tuple, list)):
 148                 convert_bandwidth(data[k], reverse)
 149         if "bandwidth" in data:
 150             try:
 151                 value=str(data["bandwidth"])
 152                 if not reverse:
 153                     pos = value.find("bps")
 154                     if pos>0:
 155                         if value[pos-1]=="G":
 156                             data["bandwidth"] = int(data["bandwidth"][:pos-1]) * 1000
 157                         elif value[pos-1]=="k":
 158                             data["bandwidth"]= int(data["bandwidth"][:pos-1]) // 1000
 159                         else:
 160                             data["bandwidth"]= int(data["bandwidth"][:pos])
 161                 else:
 162                     value = int(data["bandwidth"])
 163                     if value % 1000 == 0 and value > 1000:
 164                         data["bandwidth"] = str(value // 1000) + " Gbps"
 165                     else:
 166                         data["bandwidth"] = str(value) + " Mbps"
 167             except:
 168                 print("convert_bandwidth exception for type", type(data["bandwidth"]), " data", data["bandwidth"])
 169                 return
 170     if isinstance(data, (tuple, list)):
 171         for k in data:
 172             if isinstance(k, (dict, tuple, list)):
 173                 convert_bandwidth(k, reverse)
 174
 175 def convert_float_timestamp2str(var):
 176     '''Converts timestamps (created_at, modified_at fields) represented as float
 177     to a string with the format '%Y-%m-%dT%H:%i:%s'
 178     It enters recursively in the dict var finding this kind of variables
 179     '''
 180     if type(var) is dict:
 181         for k,v in var.items():
 182             if type(v) is float and k in ("created_at", "modified_at"):
 183                 var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) )
 184             elif type(v) is dict or type(v) is list or type(v) is tuple:
 185                 convert_float_timestamp2str(v)
 186         if len(var) == 0: return True
 187     elif type(var) is list or type(var) is tuple:
 188         for v in var:
 189             convert_float_timestamp2str(v)
 190
 191 def convert_datetime2str(var):
 192     '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
 193     It enters recursively in the dict var finding this kind of variables
 194     '''
 195     if type(var) is dict:
 196         for k,v in var.items():
 197             if type(v) is datetime.datetime:
 198                 var[k]= v.strftime('%Y-%m-%dT%H:%M:%S')
 199             elif type(v) is dict or type(v) is list or type(v) is tuple:
 200                 convert_datetime2str(v)
 201         if len(var) == 0: return True
 202     elif type(var) is list or type(var) is tuple:
 203         for v in var:
 204             convert_datetime2str(v)
 205
 206 def convert_str2boolean(data, items):
 207     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
 208     Done recursively
 209     Attributes:
 210         'data': dictionary variable to be checked. None or empty is considered valid
 211         'items': tuple of keys to convert
 212     Return:
 213         None
 214     '''
 215     if type(data) is dict:
 216         for k in data.keys():
 217             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 218                 convert_str2boolean(data[k], items)
 219             if k in items:
 220                 if type(data[k]) is str:
 221                     if   data[k]=="false" or data[k]=="False": data[k]=False
 222                     elif data[k]=="true"  or data[k]=="True":  data[k]=True
 223     if type(data) is tuple or type(data) is list:
 224         for k in data:
 225             if type(k) is dict or type(k) is tuple or type(k) is list:
 226                 convert_str2boolean(k, items)
 227
 228 def check_valid_uuid(uuid):
 229     id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
 230     id_schema2 = {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
 231     try:
 232         js_v(uuid, id_schema)
 233         return True
 234     except js_e.ValidationError:
 235         try:
 236             js_v(uuid, id_schema2)
 237             return True
 238         except js_e.ValidationError:
 239             return False
 240     return False
 241
 242
 243 def expand_brackets(text):
 244     """
 245     Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
 246     if no bracket is used it just return the a list with the single text
 247     It uses recursivity to allow several [] in the text
 248     :param text:
 249     :return:
 250     """
 251     if text is None:
 252         return (None, )
 253     start = text.find("[")
 254     end = text.find("]")
 255     if start < 0 or end < 0:
 256         return [text]
 257     text_list = []
 258     for char in text[start+1:end]:
 259         text_list += expand_brackets(text[:start] + char + text[end+1:])
 260     return text_list
 261
 262 def deprecated(message):
 263   def deprecated_decorator(func):
 264       def deprecated_func(*args, **kwargs):
 265           warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
 266                         category=DeprecationWarning,
 267                         stacklevel=2)
 268           warnings.simplefilter('default', DeprecationWarning)
 269           return func(*args, **kwargs)
 270       return deprecated_func
 271   return deprecated_decorator
 272
 273
 274 def truncate(text, max_length=1024):
 275     """Limit huge texts in number of characters"""
 276     text = str(text)
 277     if text and len(text) >= max_length:
 278         return text[:max_length//2-3] + " ... " + text[-max_length//2+3:]
 279     return text
 280
 281
 282 def merge_dicts(*dicts, **kwargs):
 283     """Creates a new dict merging N others and keyword arguments.
 284     Right-most dicts take precedence.
 285     Keyword args take precedence.
 286     """
 287     return reduce(
 288         lambda acc, x: acc.update(x) or acc,
 289         list(dicts) + [kwargs], {})
 290
 291
 292 def remove_none_items(adict):
 293     """Return a similar dict without keys associated to None values"""
 294     return {k: v for k, v in adict.items() if v is not None}
 295
 296
 297 def filter_dict_keys(adict, allow):
 298     """Return a similar dict, but just containing the explicitly allowed keys
 299
 300     Arguments:
 301         adict (dict): Simple python dict data struct
 302         allow (list): Explicits allowed keys
 303     """
 304     return {k: v for k, v in adict.items() if k in allow}
 305
 306
 307 def filter_out_dict_keys(adict, deny):
 308     """Return a similar dict, but not containing the explicitly denied keys
 309
 310     Arguments:
 311         adict (dict): Simple python dict data struct
 312         deny (list): Explicits denied keys
 313     """
 314     return {k: v for k, v in adict.items() if k not in deny}
 315
 316
 317 def expand_joined_fields(record):
 318     """Given a db query result, explode the fields that contains `.` (join
 319     operations).
 320
 321     Example
 322         >> expand_joined_fiels({'wim.id': 2})
 323         # {'wim': {'id': 2}}
 324     """
 325     result = {}
 326     for field, value in record.items():
 327         keys = field.split('.')
 328         target = result
 329         target = reduce(lambda target, key: target.setdefault(key, {}),
 330                         keys[:-1], result)
 331         target[keys[-1]] = value
 332
 333     return result
 334
 335
 336 def ensure(condition, exception):
 337     """Raise an exception if condition is not met"""
 338     if not condition:
 339         raise exception
 340
 341
 342 def partition(predicate, iterable):
 343     """Create two derived iterators from a single one
 344     The first iterator created will loop thought the values where the function
 345     predicate is True, the second one will iterate over the values where it is
 346     false.
 347     """
 348     iterable1, iterable2 = tee(iterable)
 349     return filter(predicate, iterable2), filterfalse(predicate, iterable1)
 350
 351
 352 def pipe(*functions):
 353     """Compose functions of one argument in the opposite order,
 354     So pipe(f, g)(x) = g(f(x))
 355     """
 356     return lambda x: reduce(lambda acc, f: f(acc), functions, x)
 357
 358
 359 def compose(*functions):
 360     """Compose functions of one argument,
 361     So compose(f, g)(x) = f(g(x))
 362     """
 363     return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x)
 364
 365
 366 def safe_get(target, key_path, default=None):
 367     """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
 368     a nested dict if present, or the default value
 369     """
 370     keys = key_path.split('.')
 371     target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target)
 372     return target.get(keys[-1], default)
 373
 374
 375 class Attempt(object):
 376     """Auxiliary class to be used in an attempt to retry executing a failing
 377     procedure
 378
 379     Attributes:
 380         count (int): 0-based "retries" counter
 381         max_attempts (int): maximum number of "retries" allowed
 382         info (dict): extra information about the specific attempt
 383             (can be used to produce more meaningful error messages)
 384     """
 385     __slots__ = ('count', 'max', 'info')
 386
 387     MAX = 3
 388
 389     def __init__(self, count=0, max_attempts=MAX, info=None):
 390         self.count = count
 391         self.max = max_attempts
 392         self.info = info or {}
 393
 394     @property
 395     def countdown(self):
 396         """Like count, but in the opposite direction"""
 397         return self.max - self.count
 398
 399     @property
 400     def number(self):
 401         """1-based counter"""
 402         return self.count + 1
 403
 404
 405 def inject_args(fn=None, **args):
 406     """Partially apply keyword arguments in a function, but only if the function
 407     define them in the first place
 408     """
 409     if fn is None:  # Allows calling the decorator directly or with parameters
 410         return partial(inject_args, **args)
 411
 412     spec = getspec(fn)
 413     return wraps(fn)(partial(fn, **filter_dict_keys(args, spec.args)))
 414
 415
 416 def get_arg(name, fn, args, kwargs):
 417     """Find the value of an argument for a function, given its argument list.
 418
 419     This function can be used to display more meaningful errors for debugging
 420     """
 421     if name in kwargs:
 422         return kwargs[name]
 423
 424     spec = getspec(fn)
 425     if name in spec.args:
 426         i = spec.args.index(name)
 427         return args[i] if i < len(args) else None
 428
 429     return None