osm_ro/utils.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 utils is a module that implements functions that are used by all openmano modules,
  26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
  27 from dictionaries to appropriate database dictionaries, etc.
  28 '''
  29 __author__="Alfonso Tierno, Gerardo Garcia"
  30 __date__ ="$08-sep-2014 12:21:22$"
  31
  32 import datetime
  33 import time
  34 import warnings
  35 from functools import reduce, partial, wraps
  36 from itertools import tee
  37
  38 import six
  39 from six.moves import filter, filterfalse
  40
  41 from jsonschema import exceptions as js_e
  42 from jsonschema import validate as js_v
  43
  44 if six.PY3:
  45     from inspect import getfullargspec as getspec
  46 else:
  47     from inspect import getargspec as getspec
  48
  49 #from bs4 import BeautifulSoup
  50
  51 def read_file(file_to_read):
  52     """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
  53     try:
  54         f = open(file_to_read, 'r')
  55         read_data = f.read()
  56         f.close()
  57     except Exception as e:
  58         return (False, str(e))
  59
  60     return (True, read_data)
  61
  62 def write_file(file_to_write, text):
  63     """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
  64     try:
  65         f = open(file_to_write, 'w')
  66         f.write(text)
  67         f.close()
  68     except Exception as e:
  69         return (False, str(e))
  70
  71     return (True, None)
  72
  73 def format_in(http_response, schema):
  74     try:
  75         client_data = http_response.json()
  76         js_v(client_data, schema)
  77         #print "Input data: ", str(client_data)
  78         return True, client_data
  79     except js_e.ValidationError as exc:
  80         print "validate_in error, jsonschema exception ", exc.message, "at", exc.path
  81         return False, ("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  82
  83 def remove_extra_items(data, schema):
  84     deleted=[]
  85     if type(data) is tuple or type(data) is list:
  86         for d in data:
  87             a= remove_extra_items(d, schema['items'])
  88             if a is not None: deleted.append(a)
  89     elif type(data) is dict:
  90         #TODO deal with patternProperties
  91         if 'properties' not in schema:
  92             return None
  93         for k in data.keys():
  94             if k not in schema['properties'].keys():
  95                 del data[k]
  96                 deleted.append(k)
  97             else:
  98                 a = remove_extra_items(data[k], schema['properties'][k])
  99                 if a is not None:  deleted.append({k:a})
 100     if len(deleted) == 0: return None
 101     elif len(deleted) == 1: return deleted[0]
 102     else: return deleted
 103
 104 #def format_html2text(http_content):
 105 #    soup=BeautifulSoup(http_content)
 106 #    text = soup.p.get_text() + " " + soup.pre.get_text()
 107 #    return text
 108
 109
 110 def delete_nulls(var):
 111     if type(var) is dict:
 112         for k in var.keys():
 113             if var[k] is None: del var[k]
 114             elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple:
 115                 if delete_nulls(var[k]): del var[k]
 116         if len(var) == 0: return True
 117     elif type(var) is list or type(var) is tuple:
 118         for k in var:
 119             if type(k) is dict: delete_nulls(k)
 120         if len(var) == 0: return True
 121     return False
 122
 123
 124 def convert_bandwidth(data, reverse=False):
 125     '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
 126     It assumes that bandwidth is well formed
 127     Attributes:
 128         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
 129         'reverse': by default convert form str to int (Mbps), if True it convert from number to units
 130     Return:
 131         None
 132     '''
 133     if type(data) is dict:
 134         for k in data.keys():
 135             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 136                 convert_bandwidth(data[k], reverse)
 137         if "bandwidth" in data:
 138             try:
 139                 value=str(data["bandwidth"])
 140                 if not reverse:
 141                     pos = value.find("bps")
 142                     if pos>0:
 143                         if value[pos-1]=="G": data["bandwidth"] =  int(data["bandwidth"][:pos-1]) * 1000
 144                         elif value[pos-1]=="k": data["bandwidth"]= int(data["bandwidth"][:pos-1]) / 1000
 145                         else: data["bandwidth"]= int(data["bandwidth"][:pos-1])
 146                 else:
 147                     value = int(data["bandwidth"])
 148                     if value % 1000 == 0: data["bandwidth"]=str(value/1000) + " Gbps"
 149                     else: data["bandwidth"]=str(value) + " Mbps"
 150             except:
 151                 print "convert_bandwidth exception for type", type(data["bandwidth"]), " data", data["bandwidth"]
 152                 return
 153     if type(data) is tuple or type(data) is list:
 154         for k in data:
 155             if type(k) is dict or type(k) is tuple or type(k) is list:
 156                 convert_bandwidth(k, reverse)
 157
 158 def convert_float_timestamp2str(var):
 159     '''Converts timestamps (created_at, modified_at fields) represented as float
 160     to a string with the format '%Y-%m-%dT%H:%i:%s'
 161     It enters recursively in the dict var finding this kind of variables
 162     '''
 163     if type(var) is dict:
 164         for k,v in var.items():
 165             if type(v) is float and k in ("created_at", "modified_at"):
 166                 var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) )
 167             elif type(v) is dict or type(v) is list or type(v) is tuple:
 168                 convert_float_timestamp2str(v)
 169         if len(var) == 0: return True
 170     elif type(var) is list or type(var) is tuple:
 171         for v in var:
 172             convert_float_timestamp2str(v)
 173
 174 def convert_datetime2str(var):
 175     '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
 176     It enters recursively in the dict var finding this kind of variables
 177     '''
 178     if type(var) is dict:
 179         for k,v in var.items():
 180             if type(v) is datetime.datetime:
 181                 var[k]= v.strftime('%Y-%m-%dT%H:%M:%S')
 182             elif type(v) is dict or type(v) is list or type(v) is tuple:
 183                 convert_datetime2str(v)
 184         if len(var) == 0: return True
 185     elif type(var) is list or type(var) is tuple:
 186         for v in var:
 187             convert_datetime2str(v)
 188
 189 def convert_str2boolean(data, items):
 190     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
 191     Done recursively
 192     Attributes:
 193         'data': dictionary variable to be checked. None or empty is considered valid
 194         'items': tuple of keys to convert
 195     Return:
 196         None
 197     '''
 198     if type(data) is dict:
 199         for k in data.keys():
 200             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 201                 convert_str2boolean(data[k], items)
 202             if k in items:
 203                 if type(data[k]) is str:
 204                     if   data[k]=="false" or data[k]=="False": data[k]=False
 205                     elif data[k]=="true"  or data[k]=="True":  data[k]=True
 206     if type(data) is tuple or type(data) is list:
 207         for k in data:
 208             if type(k) is dict or type(k) is tuple or type(k) is list:
 209                 convert_str2boolean(k, items)
 210
 211 def check_valid_uuid(uuid):
 212     id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
 213     id_schema2 = {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
 214     try:
 215         js_v(uuid, id_schema)
 216         return True
 217     except js_e.ValidationError:
 218         try:
 219             js_v(uuid, id_schema2)
 220             return True
 221         except js_e.ValidationError:
 222             return False
 223     return False
 224
 225
 226 def expand_brackets(text):
 227     """
 228     Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
 229     if no bracket is used it just return the a list with the single text
 230     It uses recursivity to allow several [] in the text
 231     :param text:
 232     :return:
 233     """
 234     if text is None:
 235         return (None, )
 236     start = text.find("[")
 237     end = text.find("]")
 238     if start < 0 or end < 0:
 239         return [text]
 240     text_list = []
 241     for char in text[start+1:end]:
 242         text_list += expand_brackets(text[:start] + char + text[end+1:])
 243     return text_list
 244
 245 def deprecated(message):
 246   def deprecated_decorator(func):
 247       def deprecated_func(*args, **kwargs):
 248           warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
 249                         category=DeprecationWarning,
 250                         stacklevel=2)
 251           warnings.simplefilter('default', DeprecationWarning)
 252           return func(*args, **kwargs)
 253       return deprecated_func
 254   return deprecated_decorator
 255
 256
 257 def truncate(text, max_length=1024):
 258     """Limit huge texts in number of characters"""
 259     text = str(text)
 260     if text and len(text) >= max_length:
 261         return text[:max_length//2-3] + " ... " + text[-max_length//2+3:]
 262     return text
 263
 264
 265 def merge_dicts(*dicts, **kwargs):
 266     """Creates a new dict merging N others and keyword arguments.
 267     Right-most dicts take precedence.
 268     Keyword args take precedence.
 269     """
 270     return reduce(
 271         lambda acc, x: acc.update(x) or acc,
 272         list(dicts) + [kwargs], {})
 273
 274
 275 def remove_none_items(adict):
 276     """Return a similar dict without keys associated to None values"""
 277     return {k: v for k, v in adict.items() if v is not None}
 278
 279
 280 def filter_dict_keys(adict, allow):
 281     """Return a similar dict, but just containing the explicitly allowed keys
 282
 283     Arguments:
 284         adict (dict): Simple python dict data struct
 285         allow (list): Explicits allowed keys
 286     """
 287     return {k: v for k, v in adict.items() if k in allow}
 288
 289
 290 def filter_out_dict_keys(adict, deny):
 291     """Return a similar dict, but not containing the explicitly denied keys
 292
 293     Arguments:
 294         adict (dict): Simple python dict data struct
 295         deny (list): Explicits denied keys
 296     """
 297     return {k: v for k, v in adict.items() if k not in deny}
 298
 299
 300 def expand_joined_fields(record):
 301     """Given a db query result, explode the fields that contains `.` (join
 302     operations).
 303
 304     Example
 305         >> expand_joined_fiels({'wim.id': 2})
 306         # {'wim': {'id': 2}}
 307     """
 308     result = {}
 309     for field, value in record.items():
 310         keys = field.split('.')
 311         target = result
 312         target = reduce(lambda target, key: target.setdefault(key, {}),
 313                         keys[:-1], result)
 314         target[keys[-1]] = value
 315
 316     return result
 317
 318
 319 def ensure(condition, exception):
 320     """Raise an exception if condition is not met"""
 321     if not condition:
 322         raise exception
 323
 324
 325 def partition(predicate, iterable):
 326     """Create two derived iterators from a single one
 327     The first iterator created will loop thought the values where the function
 328     predicate is True, the second one will iterate over the values where it is
 329     false.
 330     """
 331     iterable1, iterable2 = tee(iterable)
 332     return filter(predicate, iterable2), filterfalse(predicate, iterable1)
 333
 334
 335 def pipe(*functions):
 336     """Compose functions of one argument in the opposite order,
 337     So pipe(f, g)(x) = g(f(x))
 338     """
 339     return lambda x: reduce(lambda acc, f: f(acc), functions, x)
 340
 341
 342 def compose(*functions):
 343     """Compose functions of one argument,
 344     So compose(f, g)(x) = f(g(x))
 345     """
 346     return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x)
 347
 348
 349 def safe_get(target, key_path, default=None):
 350     """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
 351     a nested dict if present, or the default value
 352     """
 353     keys = key_path.split('.')
 354     target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target)
 355     return target.get(keys[-1], default)
 356
 357
 358 class Attempt(object):
 359     """Auxiliary class to be used in an attempt to retry executing a failing
 360     procedure
 361
 362     Attributes:
 363         count (int): 0-based "retries" counter
 364         max_attempts (int): maximum number of "retries" allowed
 365         info (dict): extra information about the specific attempt
 366             (can be used to produce more meaningful error messages)
 367     """
 368     __slots__ = ('count', 'max', 'info')
 369
 370     MAX = 3
 371
 372     def __init__(self, count=0, max_attempts=MAX, info=None):
 373         self.count = count
 374         self.max = max_attempts
 375         self.info = info or {}
 376
 377     @property
 378     def countdown(self):
 379         """Like count, but in the opposite direction"""
 380         return self.max - self.count
 381
 382     @property
 383     def number(self):
 384         """1-based counter"""
 385         return self.count + 1
 386
 387
 388 def inject_args(fn=None, **args):
 389     """Partially apply keyword arguments in a function, but only if the function
 390     define them in the first place
 391     """
 392     if fn is None:  # Allows calling the decorator directly or with parameters
 393         return partial(inject_args, **args)
 394
 395     spec = getspec(fn)
 396     return wraps(fn)(partial(fn, **filter_dict_keys(args, spec.args)))
 397
 398
 399 def get_arg(name, fn, args, kwargs):
 400     """Find the value of an argument for a function, given its argument list.
 401
 402     This function can be used to display more meaningful errors for debugging
 403     """
 404     if name in kwargs:
 405         return kwargs[name]
 406
 407     spec = getspec(fn)
 408     if name in spec.args:
 409         i = spec.args.index(name)
 410         return args[i] if i < len(args) else None
 411
 412     return None