osm_ro/utils.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 utils is a module that implements functions that are used by all openmano modules,
  26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
  27 from dictionaries to appropriate database dictionaries, etc.
  28 '''
  29 __author__="Alfonso Tierno, Gerardo Garcia"
  30 __date__ ="$08-sep-2014 12:21:22$"
  31
  32 import datetime
  33 import time
  34 import warnings
  35 from functools import reduce
  36 from itertools import tee
  37
  38 from six.moves import filter, filterfalse
  39
  40 from jsonschema import exceptions as js_e
  41 from jsonschema import validate as js_v
  42
  43 #from bs4 import BeautifulSoup
  44
  45 def read_file(file_to_read):
  46     """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
  47     try:
  48         f = open(file_to_read, 'r')
  49         read_data = f.read()
  50         f.close()
  51     except Exception as e:
  52         return (False, str(e))
  53
  54     return (True, read_data)
  55
  56 def write_file(file_to_write, text):
  57     """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
  58     try:
  59         f = open(file_to_write, 'w')
  60         f.write(text)
  61         f.close()
  62     except Exception as e:
  63         return (False, str(e))
  64
  65     return (True, None)
  66
  67 def format_in(http_response, schema):
  68     try:
  69         client_data = http_response.json()
  70         js_v(client_data, schema)
  71         #print "Input data: ", str(client_data)
  72         return True, client_data
  73     except js_e.ValidationError as exc:
  74         print "validate_in error, jsonschema exception ", exc.message, "at", exc.path
  75         return False, ("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  76
  77 def remove_extra_items(data, schema):
  78     deleted=[]
  79     if type(data) is tuple or type(data) is list:
  80         for d in data:
  81             a= remove_extra_items(d, schema['items'])
  82             if a is not None: deleted.append(a)
  83     elif type(data) is dict:
  84         #TODO deal with patternProperties
  85         if 'properties' not in schema:
  86             return None
  87         for k in data.keys():
  88             if k not in schema['properties'].keys():
  89                 del data[k]
  90                 deleted.append(k)
  91             else:
  92                 a = remove_extra_items(data[k], schema['properties'][k])
  93                 if a is not None:  deleted.append({k:a})
  94     if len(deleted) == 0: return None
  95     elif len(deleted) == 1: return deleted[0]
  96     else: return deleted
  97
  98 #def format_html2text(http_content):
  99 #    soup=BeautifulSoup(http_content)
 100 #    text = soup.p.get_text() + " " + soup.pre.get_text()
 101 #    return text
 102
 103
 104 def delete_nulls(var):
 105     if type(var) is dict:
 106         for k in var.keys():
 107             if var[k] is None: del var[k]
 108             elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple:
 109                 if delete_nulls(var[k]): del var[k]
 110         if len(var) == 0: return True
 111     elif type(var) is list or type(var) is tuple:
 112         for k in var:
 113             if type(k) is dict: delete_nulls(k)
 114         if len(var) == 0: return True
 115     return False
 116
 117
 118 def convert_bandwidth(data, reverse=False):
 119     '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
 120     It assumes that bandwidth is well formed
 121     Attributes:
 122         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
 123         'reverse': by default convert form str to int (Mbps), if True it convert from number to units
 124     Return:
 125         None
 126     '''
 127     if type(data) is dict:
 128         for k in data.keys():
 129             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 130                 convert_bandwidth(data[k], reverse)
 131         if "bandwidth" in data:
 132             try:
 133                 value=str(data["bandwidth"])
 134                 if not reverse:
 135                     pos = value.find("bps")
 136                     if pos>0:
 137                         if value[pos-1]=="G": data["bandwidth"] =  int(data["bandwidth"][:pos-1]) * 1000
 138                         elif value[pos-1]=="k": data["bandwidth"]= int(data["bandwidth"][:pos-1]) / 1000
 139                         else: data["bandwidth"]= int(data["bandwidth"][:pos-1])
 140                 else:
 141                     value = int(data["bandwidth"])
 142                     if value % 1000 == 0: data["bandwidth"]=str(value/1000) + " Gbps"
 143                     else: data["bandwidth"]=str(value) + " Mbps"
 144             except:
 145                 print "convert_bandwidth exception for type", type(data["bandwidth"]), " data", data["bandwidth"]
 146                 return
 147     if type(data) is tuple or type(data) is list:
 148         for k in data:
 149             if type(k) is dict or type(k) is tuple or type(k) is list:
 150                 convert_bandwidth(k, reverse)
 151
 152 def convert_float_timestamp2str(var):
 153     '''Converts timestamps (created_at, modified_at fields) represented as float
 154     to a string with the format '%Y-%m-%dT%H:%i:%s'
 155     It enters recursively in the dict var finding this kind of variables
 156     '''
 157     if type(var) is dict:
 158         for k,v in var.items():
 159             if type(v) is float and k in ("created_at", "modified_at"):
 160                 var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) )
 161             elif type(v) is dict or type(v) is list or type(v) is tuple:
 162                 convert_float_timestamp2str(v)
 163         if len(var) == 0: return True
 164     elif type(var) is list or type(var) is tuple:
 165         for v in var:
 166             convert_float_timestamp2str(v)
 167
 168 def convert_datetime2str(var):
 169     '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
 170     It enters recursively in the dict var finding this kind of variables
 171     '''
 172     if type(var) is dict:
 173         for k,v in var.items():
 174             if type(v) is datetime.datetime:
 175                 var[k]= v.strftime('%Y-%m-%dT%H:%M:%S')
 176             elif type(v) is dict or type(v) is list or type(v) is tuple:
 177                 convert_datetime2str(v)
 178         if len(var) == 0: return True
 179     elif type(var) is list or type(var) is tuple:
 180         for v in var:
 181             convert_datetime2str(v)
 182
 183 def convert_str2boolean(data, items):
 184     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
 185     Done recursively
 186     Attributes:
 187         'data': dictionary variable to be checked. None or empty is considered valid
 188         'items': tuple of keys to convert
 189     Return:
 190         None
 191     '''
 192     if type(data) is dict:
 193         for k in data.keys():
 194             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 195                 convert_str2boolean(data[k], items)
 196             if k in items:
 197                 if type(data[k]) is str:
 198                     if   data[k]=="false" or data[k]=="False": data[k]=False
 199                     elif data[k]=="true"  or data[k]=="True":  data[k]=True
 200     if type(data) is tuple or type(data) is list:
 201         for k in data:
 202             if type(k) is dict or type(k) is tuple or type(k) is list:
 203                 convert_str2boolean(k, items)
 204
 205 def check_valid_uuid(uuid):
 206     id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
 207     id_schema2 = {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
 208     try:
 209         js_v(uuid, id_schema)
 210         return True
 211     except js_e.ValidationError:
 212         try:
 213             js_v(uuid, id_schema2)
 214             return True
 215         except js_e.ValidationError:
 216             return False
 217     return False
 218
 219
 220 def expand_brackets(text):
 221     """
 222     Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
 223     if no bracket is used it just return the a list with the single text
 224     It uses recursivity to allow several [] in the text
 225     :param text:
 226     :return:
 227     """
 228     if text is None:
 229         return (None, )
 230     start = text.find("[")
 231     end = text.find("]")
 232     if start < 0 or end < 0:
 233         return [text]
 234     text_list = []
 235     for char in text[start+1:end]:
 236         text_list += expand_brackets(text[:start] + char + text[end+1:])
 237     return text_list
 238
 239 def deprecated(message):
 240   def deprecated_decorator(func):
 241       def deprecated_func(*args, **kwargs):
 242           warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
 243                         category=DeprecationWarning,
 244                         stacklevel=2)
 245           warnings.simplefilter('default', DeprecationWarning)
 246           return func(*args, **kwargs)
 247       return deprecated_func
 248   return deprecated_decorator
 249
 250
 251 def truncate(text, max_length=1024):
 252     """Limit huge texts in number of characters"""
 253     text = str(text)
 254     if text and len(text) >= max_length:
 255         return text[:max_length//2-3] + " ... " + text[-max_length//2+3:]
 256     return text
 257
 258
 259 def merge_dicts(*dicts, **kwargs):
 260     """Creates a new dict merging N others and keyword arguments.
 261     Right-most dicts take precedence.
 262     Keyword args take precedence.
 263     """
 264     return reduce(
 265         lambda acc, x: acc.update(x) or acc,
 266         list(dicts) + [kwargs], {})
 267
 268
 269 def remove_none_items(adict):
 270     """Return a similar dict without keys associated to None values"""
 271     return {k: v for k, v in adict.items() if v is not None}
 272
 273
 274 def filter_dict_keys(adict, allow):
 275     """Return a similar dict, but just containing the explicitly allowed keys
 276
 277     Arguments:
 278         adict (dict): Simple python dict data struct
 279         allow (list): Explicits allowed keys
 280     """
 281     return {k: v for k, v in adict.items() if k in allow}
 282
 283
 284 def filter_out_dict_keys(adict, deny):
 285     """Return a similar dict, but not containing the explicitly denied keys
 286
 287     Arguments:
 288         adict (dict): Simple python dict data struct
 289         deny (list): Explicits denied keys
 290     """
 291     return {k: v for k, v in adict.items() if k not in deny}
 292
 293
 294 def expand_joined_fields(record):
 295     """Given a db query result, explode the fields that contains `.` (join
 296     operations).
 297
 298     Example
 299         >> expand_joined_fiels({'wim.id': 2})
 300         # {'wim': {'id': 2}}
 301     """
 302     result = {}
 303     for field, value in record.items():
 304         keys = field.split('.')
 305         target = result
 306         target = reduce(lambda target, key: target.setdefault(key, {}),
 307                         keys[:-1], result)
 308         target[keys[-1]] = value
 309
 310     return result
 311
 312
 313 def ensure(condition, exception):
 314     """Raise an exception if condition is not met"""
 315     if not condition:
 316         raise exception
 317
 318
 319 def partition(predicate, iterable):
 320     """Create two derived iterators from a single one
 321     The first iterator created will loop thought the values where the function
 322     predicate is True, the second one will iterate over the values where it is
 323     false.
 324     """
 325     iterable1, iterable2 = tee(iterable)
 326     return filter(predicate, iterable2), filterfalse(predicate, iterable1)
 327
 328
 329 def pipe(*functions):
 330     """Compose functions of one argument in the opposite order,
 331     So pipe(f, g)(x) = g(f(x))
 332     """
 333     return lambda x: reduce(lambda acc, f: f(acc), functions, x)
 334
 335
 336 def compose(*functions):
 337     """Compose functions of one argument,
 338     So compose(f, g)(x) = f(g(x))
 339     """
 340     return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x)
 341
 342
 343 def safe_get(target, key_path, default=None):
 344     """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
 345     a nested dict if present, or the default value
 346     """
 347     keys = key_path.split('.')
 348     target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target)
 349     return target.get(keys[-1], default)