osm_ro/utils.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 utils is a module that implements functions that are used by all openmano modules,
  26 dealing with aspects such as reading/writing files, formatting inputs/outputs for quick translation
  27 from dictionaries to appropriate database dictionaries, etc.
  28 '''
  29 __author__="Alfonso Tierno, Gerardo Garcia"
  30 __date__ ="$08-sep-2014 12:21:22$"
  31
  32 import datetime
  33 import time
  34 import warnings
  35 from functools import reduce
  36 from itertools import tee
  37
  38 from six.moves import filter, filterfalse
  39
  40 from jsonschema import exceptions as js_e
  41 from jsonschema import validate as js_v
  42
  43 #from bs4 import BeautifulSoup
  44
  45 def read_file(file_to_read):
  46     """Reads a file specified by 'file_to_read' and returns (True,<its content as a string>) in case of success or (False, <error message>) in case of failure"""
  47     try:
  48         f = open(file_to_read, 'r')
  49         read_data = f.read()
  50         f.close()
  51     except Exception as e:
  52         return (False, str(e))
  53
  54     return (True, read_data)
  55
  56 def write_file(file_to_write, text):
  57     """Write a file specified by 'file_to_write' and returns (True,NOne) in case of success or (False, <error message>) in case of failure"""
  58     try:
  59         f = open(file_to_write, 'w')
  60         f.write(text)
  61         f.close()
  62     except Exception as e:
  63         return (False, str(e))
  64
  65     return (True, None)
  66
  67 def format_in(http_response, schema):
  68     try:
  69         client_data = http_response.json()
  70         js_v(client_data, schema)
  71         #print "Input data: ", str(client_data)
  72         return True, client_data
  73     except js_e.ValidationError as exc:
  74         print "validate_in error, jsonschema exception ", exc.message, "at", exc.path
  75         return False, ("validate_in error, jsonschema exception ", exc.message, "at", exc.path)
  76
  77 def remove_extra_items(data, schema):
  78     deleted=[]
  79     if type(data) is tuple or type(data) is list:
  80         for d in data:
  81             a= remove_extra_items(d, schema['items'])
  82             if a is not None: deleted.append(a)
  83     elif type(data) is dict:
  84         #TODO deal with patternProperties
  85         if 'properties' not in schema:
  86             return None
  87         for k in data.keys():
  88             if k not in schema['properties'].keys():
  89                 del data[k]
  90                 deleted.append(k)
  91             else:
  92                 a = remove_extra_items(data[k], schema['properties'][k])
  93                 if a is not None:  deleted.append({k:a})
  94     if len(deleted) == 0: return None
  95     elif len(deleted) == 1: return deleted[0]
  96     else: return deleted
  97
  98 #def format_html2text(http_content):
  99 #    soup=BeautifulSoup(http_content)
 100 #    text = soup.p.get_text() + " " + soup.pre.get_text()
 101 #    return text
 102
 103
 104 def delete_nulls(var):
 105     if type(var) is dict:
 106         for k in var.keys():
 107             if var[k] is None: del var[k]
 108             elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple:
 109                 if delete_nulls(var[k]): del var[k]
 110         if len(var) == 0: return True
 111     elif type(var) is list or type(var) is tuple:
 112         for k in var:
 113             if type(k) is dict: delete_nulls(k)
 114         if len(var) == 0: return True
 115     return False
 116
 117
 118 def convert_bandwidth(data, reverse=False):
 119     '''Check the field bandwidth recursivelly and when found, it removes units and convert to number
 120     It assumes that bandwidth is well formed
 121     Attributes:
 122         'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid
 123         'reverse': by default convert form str to int (Mbps), if True it convert from number to units
 124     Return:
 125         None
 126     '''
 127     if type(data) is dict:
 128         for k in data.keys():
 129             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 130                 convert_bandwidth(data[k], reverse)
 131         if "bandwidth" in data:
 132             try:
 133                 value=str(data["bandwidth"])
 134                 if not reverse:
 135                     pos = value.find("bps")
 136                     if pos>0:
 137                         if value[pos-1]=="G": data["bandwidth"] =  int(data["bandwidth"][:pos-1]) * 1000
 138                         elif value[pos-1]=="k": data["bandwidth"]= int(data["bandwidth"][:pos-1]) / 1000
 139                         else: data["bandwidth"]= int(data["bandwidth"][:pos-1])
 140                 else:
 141                     value = int(data["bandwidth"])
 142                     if value % 1000 == 0: data["bandwidth"]=str(value/1000) + " Gbps"
 143                     else: data["bandwidth"]=str(value) + " Mbps"
 144             except:
 145                 print "convert_bandwidth exception for type", type(data["bandwidth"]), " data", data["bandwidth"]
 146                 return
 147     if type(data) is tuple or type(data) is list:
 148         for k in data:
 149             if type(k) is dict or type(k) is tuple or type(k) is list:
 150                 convert_bandwidth(k, reverse)
 151
 152 def convert_float_timestamp2str(var):
 153     '''Converts timestamps (created_at, modified_at fields) represented as float
 154     to a string with the format '%Y-%m-%dT%H:%i:%s'
 155     It enters recursively in the dict var finding this kind of variables
 156     '''
 157     if type(var) is dict:
 158         for k,v in var.items():
 159             if type(v) is float and k in ("created_at", "modified_at"):
 160                 var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) )
 161             elif type(v) is dict or type(v) is list or type(v) is tuple:
 162                 convert_float_timestamp2str(v)
 163         if len(var) == 0: return True
 164     elif type(var) is list or type(var) is tuple:
 165         for v in var:
 166             convert_float_timestamp2str(v)
 167
 168 def convert_datetime2str(var):
 169     '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s'
 170     It enters recursively in the dict var finding this kind of variables
 171     '''
 172     if type(var) is dict:
 173         for k,v in var.items():
 174             if type(v) is datetime.datetime:
 175                 var[k]= v.strftime('%Y-%m-%dT%H:%M:%S')
 176             elif type(v) is dict or type(v) is list or type(v) is tuple:
 177                 convert_datetime2str(v)
 178         if len(var) == 0: return True
 179     elif type(var) is list or type(var) is tuple:
 180         for v in var:
 181             convert_datetime2str(v)
 182
 183 def convert_str2boolean(data, items):
 184     '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean
 185     Done recursively
 186     Attributes:
 187         'data': dictionary variable to be checked. None or empty is considered valid
 188         'items': tuple of keys to convert
 189     Return:
 190         None
 191     '''
 192     if type(data) is dict:
 193         for k in data.keys():
 194             if type(data[k]) is dict or type(data[k]) is tuple or type(data[k]) is list:
 195                 convert_str2boolean(data[k], items)
 196             if k in items:
 197                 if type(data[k]) is str:
 198                     if   data[k]=="false" or data[k]=="False": data[k]=False
 199                     elif data[k]=="true"  or data[k]=="True":  data[k]=True
 200     if type(data) is tuple or type(data) is list:
 201         for k in data:
 202             if type(k) is dict or type(k) is tuple or type(k) is list:
 203                 convert_str2boolean(k, items)
 204
 205 def check_valid_uuid(uuid):
 206     id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"}
 207     id_schema2 = {"type" : "string", "pattern": "^[a-fA-F0-9]{32}$"}
 208     try:
 209         js_v(uuid, id_schema)
 210         return True
 211     except js_e.ValidationError:
 212         try:
 213             js_v(uuid, id_schema2)
 214             return True
 215         except js_e.ValidationError:
 216             return False
 217     return False
 218
 219
 220 def expand_brackets(text):
 221     """
 222     Change a text with TEXT[ABC..] into a list with [TEXTA, TEXTB, TEXC, ...
 223     if no bracket is used it just return the a list with the single text
 224     It uses recursivity to allow several [] in the text
 225     :param text:
 226     :return:
 227     """
 228     start = text.find("[")
 229     end = text.find("]")
 230     if start < 0 or end < 0:
 231         return [text]
 232     text_list = []
 233     for char in text[start+1:end]:
 234         text_list += expand_brackets(text[:start] + char + text[end+1:])
 235     return text_list
 236
 237 def deprecated(message):
 238   def deprecated_decorator(func):
 239       def deprecated_func(*args, **kwargs):
 240           warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
 241                         category=DeprecationWarning,
 242                         stacklevel=2)
 243           warnings.simplefilter('default', DeprecationWarning)
 244           return func(*args, **kwargs)
 245       return deprecated_func
 246   return deprecated_decorator
 247
 248
 249 def truncate(text, max_length=1024):
 250     """Limit huge texts in number of characters"""
 251     text = str(text)
 252     if text and len(text) >= max_length:
 253         return text[:max_length//2-3] + " ... " + text[-max_length//2+3:]
 254     return text
 255
 256
 257 def merge_dicts(*dicts, **kwargs):
 258     """Creates a new dict merging N others and keyword arguments.
 259     Right-most dicts take precedence.
 260     Keyword args take precedence.
 261     """
 262     return reduce(
 263         lambda acc, x: acc.update(x) or acc,
 264         list(dicts) + [kwargs], {})
 265
 266
 267 def remove_none_items(adict):
 268     """Return a similar dict without keys associated to None values"""
 269     return {k: v for k, v in adict.items() if v is not None}
 270
 271
 272 def filter_dict_keys(adict, allow):
 273     """Return a similar dict, but just containing the explicitly allowed keys
 274
 275     Arguments:
 276         adict (dict): Simple python dict data struct
 277         allow (list): Explicits allowed keys
 278     """
 279     return {k: v for k, v in adict.items() if k in allow}
 280
 281
 282 def filter_out_dict_keys(adict, deny):
 283     """Return a similar dict, but not containing the explicitly denied keys
 284
 285     Arguments:
 286         adict (dict): Simple python dict data struct
 287         deny (list): Explicits denied keys
 288     """
 289     return {k: v for k, v in adict.items() if k not in deny}
 290
 291
 292 def expand_joined_fields(record):
 293     """Given a db query result, explode the fields that contains `.` (join
 294     operations).
 295
 296     Example
 297         >> expand_joined_fiels({'wim.id': 2})
 298         # {'wim': {'id': 2}}
 299     """
 300     result = {}
 301     for field, value in record.items():
 302         keys = field.split('.')
 303         target = result
 304         target = reduce(lambda target, key: target.setdefault(key, {}),
 305                         keys[:-1], result)
 306         target[keys[-1]] = value
 307
 308     return result
 309
 310
 311 def ensure(condition, exception):
 312     """Raise an exception if condition is not met"""
 313     if not condition:
 314         raise exception
 315
 316
 317 def partition(predicate, iterable):
 318     """Create two derived iterators from a single one
 319     The first iterator created will loop thought the values where the function
 320     predicate is True, the second one will iterate over the values where it is
 321     false.
 322     """
 323     iterable1, iterable2 = tee(iterable)
 324     return filter(predicate, iterable2), filterfalse(predicate, iterable1)
 325
 326
 327 def pipe(*functions):
 328     """Compose functions of one argument in the opposite order,
 329     So pipe(f, g)(x) = g(f(x))
 330     """
 331     return lambda x: reduce(lambda acc, f: f(acc), functions, x)
 332
 333
 334 def compose(*functions):
 335     """Compose functions of one argument,
 336     So compose(f, g)(x) = f(g(x))
 337     """
 338     return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x)
 339
 340
 341 def safe_get(target, key_path, default=None):
 342     """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in
 343     a nested dict if present, or the default value
 344     """
 345     keys = key_path.split('.')
 346     target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target)
 347     return target.get(keys[-1], default)