common/python/rift/mano/tosca_translator/common/utils.py

   1 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   2 #    not use this file except in compliance with the License. You may obtain
   3 #    a copy of the License at
   4 #
   5 #         http://www.apache.org/licenses/LICENSE-2.0
   6 #
   7 #    Unless required by applicable law or agreed to in writing, software
   8 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
   9 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10 #    License for the specific language governing permissions and limitations
  11 #    under the License.
  12 #
  13 #    Copyright 2016 RIFT.io Inc
  14
  15
  16 import gettext
  17 import json
  18 import logging
  19 import math
  20 import numbers
  21 import os
  22 import re
  23 import requests
  24 from six.moves.urllib.parse import urlparse
  25 import yaml
  26
  27 from hashlib import md5
  28 from hashlib import sha256
  29
  30 import toscaparser.utils.yamlparser
  31
  32 _localedir = os.environ.get('tosca-translator'.upper() + '_LOCALEDIR')
  33 _t = gettext.translation('tosca-translator', localedir=_localedir,
  34                          fallback=True)
  35
  36
  37 def _(msg):
  38     return _t.gettext(msg)
  39
  40
  41 YAML_ORDER_PARSER = toscaparser.utils.yamlparser.simple_ordered_parse
  42 log = logging.getLogger('tosca-translator')
  43
  44 # Required environment variables to create openstackclient object.
  45 ENV_VARIABLES = ['OS_AUTH_URL', 'OS_PASSWORD', 'OS_USERNAME', 'OS_TENANT_NAME']
  46
  47
  48 class MemoryUnit(object):
  49
  50     UNIT_SIZE_DEFAULT = 'B'
  51     UNIT_SIZE_DICT = {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000000,
  52                       'MiB': 1048576, 'GB': 1000000000,
  53                       'GiB': 1073741824, 'TB': 1000000000000,
  54                       'TiB': 1099511627776}
  55
  56     @staticmethod
  57     def convert_unit_size_to_num(size, unit=None):
  58         """Convert given size to a number representing given unit.
  59
  60         If unit is None, convert to a number representing UNIT_SIZE_DEFAULT
  61         :param size: unit size e.g. 1 TB
  62         :param unit: unit to be converted to e.g GB
  63         :return: converted number e.g. 1000 for 1 TB size and unit GB
  64         """
  65         if unit:
  66             unit = MemoryUnit.validate_unit(unit)
  67         else:
  68             unit = MemoryUnit.UNIT_SIZE_DEFAULT
  69             log.info(_('A memory unit is not provided for size; using the '
  70                        'default unit %(default)s.') % {'default': 'B'})
  71         regex = re.compile('(\d*)\s*(\w*)')
  72         result = regex.match(str(size)).groups()
  73         if result[1]:
  74             unit_size = MemoryUnit.validate_unit(result[1])
  75             converted = int(str_to_num(result[0])
  76                             * MemoryUnit.UNIT_SIZE_DICT[unit_size]
  77                             * math.pow(MemoryUnit.UNIT_SIZE_DICT
  78                                        [unit], -1))
  79             log.info(_('Given size %(size)s is converted to %(num)s '
  80                        '%(unit)s.') % {'size': size,
  81                      'num': converted, 'unit': unit})
  82         else:
  83             converted = (str_to_num(result[0]))
  84         return converted
  85
  86     @staticmethod
  87     def validate_unit(unit):
  88         if unit in MemoryUnit.UNIT_SIZE_DICT.keys():
  89             return unit
  90         else:
  91             for key in MemoryUnit.UNIT_SIZE_DICT.keys():
  92                 if key.upper() == unit.upper():
  93                     return key
  94
  95             msg = _('Provided unit "{0}" is not valid. The valid units are'
  96                     ' {1}').format(unit, MemoryUnit.UNIT_SIZE_DICT.keys())
  97             log.error(msg)
  98             raise ValueError(msg)
  99
 100
 101 class CompareUtils(object):
 102
 103     MISMATCH_VALUE1_LABEL = "<Expected>"
 104     MISMATCH_VALUE2_LABEL = "<Provided>"
 105     ORDERLESS_LIST_KEYS = ['allowed_values', 'depends_on']
 106
 107     @staticmethod
 108     def compare_dicts(dict1, dict2):
 109         """Return False if not equal, True if both are equal."""
 110
 111         if dict1 is None and dict2 is None:
 112             return True
 113         if dict1 is None or dict2 is None:
 114             return False
 115
 116         both_equal = True
 117         for dict1_item, dict2_item in zip(dict1.items(), dict2.items()):
 118             if dict1_item != dict2_item:
 119                 msg = (_("%(label1)s: %(item1)s \n is not equal to \n:"
 120                          "%(label2)s: %(item2)s")
 121                        % {'label1': CompareUtils.MISMATCH_VALUE2_LABEL,
 122                           'item1': dict1_item,
 123                           'label2': CompareUtils.MISMATCH_VALUE1_LABEL,
 124                           'item2': dict2_item})
 125                 log.warning(msg)
 126                 both_equal = False
 127                 break
 128         return both_equal
 129
 130     @staticmethod
 131     def compare_mano_yamls(generated_yaml, expected_yaml):
 132         mano_translated_dict = YAML_ORDER_PARSER(generated_yaml)
 133         mano_expected_dict = YAML_ORDER_PARSER(expected_yaml)
 134         return CompareUtils.compare_dicts(mano_translated_dict,
 135                                           mano_expected_dict)
 136
 137     @staticmethod
 138     def reorder(dic):
 139         '''Canonicalize list items in the dictionary for ease of comparison.
 140
 141         For properties whose value is a list in which the order does not
 142         matter, some pre-processing is required to bring those lists into a
 143         canonical format. We use sorting just to make sure such differences
 144         in ordering would not cause to a mismatch.
 145         '''
 146
 147         if type(dic) is not dict:
 148             return None
 149
 150         reordered = {}
 151         for key in dic.keys():
 152             value = dic[key]
 153             if type(value) is dict:
 154                 reordered[key] = CompareUtils.reorder(value)
 155             elif type(value) is list \
 156                 and key in CompareUtils.ORDERLESS_LIST_KEYS:
 157                 reordered[key] = sorted(value)
 158             else:
 159                 reordered[key] = value
 160         return reordered
 161
 162     @staticmethod
 163     def diff_dicts(dict1, dict2, reorder=True):
 164         '''Compares two dictionaries and returns their differences.
 165
 166         Returns a dictionary of mismatches between the two dictionaries.
 167         An empty dictionary is returned if two dictionaries are equivalent.
 168         The reorder parameter indicates whether reordering is required
 169         before comparison or not.
 170         '''
 171
 172         if reorder:
 173             dict1 = CompareUtils.reorder(dict1)
 174             dict2 = CompareUtils.reorder(dict2)
 175
 176         if dict1 is None and dict2 is None:
 177             return {}
 178         if dict1 is None or dict2 is None:
 179             return {CompareUtils.MISMATCH_VALUE1_LABEL: dict1,
 180                     CompareUtils.MISMATCH_VALUE2_LABEL: dict2}
 181
 182         diff = {}
 183         keys1 = set(dict1.keys())
 184         keys2 = set(dict2.keys())
 185         for key in keys1.union(keys2):
 186             if key in keys1 and key not in keys2:
 187                 diff[key] = {CompareUtils.MISMATCH_VALUE1_LABEL: dict1[key],
 188                              CompareUtils.MISMATCH_VALUE2_LABEL: None}
 189             elif key not in keys1 and key in keys2:
 190                 diff[key] = {CompareUtils.MISMATCH_VALUE1_LABEL: None,
 191                              CompareUtils.MISMATCH_VALUE2_LABEL: dict2[key]}
 192             else:
 193                 val1 = dict1[key]
 194                 val2 = dict2[key]
 195                 if val1 != val2:
 196                     if type(val1) is dict and type(val2) is dict:
 197                         diff[key] = CompareUtils.diff_dicts(val1, val2, False)
 198                     else:
 199                         diff[key] = {CompareUtils.MISMATCH_VALUE1_LABEL: val1,
 200                                      CompareUtils.MISMATCH_VALUE2_LABEL: val2}
 201         return diff
 202
 203
 204 class YamlUtils(object):
 205
 206     @staticmethod
 207     def get_dict(yaml_file):
 208         '''Returns the dictionary representation of the given YAML spec.'''
 209         try:
 210             return yaml.load(open(yaml_file))
 211         except IOError:
 212             return None
 213
 214     @staticmethod
 215     def compare_yamls(yaml1_file, yaml2_file):
 216         '''Returns true if two dictionaries are equivalent, false otherwise.'''
 217         dict1 = YamlUtils.get_dict(yaml1_file)
 218         dict2 = YamlUtils.get_dict(yaml2_file)
 219         return CompareUtils.compare_dicts(dict1, dict2)
 220
 221     @staticmethod
 222     def compare_yaml_dict(yaml_file, dic):
 223         '''Returns true if yaml matches the dictionary, false otherwise.'''
 224         return CompareUtils.compare_dicts(YamlUtils.get_dict(yaml_file), dic)
 225
 226
 227 class TranslationUtils(object):
 228
 229     @staticmethod
 230     def compare_tosca_translation_with_mano(tosca_file, mano_file, params):
 231         '''Verify tosca translation against the given mano specification.
 232
 233         inputs:
 234         tosca_file: relative local path or URL to the tosca input file
 235         mano_file: relative path to expected mano output
 236         params: dictionary of parameter name value pairs
 237
 238         Returns as a dictionary the difference between the MANO translation
 239         of the given tosca_file and the given mano_file.
 240         '''
 241
 242         from toscaparser.tosca_template import ToscaTemplate
 243         from tosca_translator.mano.tosca_translator import TOSCATranslator
 244
 245         tosca_tpl = os.path.normpath(os.path.join(
 246             os.path.dirname(os.path.abspath(__file__)), tosca_file))
 247         a_file = os.path.isfile(tosca_tpl)
 248         if not a_file:
 249             tosca_tpl = tosca_file
 250
 251         expected_mano_tpl = os.path.join(
 252             os.path.dirname(os.path.abspath(__file__)), mano_file)
 253
 254         tosca = ToscaTemplate(tosca_tpl, params, a_file)
 255         translate = TOSCATranslator(tosca, params)
 256
 257         output = translate.translate()
 258         output_dict = toscaparser.utils.yamlparser.simple_parse(output)
 259         expected_output_dict = YamlUtils.get_dict(expected_mano_tpl)
 260         return CompareUtils.diff_dicts(output_dict, expected_output_dict)
 261
 262
 263 class UrlUtils(object):
 264
 265     @staticmethod
 266     def validate_url(path):
 267         """Validates whether the given path is a URL or not.
 268
 269         If the given path includes a scheme (http, https, ftp, ...) and a net
 270         location (a domain name such as www.github.com) it is validated as a
 271         URL.
 272         """
 273         parsed = urlparse(path)
 274         return bool(parsed.scheme) and bool(parsed.netloc)
 275
 276
 277 class ChecksumUtils(object):
 278
 279     @staticmethod
 280     def get_md5(input_file_name, log=None):
 281         chunk_size = 1048576  # 1024 B * 1024 B = 1048576 B = 1 MB
 282         file_md5_checksum = md5()
 283         try:
 284             with open(input_file_name, "rb") as f:
 285                 byte = f.read(chunk_size)
 286                 # previous_byte = byte
 287                 byte_size = len(byte)
 288                 file_read_iterations = 1
 289                 while byte:
 290                     file_md5_checksum.update(byte)
 291                     # previous_byte = byte
 292                     byte = f.read(chunk_size)
 293                     byte_size += len(byte)
 294                     file_read_iterations += 1
 295
 296             cksum = file_md5_checksum.hexdigest()
 297             if log:
 298                 log.debug(_("MD5 for {0} with size {1} (iter:{2}): {3}").
 299                           format(input_file_name, byte_size,
 300                                  file_read_iterations, cksum))
 301             return cksum
 302         except IOError:
 303             if log:
 304                 log.error(_('File could not be opened: {0}').
 305                           format(input_file_name))
 306                 return
 307             else:
 308                 raise
 309         except Exception as e:
 310             raise e
 311
 312     @staticmethod
 313     def get_sha256(input_file_name, log=None):
 314         chunk_size = 1048576  # 1024 B * 1024 B = 1048576 B = 1 MB
 315         file_sha256_checksum = sha256()
 316         try:
 317             with open(input_file_name, "rb") as f:
 318                 byte = f.read(chunk_size)
 319                 # previous_byte = byte
 320                 byte_size = len(byte)
 321                 file_read_iterations = 1
 322                 while byte:
 323                     file_sha256_checksum.update(byte)
 324                     # previous_byte = byte
 325                     byte = f.read(chunk_size)
 326                     byte_size += len(byte)
 327                     file_read_iterations += 1
 328
 329             cksum = file_sha256_checksum.hexdigest()
 330             if log:
 331                 log.debug(_("SHA256 for {0} with size {1} (iter:{2}): {3}").
 332                           format(input_file_name, byte_size,
 333                                  file_read_iterations, cksum))
 334             return cksum
 335         except IOError:
 336             if log:
 337                 log.error(_('File could not be opened: {0}').
 338                           format(input_file_name))
 339                 return
 340             else:
 341                 raise
 342         except Exception as e:
 343             raise e
 344
 345
 346 def str_to_num(value):
 347     """Convert a string representation of a number into a numeric type."""
 348     if isinstance(value, numbers.Number):
 349         return value
 350     try:
 351         return int(value)
 352     except ValueError:
 353         return float(value)
 354
 355
 356 def check_for_env_variables():
 357     return set(ENV_VARIABLES) < set(os.environ.keys())
 358
 359
 360 def get_ks_access_dict():
 361     tenant_name = os.getenv('OS_TENANT_NAME')
 362     username = os.getenv('OS_USERNAME')
 363     password = os.getenv('OS_PASSWORD')
 364     auth_url = os.getenv('OS_AUTH_URL')
 365
 366     auth_dict = {
 367         "auth": {
 368             "tenantName": tenant_name,
 369             "passwordCredentials": {
 370                 "username": username,
 371                 "password": password
 372             }
 373         }
 374     }
 375     headers = {'Content-Type': 'application/json'}
 376     try:
 377         keystone_response = requests.post(auth_url + '/tokens',
 378                                           data=json.dumps(auth_dict),
 379                                           headers=headers)
 380         if keystone_response.status_code != 200:
 381             return None
 382         return json.loads(keystone_response.content)
 383     except Exception:
 384         return None
 385
 386
 387 def get_url_for(access_dict, service_type):
 388     if access_dict is None:
 389         return None
 390     service_catalog = access_dict['access']['serviceCatalog']
 391     service_url = ''
 392     for service in service_catalog:
 393         if service['type'] == service_type:
 394             service_url = service['endpoints'][0]['publicURL']
 395             break
 396     return service_url
 397
 398
 399 def get_token_id(access_dict):
 400     if access_dict is None:
 401         return None
 402     return access_dict['access']['token']['id']
 403
 404
 405 def map_name_to_python(name):
 406     if name == 'type':
 407         return 'type_yang'
 408     return name.replace('-', '_')
 409
 410 def convert_keys_to_python(d):
 411     '''Change all keys from - to _'''
 412     if isinstance(d, dict):
 413         dic = {}
 414         for key in d.keys():
 415             dic[map_name_to_python(key)] = convert_keys_to_python(d[key])
 416         return dic
 417     elif isinstance(d, list):
 418         arr = []
 419         for memb in d:
 420             arr.append(convert_keys_to_python(memb))
 421         return arr
 422     else:
 423         return d
 424
 425 def map_name_to_yang (name):
 426     return name.replace('_', '-')
 427
 428 def convert_keys_to_yang(d):
 429     '''Change all keys from _ to -'''
 430     if isinstance(d, dict):
 431         dic = {}
 432         for key in d.keys():
 433             dic[map_name_to_python(key)] = convert_keys_to_yang(d[key])
 434         return dic
 435     elif isinstance(d, list):
 436         arr = []
 437         for memb in d:
 438             arr.append(convert_keys_to_yang(memb))
 439         return arr
 440     else:
 441         return d
 442
 443
 444 def dict_convert_values_to_str(d):
 445     '''Convert all leaf values to str'''
 446     if isinstance(d, dict):
 447         for key in d.keys():
 448             d[key] = dict_convert_values_to_str(d[key])
 449         return d
 450     elif isinstance(d, list):
 451         arr = []
 452         for memb in d:
 453             arr.append(dict_convert_values_to_str(memb))
 454         return arr
 455     else:
 456         return str(d)