1 # Licensed under the Apache License, Version 2.0 (the "License"); you may
2 # not use this file except in compliance with the License. You may obtain
3 # a copy of the License at
5 # http://www.apache.org/licenses/LICENSE-2.0
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10 # License for the specific language governing permissions and limitations
13 # Copyright 2016 RIFT.io Inc
24 from six
.moves
.urllib
.parse
import urlparse
27 from hashlib
import md5
28 from hashlib
import sha256
30 import toscaparser
.utils
.yamlparser
32 _localedir
= os
.environ
.get('tosca-translator'.upper() + '_LOCALEDIR')
33 _t
= gettext
.translation('tosca-translator', localedir
=_localedir
,
38 return _t
.gettext(msg
)
41 YAML_ORDER_PARSER
= toscaparser
.utils
.yamlparser
.simple_ordered_parse
42 log
= logging
.getLogger('tosca-translator')
44 # Required environment variables to create openstackclient object.
45 ENV_VARIABLES
= ['OS_AUTH_URL', 'OS_PASSWORD', 'OS_USERNAME', 'OS_TENANT_NAME']
48 class MemoryUnit(object):
50 UNIT_SIZE_DEFAULT
= 'B'
51 UNIT_SIZE_DICT
= {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000000,
52 'MiB': 1048576, 'GB': 1000000000,
53 'GiB': 1073741824, 'TB': 1000000000000,
57 def convert_unit_size_to_num(size
, unit
=None):
58 """Convert given size to a number representing given unit.
60 If unit is None, convert to a number representing UNIT_SIZE_DEFAULT
61 :param size: unit size e.g. 1 TB
62 :param unit: unit to be converted to e.g GB
63 :return: converted number e.g. 1000 for 1 TB size and unit GB
66 unit
= MemoryUnit
.validate_unit(unit
)
68 unit
= MemoryUnit
.UNIT_SIZE_DEFAULT
69 log
.info(_('A memory unit is not provided for size; using the '
70 'default unit %(default)s.') % {'default': 'B'})
71 regex
= re
.compile('(\d*)\s*(\w*)')
72 result
= regex
.match(str(size
)).groups()
74 unit_size
= MemoryUnit
.validate_unit(result
[1])
75 converted
= int(str_to_num(result
[0])
76 * MemoryUnit
.UNIT_SIZE_DICT
[unit_size
]
77 * math
.pow(MemoryUnit
.UNIT_SIZE_DICT
79 log
.info(_('Given size %(size)s is converted to %(num)s '
80 '%(unit)s.') % {'size': size
,
81 'num': converted
, 'unit': unit
})
83 converted
= (str_to_num(result
[0]))
87 def validate_unit(unit
):
88 if unit
in MemoryUnit
.UNIT_SIZE_DICT
.keys():
91 for key
in MemoryUnit
.UNIT_SIZE_DICT
.keys():
92 if key
.upper() == unit
.upper():
95 msg
= _('Provided unit "{0}" is not valid. The valid units are'
96 ' {1}').format(unit
, MemoryUnit
.UNIT_SIZE_DICT
.keys())
101 class CompareUtils(object):
103 MISMATCH_VALUE1_LABEL
= "<Expected>"
104 MISMATCH_VALUE2_LABEL
= "<Provided>"
105 ORDERLESS_LIST_KEYS
= ['allowed_values', 'depends_on']
108 def compare_dicts(dict1
, dict2
):
109 """Return False if not equal, True if both are equal."""
111 if dict1
is None and dict2
is None:
113 if dict1
is None or dict2
is None:
117 for dict1_item
, dict2_item
in zip(dict1
.items(), dict2
.items()):
118 if dict1_item
!= dict2_item
:
119 msg
= (_("%(label1)s: %(item1)s \n is not equal to \n:"
120 "%(label2)s: %(item2)s")
121 % {'label1': CompareUtils
.MISMATCH_VALUE2_LABEL
,
123 'label2': CompareUtils
.MISMATCH_VALUE1_LABEL
,
124 'item2': dict2_item
})
131 def compare_mano_yamls(generated_yaml
, expected_yaml
):
132 mano_translated_dict
= YAML_ORDER_PARSER(generated_yaml
)
133 mano_expected_dict
= YAML_ORDER_PARSER(expected_yaml
)
134 return CompareUtils
.compare_dicts(mano_translated_dict
,
139 '''Canonicalize list items in the dictionary for ease of comparison.
141 For properties whose value is a list in which the order does not
142 matter, some pre-processing is required to bring those lists into a
143 canonical format. We use sorting just to make sure such differences
144 in ordering would not cause to a mismatch.
147 if type(dic
) is not dict:
151 for key
in dic
.keys():
153 if type(value
) is dict:
154 reordered
[key
] = CompareUtils
.reorder(value
)
155 elif type(value
) is list \
156 and key
in CompareUtils
.ORDERLESS_LIST_KEYS
:
157 reordered
[key
] = sorted(value
)
159 reordered
[key
] = value
163 def diff_dicts(dict1
, dict2
, reorder
=True):
164 '''Compares two dictionaries and returns their differences.
166 Returns a dictionary of mismatches between the two dictionaries.
167 An empty dictionary is returned if two dictionaries are equivalent.
168 The reorder parameter indicates whether reordering is required
169 before comparison or not.
173 dict1
= CompareUtils
.reorder(dict1
)
174 dict2
= CompareUtils
.reorder(dict2
)
176 if dict1
is None and dict2
is None:
178 if dict1
is None or dict2
is None:
179 return {CompareUtils
.MISMATCH_VALUE1_LABEL
: dict1
,
180 CompareUtils
.MISMATCH_VALUE2_LABEL
: dict2
}
183 keys1
= set(dict1
.keys())
184 keys2
= set(dict2
.keys())
185 for key
in keys1
.union(keys2
):
186 if key
in keys1
and key
not in keys2
:
187 diff
[key
] = {CompareUtils
.MISMATCH_VALUE1_LABEL
: dict1
[key
],
188 CompareUtils
.MISMATCH_VALUE2_LABEL
: None}
189 elif key
not in keys1
and key
in keys2
:
190 diff
[key
] = {CompareUtils
.MISMATCH_VALUE1_LABEL
: None,
191 CompareUtils
.MISMATCH_VALUE2_LABEL
: dict2
[key
]}
196 if type(val1
) is dict and type(val2
) is dict:
197 diff
[key
] = CompareUtils
.diff_dicts(val1
, val2
, False)
199 diff
[key
] = {CompareUtils
.MISMATCH_VALUE1_LABEL
: val1
,
200 CompareUtils
.MISMATCH_VALUE2_LABEL
: val2
}
204 class YamlUtils(object):
207 def get_dict(yaml_file
):
208 '''Returns the dictionary representation of the given YAML spec.'''
210 return yaml
.load(open(yaml_file
))
215 def compare_yamls(yaml1_file
, yaml2_file
):
216 '''Returns true if two dictionaries are equivalent, false otherwise.'''
217 dict1
= YamlUtils
.get_dict(yaml1_file
)
218 dict2
= YamlUtils
.get_dict(yaml2_file
)
219 return CompareUtils
.compare_dicts(dict1
, dict2
)
222 def compare_yaml_dict(yaml_file
, dic
):
223 '''Returns true if yaml matches the dictionary, false otherwise.'''
224 return CompareUtils
.compare_dicts(YamlUtils
.get_dict(yaml_file
), dic
)
227 class TranslationUtils(object):
230 def compare_tosca_translation_with_mano(tosca_file
, mano_file
, params
):
231 '''Verify tosca translation against the given mano specification.
234 tosca_file: relative local path or URL to the tosca input file
235 mano_file: relative path to expected mano output
236 params: dictionary of parameter name value pairs
238 Returns as a dictionary the difference between the MANO translation
239 of the given tosca_file and the given mano_file.
242 from toscaparser
.tosca_template
import ToscaTemplate
243 from tosca_translator
.mano
.tosca_translator
import TOSCATranslator
245 tosca_tpl
= os
.path
.normpath(os
.path
.join(
246 os
.path
.dirname(os
.path
.abspath(__file__
)), tosca_file
))
247 a_file
= os
.path
.isfile(tosca_tpl
)
249 tosca_tpl
= tosca_file
251 expected_mano_tpl
= os
.path
.join(
252 os
.path
.dirname(os
.path
.abspath(__file__
)), mano_file
)
254 tosca
= ToscaTemplate(tosca_tpl
, params
, a_file
)
255 translate
= TOSCATranslator(tosca
, params
)
257 output
= translate
.translate()
258 output_dict
= toscaparser
.utils
.yamlparser
.simple_parse(output
)
259 expected_output_dict
= YamlUtils
.get_dict(expected_mano_tpl
)
260 return CompareUtils
.diff_dicts(output_dict
, expected_output_dict
)
263 class UrlUtils(object):
266 def validate_url(path
):
267 """Validates whether the given path is a URL or not.
269 If the given path includes a scheme (http, https, ftp, ...) and a net
270 location (a domain name such as www.github.com) it is validated as a
273 parsed
= urlparse(path
)
274 return bool(parsed
.scheme
) and bool(parsed
.netloc
)
277 class ChecksumUtils(object):
280 def get_md5(input_file_name
, log
=None):
281 chunk_size
= 1048576 # 1024 B * 1024 B = 1048576 B = 1 MB
282 file_md5_checksum
= md5()
284 with
open(input_file_name
, "rb") as f
:
285 byte
= f
.read(chunk_size
)
286 # previous_byte = byte
287 byte_size
= len(byte
)
288 file_read_iterations
= 1
290 file_md5_checksum
.update(byte
)
291 # previous_byte = byte
292 byte
= f
.read(chunk_size
)
293 byte_size
+= len(byte
)
294 file_read_iterations
+= 1
296 cksum
= file_md5_checksum
.hexdigest()
298 log
.debug(_("MD5 for {0} with size {1} (iter:{2}): {3}").
299 format(input_file_name
, byte_size
,
300 file_read_iterations
, cksum
))
304 log
.error(_('File could not be opened: {0}').
305 format(input_file_name
))
309 except Exception as e
:
313 def get_sha256(input_file_name
, log
=None):
314 chunk_size
= 1048576 # 1024 B * 1024 B = 1048576 B = 1 MB
315 file_sha256_checksum
= sha256()
317 with
open(input_file_name
, "rb") as f
:
318 byte
= f
.read(chunk_size
)
319 # previous_byte = byte
320 byte_size
= len(byte
)
321 file_read_iterations
= 1
323 file_sha256_checksum
.update(byte
)
324 # previous_byte = byte
325 byte
= f
.read(chunk_size
)
326 byte_size
+= len(byte
)
327 file_read_iterations
+= 1
329 cksum
= file_sha256_checksum
.hexdigest()
331 log
.debug(_("SHA256 for {0} with size {1} (iter:{2}): {3}").
332 format(input_file_name
, byte_size
,
333 file_read_iterations
, cksum
))
337 log
.error(_('File could not be opened: {0}').
338 format(input_file_name
))
342 except Exception as e
:
346 def str_to_num(value
):
347 """Convert a string representation of a number into a numeric type."""
348 if isinstance(value
, numbers
.Number
):
356 def check_for_env_variables():
357 return set(ENV_VARIABLES
) < set(os
.environ
.keys())
360 def get_ks_access_dict():
361 tenant_name
= os
.getenv('OS_TENANT_NAME')
362 username
= os
.getenv('OS_USERNAME')
363 password
= os
.getenv('OS_PASSWORD')
364 auth_url
= os
.getenv('OS_AUTH_URL')
368 "tenantName": tenant_name
,
369 "passwordCredentials": {
370 "username": username
,
375 headers
= {'Content-Type': 'application/json'}
377 keystone_response
= requests
.post(auth_url
+ '/tokens',
378 data
=json
.dumps(auth_dict
),
380 if keystone_response
.status_code
!= 200:
382 return json
.loads(keystone_response
.content
)
387 def get_url_for(access_dict
, service_type
):
388 if access_dict
is None:
390 service_catalog
= access_dict
['access']['serviceCatalog']
392 for service
in service_catalog
:
393 if service
['type'] == service_type
:
394 service_url
= service
['endpoints'][0]['publicURL']
399 def get_token_id(access_dict
):
400 if access_dict
is None:
402 return access_dict
['access']['token']['id']
405 def map_name_to_python(name
):
408 return name
.replace('-', '_')
410 def convert_keys_to_python(d
):
411 '''Change all keys from - to _'''
412 if isinstance(d
, dict):
415 dic
[map_name_to_python(key
)] = convert_keys_to_python(d
[key
])
417 elif isinstance(d
, list):
420 arr
.append(convert_keys_to_python(memb
))
425 def map_name_to_yang (name
):
426 return name
.replace('_', '-')
428 def convert_keys_to_yang(d
):
429 '''Change all keys from _ to -'''
430 if isinstance(d
, dict):
433 dic
[map_name_to_python(key
)] = convert_keys_to_yang(d
[key
])
435 elif isinstance(d
, list):
438 arr
.append(convert_keys_to_yang(memb
))
444 def dict_convert_values_to_str(d
):
445 '''Convert all leaf values to str'''
446 if isinstance(d
, dict):
448 d
[key
] = dict_convert_values_to_str(d
[key
])
450 elif isinstance(d
, list):
453 arr
.append(dict_convert_values_to_str(memb
))