X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osmclient%2Fcommon%2Fpackage_tool.py;h=7d582ecd1ca205fe07af67fb82132259da221337;hb=ef473491254e99f791481b2c0e67066a003c675e;hp=f35a7f33d63018dbcc5062c42df3048defa0fc31;hpb=76c880e2d5929a0298b31d5b0bf2431d897b2508;p=osm%2Fosmclient.git diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index f35a7f3..7d582ec 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -15,18 +15,20 @@ # License for the specific language governing permissions and limitations # under the License. -from osmclient.common.exceptions import ClientException -import os import glob -import time -import tarfile import hashlib -from osm_im.validation import Validation as validation_im -from jinja2 import Environment, PackageLoader -import subprocess +import logging +import os import shutil +import subprocess +import tarfile +import time + +from jinja2 import Environment, PackageLoader +from osm_im.validation import Validation as validation_im +from osmclient.common.exceptions import ClientException import yaml -import logging + class PackageTool(object): def __init__(self, client=None): @@ -150,19 +152,18 @@ class PackageTool(object): :returns: None """ self._logger.debug("") - files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)] - checksum = open("{}/checksums.txt".format(package_folder), "w+") - for file_item in files: - if "checksums.txt" in file_item: - continue - # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html - md5_hash = hashlib.md5() - with open(file_item, "rb") as f: - # Read and update hash in chunks of 4K - for byte_block in iter(lambda: f.read(4096), b""): - md5_hash.update(byte_block) - checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) - checksum.close() + files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)] + with open("{}/checksums.txt".format(package_folder), "w+") as checksum: + for file_item in files: + if "checksums.txt" in file_item: + continue + # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html + md5_hash = hashlib.md5() + with open(file_item, "rb") as f: + # Read and update hash in chunks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + md5_hash.update(byte_block) + checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) def create_folders(self, folders, package_type): """ @@ -276,26 +277,26 @@ class PackageTool(object): descriptor_file = False descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] for file in descriptors_paths: - if 'nfd.yaml' in file: + if file.endswith('nfd.yaml'): descriptor_file = True listCharms = self.charms_search(file, 'vnf') - if 'nsd.yaml' in file: + if file.endswith('nsd.yaml'): descriptor_file = True listCharms = self.charms_search(file, 'ns') print("List of charms in the descriptor: {}".format(listCharms)) if not descriptor_file: - raise ClientException ('descriptor name is not correct in: {}'.format(package_folder)) + raise ClientException('descriptor name is not correct in: {}'.format(package_folder)) if listCharms and not skip_charm_build: for charmName in listCharms: - if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)): + if os.path.isdir('{}/charms/layers/{}'.format(package_folder, charmName)): print('Building charm {}/charms/layers/{}'.format(package_folder, charmName)) self.charm_build(package_folder, charmName) - print('Charm built'.format(charmName)) + print('Charm built: {}'.format(charmName)) else: - if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)): - raise ClientException ('The charm: {} referenced in the descriptor file ' - 'is not present either in {}/charms or in {}/charms/layers'. - format(charmName, package_folder,package_folder)) + if not os.path.isdir('{}/charms/{}'.format(package_folder, charmName)): + raise ClientException('The charm: {} referenced in the descriptor file ' + 'is not present either in {}/charms or in {}/charms/layers'. + format(charmName, package_folder, package_folder)) self._logger.debug("Return list of charms: {}".format(listCharms)) return listCharms @@ -362,27 +363,33 @@ class PackageTool(object): returns: .tar.gz name """ self._logger.debug("") + cwd = None try: - directory_name = self.create_temp_dir(package_folder, charm_list) + directory_name, package_name = self.create_temp_dir(package_folder, charm_list) cwd = os.getcwd() os.chdir(directory_name) - self.calculate_checksum(package_folder) - with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive: - print("Adding File: {}".format(package_folder)) - archive.add('{}'.format(package_folder), recursive=True) - #return "Created {}.tar.gz".format(package_folder) - #self.build("{}".format(os.path.basename(package_folder))) + self.calculate_checksum(package_name) + with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive: + print("Adding File: {}".format(package_name)) + archive.add('{}'.format(package_name), recursive=True) + # return "Created {}.tar.gz".format(package_folder) + # self.build("{}".format(os.path.basename(package_folder))) os.chdir(cwd) + cwd = None + created_package = "{}/{}.tar.gz".format(os.path.dirname(package_folder) or '.', package_name) + os.rename("{}/{}.tar.gz".format(directory_name, package_name), + created_package) + os.rename("{}/{}/checksums.txt".format(directory_name, package_name), + "{}/checksums.txt".format(package_folder)) + print("Package created: {}".format(created_package)) + return created_package except Exception as exc: + raise ClientException('failure during build of targz file (create temp dir, calculate checksum, ' + 'tar.gz file): {}'.format(exc)) + finally: + if cwd: + os.chdir(cwd) shutil.rmtree(os.path.join(package_folder, "tmp")) - raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc)) - os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)), - "{}.tar.gz".format(os.path.basename(package_folder))) - os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)), - "{}/checksums.txt".format(package_folder)) - shutil.rmtree(os.path.join(package_folder, "tmp")) - print("Package created: {}.tar.gz".format(os.path.basename(package_folder))) - return "{}.tar.gz".format(package_folder) def create_temp_dir(self, package_folder, charm_list=None): """ @@ -391,14 +398,16 @@ class PackageTool(object): self._logger.debug("") ignore_patterns = ('.gitignore') ignore = shutil.ignore_patterns(ignore_patterns) - directory_name = os.path.abspath("{}/tmp".format(package_folder)) - os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True)) - self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, os.path.basename(package_folder))) + directory_name = os.path.abspath(package_folder) + package_name = os.path.basename(directory_name) + directory_name += "/tmp" + os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True) + self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name)) for item in os.listdir(package_folder): self._logger.debug("Item: {}".format(item)) if item != "tmp": s = os.path.join(package_folder, item) - d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item) + d = os.path.join(os.path.join(directory_name, package_name), item) if os.path.isdir(s): if item == "charms": os.makedirs(d, exist_ok=True) @@ -415,40 +424,62 @@ class PackageTool(object): format(charm, package_folder, package_folder)) d_temp = os.path.join(d, charm) self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp)) - shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore) + shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore) self._logger.debug("DONE") else: - self._logger.debug("Copying tree: {} -> {}".format(s,d)) - shutil.copytree(s, d, symlinks = True, ignore = ignore) + self._logger.debug("Copying tree: {} -> {}".format(s, d)) + shutil.copytree(s, d, symlinks=True, ignore=ignore) self._logger.debug("DONE") else: if item in ignore_patterns: continue - self._logger.debug("Copying file: {} -> {}".format(s,d)) + self._logger.debug("Copying file: {} -> {}".format(s, d)) shutil.copy2(s, d) self._logger.debug("DONE") - return directory_name + return directory_name, package_name def charms_search(self, descriptor_file, desc_type): self._logger.debug("") - dict = {} - list = [] with open("{}".format(descriptor_file)) as yaml_desc: - dict = yaml.safe_load(yaml_desc) - for k1, v1 in dict.items(): - for k2, v2 in v1.items(): - for entry in v2: - if '{}-configuration'.format(desc_type) in entry: - name = entry['{}-configuration'.format(desc_type)] - for k3, v3 in name.items(): - if 'charm' in v3: - list.append((v3['charm'])) - if 'vdu' in entry: - name = entry['vdu'] - for vdu in name: - if 'vdu-configuration' in vdu: - for k4, v4 in vdu['vdu-configuration'].items(): - if 'charm' in v4: - list.append((v4['charm'])) - return list - + descriptor_dict = yaml.safe_load(yaml_desc) + if "catalog" in descriptor_dict: # Match OSM-IM vnfd-catalog and nsd-catalog + charms_list = self._charms_search_on_sol006_dict(descriptor_dict, desc_type) + else: + charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type) + + return charms_list + + def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type): + charms_list = [] + for k1, v1 in osm_im_dict.items(): + for k2, v2 in v1.items(): + for entry in v2: + if '{}-configuration'.format(desc_type) in entry: + vnf_config = entry['{}-configuration'.format(desc_type)] + for k3, v3 in vnf_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + if 'vdu' in entry: + vdus = entry['vdu'] + for vdu in vdus: + if 'vdu-configuration' in vdu: + for k4, v4 in vdu['vdu-configuration'].items(): + if 'charm' in v4: + charms_list.append((v4['charm'])) + return charms_list + + def _charms_search_on_sol006_dict(self, sol006_dict, desc_type): + charms_list = [] + for k1, v1 in sol006_dict.items(): + for k2, v2 in v1.items(): + if '{}-configuration'.format(desc_type) in k2: + for vnf_config in v2: + for k3, v3 in vnf_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + if 'vdu-configuration' in k2: + for vdu_config in v2: + for k3, v3 in vdu_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + return charms_list