X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osmclient%2Fcommon%2Fpackage_tool.py;h=abdcd3c173ff603f9f8bcc3ce03f00aee36289cc;hb=refs%2Fchanges%2F91%2F12191%2F2;hp=1d7f57f190465c946f863b6b0b558a0270df3417;hpb=70208ca1de7ff0e91a17d8f918d5b6044e9fa388;p=osm%2Fosmclient.git diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index 1d7f57f..abdcd3c 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -16,19 +16,21 @@ # under the License. import glob -import hashlib import logging import os import shutil import subprocess import tarfile import time - from jinja2 import Environment, PackageLoader from osm_im.validation import Validation as validation_im from osm_im.validation import ValidationException from osm_im import im_translation +from osmclient.common import package_handling as package_handling from osmclient.common.exceptions import ClientException +from osmclient.common import utils +from .sol004_package import SOL004Package +from .sol007_package import SOL007Package import yaml @@ -54,6 +56,7 @@ class PackageTool(object): detailed, netslice_subnets, netslice_vlds, + old, ): """ **Create a package descriptor** @@ -71,15 +74,16 @@ class PackageTool(object): - detailed: include all possible values for NSD, VNFD, NST - netslice_subnets: number of netslice_subnets for the NST - netslice_vlds: number of virtual link descriptors for the NST + - old: flag to create a descriptor using the previous OSM format (pre SOL006, OSM<9) :return: status """ self._logger.debug("") # print("location: {}".format(osmclient.__path__)) file_loader = PackageLoader("osmclient") - env = Environment(loader=file_loader) + env = Environment(loader=file_loader, autoscape=True) if package_type == "ns": - template = env.get_template("nsd.yaml.j2") + template = env.get_template("nsd.yaml.j2" if not old else "nsd_old.yaml.j2") content = { "name": package_name, "vendor": vendor, @@ -89,7 +93,9 @@ class PackageTool(object): "detailed": detailed, } elif package_type == "vnf": - template = env.get_template("vnfd.yaml.j2") + template = env.get_template( + "vnfd.yaml.j2" if not old else "vnfd_old.yaml.j2" + ) content = { "name": package_name, "vendor": vendor, @@ -103,6 +109,7 @@ class PackageTool(object): "detailed": detailed, } elif package_type == "nst": + # TODO: repo-index did not support nst in OSM<9, no changes in template template = env.get_template("nst.yaml.j2") content = { "name": package_name, @@ -117,9 +124,9 @@ class PackageTool(object): "Wrong descriptor type {}. Options: ns, vnf, nst".format(package_type) ) - # print("To be rendered: {}".format(content)) + self._logger.debug("To be rendered: {}".format(content)) output = template.render(content) - # print(output) + self._logger.debug(output) structure = self.discover_folder_structure( base_directory, package_name, override @@ -150,12 +157,15 @@ class PackageTool(object): f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive) ] self._logger.info("Base directory: {}".format(base_directory)) - self._logger.info("{} Descriptors found to validate".format(len(descriptors_paths))) + self._logger.info( + "{} Descriptors found to validate".format(len(descriptors_paths)) + ) for desc_path in descriptors_paths: with open(desc_path) as descriptor_file: descriptor_data = descriptor_file.read() desc_type = "-" try: + # TODO: refactor validation_im.yaml_validation to @staticmethod desc_type, descriptor_data = validation_im.yaml_validation( self, descriptor_data ) @@ -179,6 +189,7 @@ class PackageTool(object): {"type": desc_type, "path": desc_path, "valid": "OK", "error": "-"} ) except Exception as e: + self._logger.error(f"Validation error: {e}", exc_info=True) table.append( { "type": desc_type, @@ -356,8 +367,16 @@ class PackageTool(object): raise ClientException( "No descriptor file found in: {}".format(package_folder) ) - charm_list = self.build_all_charms(package_folder, skip_charm_build) - return self.build_tarfile(package_folder, charm_list) + + is_sol004_007 = ( + package_handling.get_package_type(package_folder) + != package_handling.OSM_OLD + ) + + charm_list = self.build_all_charms( + package_folder, skip_charm_build, is_sol004_007 + ) + return self.build_compressed_file(package_folder, charm_list, is_sol004_007) def calculate_checksum(self, package_folder): """ @@ -377,13 +396,7 @@ class PackageTool(object): for file_item in files: if "checksums.txt" in file_item: continue - # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html - md5_hash = hashlib.md5() - with open(file_item, "rb") as f: - # Read and update hash in chunks of 4K - for byte_block in iter(lambda: f.read(4096), b""): - md5_hash.update(byte_block) - checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) + checksum.write("{}\t{}\n".format(utils.md5(file_item), file_item)) def create_folders(self, folders, package_type): """ @@ -488,7 +501,7 @@ class PackageTool(object): return missing_paths - def build_all_charms(self, package_folder, skip_charm_build): + def build_all_charms(self, package_folder, skip_charm_build, sol004_007=True): """ **Read the descriptor file, check that the charms referenced are in the folder and compiles them** @@ -499,7 +512,13 @@ class PackageTool(object): self._logger.debug("") charms_set = set() descriptor_file = False - descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] + package_type = package_handling.get_package_type(package_folder) + if sol004_007 and package_type.find("TOSCA") >= 0: + descriptors_paths = [ + f for f in glob.glob(package_folder + "/Definitions/*.yaml") + ] + else: + descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] for file in descriptors_paths: if file.endswith("nfd.yaml"): descriptor_file = True @@ -517,22 +536,32 @@ class PackageTool(object): if charms_set and not skip_charm_build: for charmName in charms_set: if os.path.isdir( - "{}/charms/layers/{}".format(package_folder, charmName) + "{}/{}charms/layers/{}".format( + package_folder, "Scripts/" if sol004_007 else "", charmName + ) ): print( - "Building charm {}/charms/layers/{}".format( - package_folder, charmName + "Building charm {}/{}charms/layers/{}".format( + package_folder, "Scripts/" if sol004_007 else "", charmName ) ) - self.charm_build(package_folder, charmName) + self.charm_build(package_folder, charmName, sol004_007) print("Charm built: {}".format(charmName)) elif os.path.isdir( - "{}/charms/ops/{}".format(package_folder, charmName) + "{}/{}charms/ops/{}".format( + package_folder, "Scripts/" if sol004_007 else "", charmName + ) ): self.charmcraft_build(package_folder, charmName) else: if not os.path.isdir( - "{}/charms/{}".format(package_folder, charmName) + "{}/{}charms/{}".format( + package_folder, "Scripts/" if sol004_007 else "", charmName + ) + ) and not os.path.isfile( + "{}/{}charms/{}".format( + package_folder, "Scripts/" if sol004_007 else "", charmName + ) ): raise ClientException( "The charm: {} referenced in the descriptor file " @@ -545,7 +574,7 @@ class PackageTool(object): def discover_folder_structure(self, base_directory, name, override): """ - **Discover files and folders structure for OSM descriptors given a base_directory and name** + **Discover files and folders structure for SOL004/SOL007 descriptors given a base_directory and name** :params: - base_directory: is the location of the package to be created @@ -558,14 +587,16 @@ class PackageTool(object): files_folders = { "folders": [ ("{}_ns".format(prefix), "ns"), - ("{}_ns/icons".format(prefix), "ns"), - ("{}_ns/charms".format(prefix), "ns"), + ("{}_ns/Licenses".format(prefix), "ns"), + ("{}_ns/Files/icons".format(prefix), "ns"), + ("{}_ns/Scripts/charms".format(prefix), "ns"), ("{}_vnf".format(name), "vnf"), - ("{}_vnf/charms".format(prefix), "vnf"), - ("{}_vnf/cloud_init".format(prefix), "vnf"), - ("{}_vnf/images".format(prefix), "vnf"), - ("{}_vnf/icons".format(prefix), "vnf"), - ("{}_vnf/scripts".format(prefix), "vnf"), + ("{}_vnf/Licenses".format(prefix), "vnf"), + ("{}_vnf/Scripts/charms".format(prefix), "vnf"), + ("{}_vnf/Scripts/cloud_init".format(prefix), "vnf"), + ("{}_vnf/Files/images".format(prefix), "vnf"), + ("{}_vnf/Files/icons".format(prefix), "vnf"), + ("{}_vnf/Scripts/scripts".format(prefix), "vnf"), ("{}_nst".format(prefix), "nst"), ("{}_nst/icons".format(prefix), "nst"), ], @@ -574,7 +605,7 @@ class PackageTool(object): ("{}_ns/README.md".format(prefix), "ns", "readme"), ("{}_vnf/{}_vnfd.yaml".format(prefix, name), "vnf", "descriptor"), ( - "{}_vnf/cloud_init/cloud-config.txt".format(prefix), + "{}_vnf/Scripts/cloud_init/cloud-config.txt".format(prefix), "vnf", "cloud_init", ), @@ -587,21 +618,33 @@ class PackageTool(object): # print("Missing files and folders: {}".format(missing_files_folders)) return missing_files_folders - def charm_build(self, charms_folder, build_name): + def charm_build(self, charms_folder, build_name, sol004_007=True): """ Build the charms inside the package. params: package_folder is the name of the folder where is the charms to compile. build_name is the name of the layer or interface """ self._logger.debug("") - os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder) + + if sol004_007: + os.environ["JUJU_REPOSITORY"] = "{}/Scripts/charms".format(charms_folder) + else: + os.environ["JUJU_REPOSITORY"] = "{}/charms".format(charms_folder) + os.environ["CHARM_LAYERS_DIR"] = "{}/layers".format( os.environ["JUJU_REPOSITORY"] ) os.environ["CHARM_INTERFACES_DIR"] = "{}/interfaces".format( os.environ["JUJU_REPOSITORY"] ) - os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder) + + if sol004_007: + os.environ["CHARM_BUILD_DIR"] = "{}/Scripts/charms/builds".format( + charms_folder + ) + else: + os.environ["CHARM_BUILD_DIR"] = "{}/charms/builds".format(charms_folder) + if not os.path.exists(os.environ["CHARM_BUILD_DIR"]): os.makedirs(os.environ["CHARM_BUILD_DIR"]) src_folder = "{}/{}".format(os.environ["CHARM_LAYERS_DIR"], build_name) @@ -617,7 +660,7 @@ class PackageTool(object): build_name is the name of the layer or interface """ self._logger.debug("Building charm {}".format(charm_name)) - src_folder = f"{package_folder}/charms/ops/{charm_name}" + src_folder = f"{package_folder}/Scripts/charms/ops/{charm_name}" current_directory = os.getcwd() os.chdir(src_folder) try: @@ -632,6 +675,62 @@ class PackageTool(object): finally: os.chdir(current_directory) + def build_compressed_file(self, package_folder, charm_list=None, sol004_007=True): + if sol004_007: + return self.build_zipfile(package_folder, charm_list) + else: + return self.build_tarfile(package_folder, charm_list) + + def build_zipfile(self, package_folder, charm_list=None): + """ + Creates a zip file given a package_folder + params: package_folder is the name of the folder to be packaged + returns: .zip name + """ + self._logger.debug("") + cwd = None + try: + directory_name, package_name = self.create_temp_dir_sol004_007( + package_folder, charm_list + ) + cwd = os.getcwd() + os.chdir(directory_name) + package_type = package_handling.get_package_type(package_folder) + print(package_type) + + if ( + package_handling.SOL007 == package_type + or package_handling.SOL007_TOSCA == package_type + ): + the_package = SOL007Package(package_folder) + elif ( + package_handling.SOL004 == package_type + or package_handling.SOL004_TOSCA == package_type + ): + the_package = SOL004Package(package_folder) + + the_package.create_or_update_metadata_file() + + the_zip_package = shutil.make_archive( + os.path.join(cwd, package_name), + "zip", + os.path.join(directory_name, package_name), + ) + + print("Package created: {}".format(the_zip_package)) + + return the_zip_package + + except Exception as exc: + raise ClientException( + "failure during build of zip file (create temp dir, calculate checksum, " + "zip file): {}".format(exc) + ) + finally: + if cwd: + os.chdir(cwd) + shutil.rmtree(os.path.join(package_folder, "tmp")) + def build_tarfile(self, package_folder, charm_list=None): """ Creates a .tar.gz file given a package_folder @@ -714,9 +813,12 @@ class PackageTool(object): self._logger.debug( "Copying tree: {} -> {}".format(s_charm, d_temp) ) - shutil.copytree( - s_charm, d_temp, symlinks=True, ignore=ignore - ) + if os.path.isdir(s_charm): + shutil.copytree( + s_charm, d_temp, symlinks=True, ignore=ignore + ) + else: + shutil.copy2(s_charm, d_temp) self._logger.debug("DONE") else: self._logger.debug("Copying tree: {} -> {}".format(s, d)) @@ -730,6 +832,75 @@ class PackageTool(object): self._logger.debug("DONE") return directory_name, package_name + def copy_tree(self, s, d, ignore): + self._logger.debug("Copying tree: {} -> {}".format(s, d)) + shutil.copytree(s, d, symlinks=True, ignore=ignore) + self._logger.debug("DONE") + + def create_temp_dir_sol004_007(self, package_folder, charm_list=None): + """ + Method to create a temporary folder where we can move the files in package_folder + """ + self._logger.debug("") + ignore_patterns = ".gitignore" + ignore = shutil.ignore_patterns(ignore_patterns) + directory_name = os.path.abspath(package_folder) + package_name = os.path.basename(directory_name) + directory_name += "/tmp" + os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True) + self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name)) + for item in os.listdir(package_folder): + self._logger.debug("Item: {}".format(item)) + if item != "tmp": + s = os.path.join(package_folder, item) + d = os.path.join(os.path.join(directory_name, package_name), item) + if os.path.isdir(s): + if item == "Scripts": + os.makedirs(d, exist_ok=True) + scripts_folder = s + for script_item in os.listdir(scripts_folder): + scripts_destination_folder = os.path.join(d, script_item) + if script_item == "charms": + s_builds = os.path.join( + scripts_folder, script_item, "builds" + ) + for charm in charm_list: + self._logger.debug("Copying charm {}".format(charm)) + if charm in os.listdir( + os.path.join(scripts_folder, script_item) + ): + s_charm = os.path.join( + scripts_folder, script_item, charm + ) + elif charm in os.listdir(s_builds): + s_charm = os.path.join(s_builds, charm) + else: + raise ClientException( + "The charm {} referenced in the descriptor file " + "could not be found in {}/charms or in {}/charms/builds".format( + charm, package_folder, package_folder + ) + ) + d_temp = os.path.join( + scripts_destination_folder, charm + ) + self.copy_tree(s_charm, d_temp, ignore) + else: + self.copy_tree( + os.path.join(scripts_folder, script_item), + scripts_destination_folder, + ignore, + ) + else: + self.copy_tree(s, d, ignore) + else: + if item in ignore_patterns: + continue + self._logger.debug("Copying file: {} -> {}".format(s, d)) + shutil.copy2(s, d) + self._logger.debug("DONE") + return directory_name, package_name + def charms_search(self, descriptor_file, desc_type): self._logger.debug( "descriptor_file: {}, desc_type: {}".format(descriptor_file, desc_type) @@ -767,7 +938,7 @@ class PackageTool(object): def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type): self._logger.debug("") - charms_set = [] + charms_set = set() for k1, v1 in osm_im_dict.items(): for k2, v2 in v1.items(): for entry in v2: