X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osmclient%2Fcommon%2Fpackage_tool.py;h=3da96d57e0013d6bf3f5207b049b724d25473c88;hb=5f8a12367e3fba2a6f2ee7025e801dbff6da37a7;hp=7cc10becc4a80c723fb1eb1aecaa683dd90369d6;hpb=b7463a4821eabd214b2b38d54db38a5a1aae53d6;p=osm%2Fosmclient.git diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index 7cc10be..3da96d5 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -23,11 +23,15 @@ import tarfile import hashlib from osm_im.validation import Validation as validation_im from jinja2 import Environment, PackageLoader - +import subprocess +import shutil +import yaml +import logging class PackageTool(object): def __init__(self, client=None): self._client = client + self._logger = logging.getLogger('osmclient') def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage, interfaces, vendor, detailed, netslice_subnets, netslice_vlds): @@ -50,7 +54,7 @@ class PackageTool(object): :return: status """ - + self._logger.debug("") # print("location: {}".format(osmclient.__path__)) file_loader = PackageLoader("osmclient") env = Environment(loader=file_loader) @@ -80,7 +84,7 @@ class PackageTool(object): self.create_files(structure["files"], output, package_type) return "Created" - def validate(self, base_directory): + def validate(self, base_directory, recursive=True): """ **Validate OSM Descriptors given a path** @@ -89,8 +93,12 @@ class PackageTool(object): :return: List of dict of validated descriptors. keys: type, path, valid, error """ + self._logger.debug("") table = [] - descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=True)] + if recursive: + descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)] + else: + descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)] print("Base directory: {}".format(base_directory)) print("{} Descriptors found to validate".format(len(descriptors_paths))) for desc_path in descriptors_paths: @@ -105,7 +113,7 @@ class PackageTool(object): table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)}) return table - def build(self, package_folder, skip_validation=True): + def build(self, package_folder, skip_validation=False, skip_charm_build=False): """ **Creates a .tar.gz file given a package_folder** @@ -115,20 +123,23 @@ class PackageTool(object): :returns: message result for the build process """ - + self._logger.debug("") + package_folder = package_folder.rstrip('/') if not os.path.exists("{}".format(package_folder)): - return "Fail, package is not in the specified route" + return "Fail, package is not in the specified path" if not skip_validation: - results = self.validate(package_folder) - for result in results: - if result["valid"] != "OK": - return("There was an error validating the file: {} with error: {}".format(result["path"], - result["error"])) - self.calculate_checksum(package_folder) - with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive: - print("Adding File: {}".format(package_folder)) - archive.add('{}'.format(package_folder), recursive=True) - return "Created {}.tar.gz".format(package_folder) + print('Validating package {}'.format(package_folder)) + results = self.validate(package_folder, recursive=False) + if results: + for result in results: + if result["valid"] != "OK": + raise ClientException("There was an error validating the file {} with error: {}" + .format(result["path"], result["error"])) + print('Validation OK') + else: + raise ClientException("No descriptor file found in: {}".format(package_folder)) + charm_list = self.build_all_charms(package_folder, skip_charm_build) + return self.build_tarfile(package_folder, charm_list) def calculate_checksum(self, package_folder): """ @@ -138,19 +149,19 @@ class PackageTool(object): - package_folder: is the folder where we have the files to calculate the checksum :returns: None """ - files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)] - checksum = open("{}/checksum.txt".format(package_folder), "w+") - for file_item in files: - if "checksum.txt" in file_item: - continue - # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html - md5_hash = hashlib.md5() - with open(file_item, "rb") as f: - # Read and update hash in chunks of 4K - for byte_block in iter(lambda: f.read(4096), b""): - md5_hash.update(byte_block) - checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) - checksum.close() + self._logger.debug("") + files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True) if os.path.isfile(f)] + with open("{}/checksums.txt".format(package_folder), "w+") as checksum: + for file_item in files: + if "checksums.txt" in file_item: + continue + # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html + md5_hash = hashlib.md5() + with open(file_item, "rb") as f: + # Read and update hash in chunks of 4K + for byte_block in iter(lambda: f.read(4096), b""): + md5_hash.update(byte_block) + checksum.write("{}\t{}\n".format(md5_hash.hexdigest(), file_item)) def create_folders(self, folders, package_type): """ @@ -215,6 +226,7 @@ class PackageTool(object): :return: None """ + self._logger.debug("") for file_item, file_package, file_type in files: if package_type == file_package: if file_type == "descriptor": @@ -234,6 +246,7 @@ class PackageTool(object): :return: Missing paths Dict """ + self._logger.debug("") missing_paths = {} folders = [] files = [] @@ -249,6 +262,42 @@ class PackageTool(object): return missing_paths + def build_all_charms(self, package_folder, skip_charm_build): + """ + **Read the descriptor file, check that the charms referenced are in the folder and compiles them** + + :params: + - packet_folder: is the location of the package + :return: Files and Folders not found. In case of override, it will return all file list + """ + self._logger.debug("") + listCharms = [] + descriptor_file = False + descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] + for file in descriptors_paths: + if file.endswith('nfd.yaml'): + descriptor_file = True + listCharms = self.charms_search(file, 'vnf') + if file.endswith('nsd.yaml'): + descriptor_file = True + listCharms = self.charms_search(file, 'ns') + print("List of charms in the descriptor: {}".format(listCharms)) + if not descriptor_file: + raise ClientException ('descriptor name is not correct in: {}'.format(package_folder)) + if listCharms and not skip_charm_build: + for charmName in listCharms: + if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)): + print('Building charm {}/charms/layers/{}'.format(package_folder, charmName)) + self.charm_build(package_folder, charmName) + print('Charm built {}'.format(charmName)) + else: + if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)): + raise ClientException ('The charm: {} referenced in the descriptor file ' + 'is not present either in {}/charms or in {}/charms/layers'. + format(charmName, package_folder,package_folder)) + self._logger.debug("Return list of charms: {}".format(listCharms)) + return listCharms + def discover_folder_structure(self, base_directory, name, override): """ **Discover files and folders structure for OSM descriptors given a base_directory and name** @@ -259,6 +308,7 @@ class PackageTool(object): - override: is the flag used to indicate the creation of the list even if the file exist to override it :return: Files and Folders not found. In case of override, it will return all file list """ + self._logger.debug("") prefix = "{}/{}".format(base_directory, name) files_folders = {"folders": [("{}_ns".format(prefix), "ns"), ("{}_ns/icons".format(prefix), "ns"), @@ -284,3 +334,126 @@ class PackageTool(object): missing_files_folders = self.check_files_folders(files_folders, override) # print("Missing files and folders: {}".format(missing_files_folders)) return missing_files_folders + + def charm_build(self, charms_folder, build_name): + """ + Build the charms inside the package. + params: package_folder is the name of the folder where is the charms to compile. + build_name is the name of the layer or interface + """ + self._logger.debug("") + os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder) + os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY']) + os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY']) + os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder) + if not os.path.exists(os.environ['CHARM_BUILD_DIR']): + os.makedirs(os.environ['CHARM_BUILD_DIR']) + src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name) + result = subprocess.run(["charm", "build", "{}".format(src_folder)]) + if result.returncode == 1: + raise ClientException("failed to build the charm: {}".format(src_folder)) + self._logger.verbose("charm {} built".format(src_folder)) + + def build_tarfile(self, package_folder, charm_list=None): + """ + Creates a .tar.gz file given a package_folder + params: package_folder is the name of the folder to be packaged + returns: .tar.gz name + """ + self._logger.debug("") + cwd = None + try: + directory_name, package_name = self.create_temp_dir(package_folder, charm_list) + cwd = os.getcwd() + os.chdir(directory_name) + self.calculate_checksum(package_name) + with tarfile.open("{}.tar.gz".format(package_name), mode='w:gz') as archive: + print("Adding File: {}".format(package_name)) + archive.add('{}'.format(package_name), recursive=True) + #return "Created {}.tar.gz".format(package_folder) + #self.build("{}".format(os.path.basename(package_folder))) + os.chdir(cwd) + except Exception as exc: + if cwd: + os.chdir(cwd) + shutil.rmtree(os.path.join(package_folder, "tmp")) + raise ClientException('failure during build of targz file (create temp dir, calculate checksum, tar.gz file): {}'.format(exc)) + created_package = "{}/{}.tar.gz".format(package_folder, package_name) + os.rename("{}/{}.tar.gz".format(directory_name, package_name), + created_package) + os.rename("{}/{}/checksums.txt".format(directory_name, package_name), + "{}/checksums.txt".format(package_folder)) + shutil.rmtree(os.path.join(package_folder, "tmp")) + print("Package created: {}".format(created_package)) + return created_package + + def create_temp_dir(self, package_folder, charm_list=None): + """ + Method to create a temporary folder where we can move the files in package_folder + """ + self._logger.debug("") + ignore_patterns = ('.gitignore') + ignore = shutil.ignore_patterns(ignore_patterns) + directory_name = os.path.abspath(package_folder) + package_name = os.path.basename(directory_name) + directory_name += "/tmp" + os.makedirs("{}/{}".format(directory_name, package_name), exist_ok=True) + self._logger.debug("Makedirs DONE: {}/{}".format(directory_name, package_name)) + for item in os.listdir(package_folder): + self._logger.debug("Item: {}".format(item)) + if item != "tmp": + s = os.path.join(package_folder, item) + d = os.path.join(os.path.join(directory_name, package_name), item) + if os.path.isdir(s): + if item == "charms": + os.makedirs(d, exist_ok=True) + s_builds = os.path.join(s, "builds") + for charm in charm_list: + self._logger.debug("Copying charm {}".format(charm)) + if charm in os.listdir(s): + s_charm = os.path.join(s, charm) + elif charm in os.listdir(s_builds): + s_charm = os.path.join(s_builds, charm) + else: + raise ClientException('The charm {} referenced in the descriptor file ' + 'could not be found in {}/charms or in {}/charms/builds'. + format(charm, package_folder, package_folder)) + d_temp = os.path.join(d, charm) + self._logger.debug("Copying tree: {} -> {}".format(s_charm, d_temp)) + shutil.copytree(s_charm, d_temp, symlinks=True, ignore=ignore) + self._logger.debug("DONE") + else: + self._logger.debug("Copying tree: {} -> {}".format(s,d)) + shutil.copytree(s, d, symlinks=True, ignore=ignore) + self._logger.debug("DONE") + else: + if item in ignore_patterns: + continue + self._logger.debug("Copying file: {} -> {}".format(s,d)) + shutil.copy2(s, d) + self._logger.debug("DONE") + return directory_name, package_name + + def charms_search(self, descriptor_file, desc_type): + self._logger.debug("") + dict = {} + list = [] + with open("{}".format(descriptor_file)) as yaml_desc: + dict = yaml.safe_load(yaml_desc) + for k1, v1 in dict.items(): + for k2, v2 in v1.items(): + for entry in v2: + if '{}-configuration'.format(desc_type) in entry: + name = entry['{}-configuration'.format(desc_type)] + for k3, v3 in name.items(): + if 'charm' in v3: + list.append((v3['charm'])) + if 'vdu' in entry: + name = entry['vdu'] + for vdu in name: + if 'vdu-configuration' in vdu: + for k4, v4 in vdu['vdu-configuration'].items(): + if 'charm' in v4: + list.append((v4['charm'])) + return list +