From: gomezchavez Date: Mon, 28 Jun 2021 15:59:31 +0000 (-0400) Subject: Fixes bug 1657 repo generation from osm-packages X-Git-Tag: v9.1.4~1 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fosmclient.git;a=commitdiff_plain;h=d1e681dec82db20288abcc84cc9a4f2648060316 Fixes bug 1657 repo generation from osm-packages Change-Id: I9cdbca98484bb7431f14b368b65686794e0aaec4 Signed-off-by: gomezchavez --- diff --git a/osmclient/sol005/osmrepo.py b/osmclient/sol005/osmrepo.py index aa2a5f0..469d6fd 100644 --- a/osmclient/sol005/osmrepo.py +++ b/osmclient/sol005/osmrepo.py @@ -86,7 +86,12 @@ class OSMRepo(Repo): else: raise Exception('repository in url {} unreachable'.format(repository.get('url'))) except Exception as e: - logging.error("Error cannot read from repository {} '{}': {}".format(repository['name'], repository['url'], e)) + self._logger.error( + "Error cannot read from repository {} '{}': {}".format( + repository["name"], repository["url"], e + ), + exc_info=True + ) continue vnf_repos_filtered = [] @@ -150,29 +155,51 @@ class OSMRepo(Repo): :param origin: origin directory for getting all the artifacts :param destination: destination folder for create and index the valid artifacts """ - self._logger.debug("") - if destination == '.': + self._logger.debug("Starting index composition") + if destination == ".": if origin == destination: destination = 'repository' destination = abspath(destination) origin = abspath(origin) - - if origin[0] != '/': + self._logger.debug(f"Paths {destination}, {origin}") + if origin[0] != "/": origin = join(getcwd(), origin) if destination[0] != '/': destination = join(getcwd(), destination) self.init_directory(destination) - artifacts = [f for f in listdir(origin) if isfile(join(origin, f))] - directories = [f for f in listdir(origin) if isdir(join(origin, f))] + artifacts = [] + directories = [] + for f in listdir(origin): + if isfile(join(origin, f)) and f.endswith('.tar.gz'): + artifacts.append(f) + elif isdir(join(origin, f)) and f != destination.split('/')[-1] and not f.startswith('.'): + directories.append(f) # TODO: Document that nested directories are not supported + else: + self._logger.debug(f"Ignoring {f}") for artifact in artifacts: - self.register_artifact_in_repository(join(origin, artifact), destination, source='file') + self.register_artifact_in_repository( + join(origin, artifact), destination, source="artifact" + ) for artifact in directories: - self.register_artifact_in_repository(join(origin, artifact), destination, source='directory') - print("\nFinal Results: ") - print("VNF Packages Indexed: " + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml")))) - print("NS Packages Indexed: " + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml")))) + self.register_artifact_in_repository( + join(origin, artifact), destination, source="directory" + ) + self._logger.info("\nFinal Results: ") + self._logger.info( + "VNF Packages Indexed: " + + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml"))) + ) + self._logger.info( + "NS Packages Indexed: " + + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml"))) + ) + + self._logger.info( + "NST Packages Indexed: " + + str(len(glob.glob(destination + "/nst/*/*/metadata.yaml"))) + ) def md5(self, fname): """ @@ -189,46 +216,91 @@ class OSMRepo(Repo): def fields_building(self, descriptor_dict, file, package_type): """ - From an artifact descriptor, obtain the fields required for indexing - :param descriptor_dict: artifact description - :param file: artifact package - :param package_type: type of artifact (vnf or ns) - :return: fields + From an artifact descriptor, obtain the fields required for indexing + :param descriptor_dict: artifact description + :param file: artifact package + :param package_type: type of artifact (vnf, ns, nst) + :return: fields """ self._logger.debug("") + fields = {} base_path = '/{}/'.format(package_type) aux_dict = {} if package_type == "vnf": - if descriptor_dict.get('vnfd-catalog', False): - aux_dict = descriptor_dict.get('vnfd-catalog', {}).get('vnfd', [{}])[0] + if descriptor_dict.get("vnfd-catalog", False): + aux_dict = descriptor_dict.get("vnfd-catalog", {}).get("vnfd", [{}])[0] + elif descriptor_dict.get("vnfd:vnfd-catalog"): + aux_dict = descriptor_dict.get("vnfd:vnfd-catalog", {}).get("vnfd", [{}])[0] + elif descriptor_dict.get("vnfd"): + aux_dict = descriptor_dict["vnfd"] + if aux_dict.get("vnfd"): + aux_dict = aux_dict['vnfd'][0] else: - aux_dict = descriptor_dict.get('vnfd:vnfd-catalog', {}).get('vnfd', [{}])[0] - + msg = f"Unexpected descriptor format {descriptor_dict}" + self._logger.error(msg) + raise ValueError(msg) + self._logger.debug(f"Extracted descriptor info for {package_type}: {aux_dict}") images = [] - for vdu in aux_dict.get('vdu', ()): - images.append(vdu.get('image')) - fields['images'] = images - if package_type == "ns": - if descriptor_dict.get('nsd-catalog', False): - aux_dict = descriptor_dict.get('nsd-catalog', {}).get('nsd', [{}])[0] + for vdu in aux_dict.get("vdu", aux_dict.get('kdu', ())): + images.append(vdu.get("image", vdu.get('name'))) + fields["images"] = images + elif package_type == "ns": + if descriptor_dict.get("nsd-catalog", False): + aux_dict = descriptor_dict.get("nsd-catalog", {}).get("nsd", [{}])[0] + elif descriptor_dict.get("nsd:nsd-catalog"): + aux_dict = descriptor_dict.get("nsd:nsd-catalog", {}).get("nsd", [{}])[0] + elif descriptor_dict.get("nsd"): + aux_dict = descriptor_dict['nsd'] + if aux_dict.get("nsd"): + aux_dict = descriptor_dict["nsd"]["nsd"][0] else: - aux_dict = descriptor_dict.get('nsd:nsd-catalog', {}).get('nsd', [{}])[0] - + msg = f"Unexpected descriptor format {descriptor_dict}" + self._logger.error(msg) + raise ValueError(msg) vnfs = [] - - for vnf in aux_dict.get('constituent-vnfd', ()): - vnfs.append(vnf.get('vnfd-id-ref')) - self._logger.debug('Used VNFS in the NSD: ' + str(vnfs)) - fields['vnfd-id-ref'] = vnfs - - fields['name'] = aux_dict.get('name') - fields['id'] = aux_dict.get('id') - fields['description'] = aux_dict.get('description') - fields['vendor'] = aux_dict.get('vendor') - fields['version'] = aux_dict.get('version', '1.0') - fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'), - fields.get('version')) + if aux_dict.get("constituent-vnfd"): + for vnf in aux_dict.get("constituent-vnfd", ()): + vnfs.append(vnf.get("vnfd-id-ref")) + else: + vnfs = aux_dict.get('vnfd-id') + self._logger.debug("Used VNFS in the NSD: " + str(vnfs)) + fields["vnfd-id-ref"] = vnfs + elif package_type == 'nst': + if descriptor_dict.get("nst-catalog", False): + aux_dict = descriptor_dict.get("nst-catalog", {}).get("nst", [{}])[0] + elif descriptor_dict.get("nst:nst-catalog"): + aux_dict = descriptor_dict.get("nst:nst-catalog", {}).get("nst", [{}])[0] + elif descriptor_dict.get("nst"): + aux_dict = descriptor_dict['nst'] + if aux_dict.get("nst"): + aux_dict = descriptor_dict["nst"]["nst"][0] + nsds = [] + for nsd in aux_dict.get("netslice-subnet", ()): + nsds.append(nsd.get("nsd-ref")) + self._logger.debug("Used NSDs in the NST: " + str(nsds)) + if not nsds: + msg = f"Unexpected descriptor format {descriptor_dict}" + self._logger.error(msg) + raise ValueError(msg) + fields["nsd-id-ref"] = nsds + else: + msg = f"Unexpected descriptor format {descriptor_dict}" + self._logger.error(msg) + raise ValueError(msg) + + fields["name"] = aux_dict.get("name") + fields["id"] = aux_dict.get("id") + fields["description"] = aux_dict.get("description") + fields["vendor"] = aux_dict.get("vendor") + fields["version"] = str(aux_dict.get("version", "1.0")) + fields["path"] = "{}{}/{}/{}-{}.tar.gz".format( + base_path, + fields["id"], + fields["version"], + fields.get("id"), + fields.get("version"), + ) return fields def zip_extraction(self, file_name): @@ -251,9 +323,10 @@ class OSMRepo(Repo): def validate_artifact(self, path, source): """ - Validation of artifact. - :param path: file path - :return: status details, status, fields, package_type + Validation of artifact. + :param path: file path + :param source: flag to select the correct file type (directory or artifact) + :return: status details, status, fields, package_type """ self._logger.debug("") package_type = '' @@ -268,20 +341,37 @@ class OSMRepo(Repo): with open(descriptor_file, 'r') as f: descriptor_data = f.read() + self._logger.debug(f"Descriptor data: {descriptor_data}") validation = validation_im() desc_type, descriptor_dict = validation.yaml_validation(descriptor_data) - validation_im.pyangbind_validation(self, desc_type, descriptor_dict) - if 'vnf' in list(descriptor_dict.keys())[0]: - package_type = 'vnf' + try: + validation_im.pyangbind_validation(self, desc_type, descriptor_dict) + except Exception as e: + self._logger.error(e, exc_info=True) + raise e + descriptor_type_ref = list(descriptor_dict.keys())[0].lower() + if "vnf" in descriptor_type_ref: + package_type = "vnf" + elif "nst" in descriptor_type_ref: + package_type = "nst" + elif "ns" in descriptor_type_ref: + package_type = "ns" else: - # raise ClientException("Not VNF package") - package_type = 'ns' - + msg = f"Unknown package type {descriptor_type_ref}" + self._logger.error(msg) + raise ValueError(msg) self._logger.debug("Descriptor: {}".format(descriptor_dict)) fields = self.fields_building(descriptor_dict, path, package_type) - self._logger.debug("Descriptor sucessfully validated") - return {"detail": "{}D successfully validated".format(package_type.upper()), - "code": "OK"}, True, fields, package_type + self._logger.debug(f"Descriptor successfully validated {fields}") + return ( + { + "detail": "{}D successfully validated".format(package_type.upper()), + "code": "OK", + }, + True, + fields, + package_type, + ) except Exception as e: # Delete the folder we just created return {"detail": str(e)}, False, {}, package_type @@ -297,24 +387,27 @@ class OSMRepo(Repo): """ self._logger.debug("") pt = PackageTool() - compresed = False + compressed = False try: fields = {} _, valid, fields, package_type = self.validate_artifact(path, source) if not valid: - raise Exception('{} {} Not well configured.'.format(package_type.upper(), str(path))) + raise Exception( + "{} {} Not well configured.".format(package_type.upper(), str(path)) + ) else: - if source == 'directory': + if source == "directory": path = pt.build(path) - compresed = True - fields['checksum'] = self.md5(path) + self._logger.debug(f"Directory path {path}") + compressed = True + fields["checksum"] = self.md5(path) self.indexation(destination, path, package_type, fields) except Exception as e: self._logger.exception("Error registering artifact in Repository: {}".format(e)) finally: - if source == 'directory' and compresed: + if source == "directory" and compressed: remove(path) def indexation(self, destination, path, package_type, fields): @@ -364,7 +457,7 @@ class OSMRepo(Repo): default_flow_style=False, width=80, indent=4) self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path))) - def current_datatime(self): + def current_datetime(self): """ Datetime Generator :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z" @@ -374,18 +467,25 @@ class OSMRepo(Repo): def init_directory(self, destination): """ - Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns - :param destination: - :return: + Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns + :param destination: + :return: """ self._logger.debug("") if not isdir(destination): mkdir(destination) - if not isfile(join(destination, 'index.yaml')): - mkdir(join(destination, 'vnf')) - mkdir(join(destination, 'ns')) - index_data = {'apiVersion': 'v1', 'generated': self.current_datatime(), 'vnf_packages': {}, - 'ns_packages': {}} - with open(join(destination, 'index.yaml'), 'w') as outfile: - yaml.safe_dump(index_data, outfile, default_flow_style=False, width=80, indent=4) - + if not isfile(join(destination, "index.yaml")): + mkdir(join(destination, "vnf")) + mkdir(join(destination, "ns")) + mkdir(join(destination, "nst")) + index_data = { + "apiVersion": "v1", + "generated": self.current_datetime(), + "vnf_packages": {}, + "ns_packages": {}, + "nst_packages": {}, + } + with open(join(destination, "index.yaml"), "w") as outfile: + yaml.safe_dump( + index_data, outfile, default_flow_style=False, width=80, indent=4 + ) diff --git a/osmclient/sol005/tests/test_osmrepo.py b/osmclient/sol005/tests/test_osmrepo.py new file mode 100644 index 0000000..f337ae9 --- /dev/null +++ b/osmclient/sol005/tests/test_osmrepo.py @@ -0,0 +1,33 @@ +# Copyright 2021 ATOS. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import shutil +from pathlib import Path + +from osmclient.sol005.osmrepo import OSMRepo + + +class TestOSMRepo(unittest.TestCase): + def setUp(self): + self.repo = OSMRepo() + + def test_init_repo_structure(self): + # TODO: Mock filesystem after refactoring from os to pathlib + # TODO: Mock OSM IM repo if possible + repo_base = Path(__file__).parent / Path("test_repo") + expected_index_file_path = repo_base / Path("index.yaml") + self.repo.init_directory(str(repo_base)) + self.assertTrue(expected_index_file_path.exists()) + shutil.rmtree(expected_index_file_path.parent)