From: beierlm Date: Sat, 28 Nov 2020 01:03:08 +0000 (-0500) Subject: Fix repo-index X-Git-Tag: release-v9.0-start~10 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F49%2F10049%2F2;p=osm%2Fosmclient.git Fix repo-index Fully changes over to ruamel as the mixing of yaml and ruamel packages was causing exceptions. Report exceptions instead of silently ignoring them. Change-Id: Iede687b558901340a9aa30670f3474c776a690a2 Signed-off-by: beierlm --- diff --git a/osmclient/sol005/osmrepo.py b/osmclient/sol005/osmrepo.py index b4d49cf..c2bb0c9 100644 --- a/osmclient/sol005/osmrepo.py +++ b/osmclient/sol005/osmrepo.py @@ -15,22 +15,22 @@ """ OSM Repo API handling """ -from osmclient.common.exceptions import ClientException -from osmclient.sol005.repo import Repo -from osmclient.common.package_tool import PackageTool -import requests +import glob +import hashlib import logging -import tempfile +from os import listdir, mkdir, getcwd, remove +from os.path import isfile, isdir, join, abspath from shutil import copyfile, rmtree -import yaml import tarfile -import glob -from packaging import version as versioning +import tempfile import time -from os import listdir, mkdir, getcwd, remove -from os.path import isfile, isdir, join, abspath -import hashlib + from osm_im.validation import Validation as validation_im +from osmclient.common.exceptions import ClientException +from osmclient.common.package_tool import PackageTool +from osmclient.sol005.repo import Repo +from packaging import version as versioning +import requests import ruamel.yaml @@ -64,7 +64,7 @@ class OSMRepo(Repo): r = requests.get('{}/index.yaml'.format(repository.get('url'))) if r.status_code == 200: - repo_list = yaml.safe_load(r.text) + repo_list = ruamel.yaml.safe_load(r.text) vnf_packages = repo_list.get('{}_packages'.format(pkgtype)) for repo in vnf_packages: versions = vnf_packages.get(repo) @@ -127,7 +127,7 @@ class OSMRepo(Repo): f.write(r.raw.read()) f_name = f.name if not f_name: - raise ClientException("{} {} not found at repo {}".format(pkgtype,name, repo)) + raise ClientException("{} {} not found at repo {}".format(pkgtype, name, repo)) return f_name def pkg_get(self, pkgtype, name, repo, version, filter): @@ -137,7 +137,7 @@ class OSMRepo(Repo): raise ClientException('Package not found') folder, descriptor = self.zip_extraction(pkg_name) with open(descriptor) as pkg: - pkg_descriptor = yaml.safe_load(pkg) + pkg_descriptor = ruamel.yaml.safe_load(pkg) rmtree(folder, ignore_errors=False) if ((pkgtype == 'vnf' and (pkg_descriptor.get('vnfd') or pkg_descriptor.get('vnfd:vnfd_catalog'))) or (pkgtype == 'ns' and (pkg_descriptor.get('nsd') or pkg_descriptor.get('nsd:nsd_catalog')))): @@ -224,8 +224,8 @@ class OSMRepo(Repo): fields['description'] = aux_dict.get('description') fields['vendor'] = aux_dict.get('vendor') fields['version'] = aux_dict.get('version', '1.0') - fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'), \ - fields.get('version')) + fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'), + fields.get('version')) return fields def zip_extraction(self, file_name): @@ -306,7 +306,7 @@ class OSMRepo(Repo): self.indexation(destination, path, package_type, fields) except Exception as e: - self._logger.debug("Error registering artifact in Repository: {}".format(e)) + self._logger.exception("Error registering artifact in Repository: {}".format(e)) finally: if source == 'directory' and compresed: @@ -331,28 +331,28 @@ class OSMRepo(Repo): mkdir(final_path) copyfile(path, final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz') - yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'), - Dumper=ruamel.yaml.RoundTripDumper) - index = yaml.load(open(destination + '/index.yaml')) + ruamel.yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'), + Dumper=ruamel.yaml.RoundTripDumper) + index = ruamel.yaml.load(open(destination + '/index.yaml'), Loader=ruamel.yaml.Loader) index['{}_packages'.format(package_type)][fields.get('id')][fields.get('version')] = data_ind if versioning.parse(index['{}_packages'.format(package_type)][fields.get('id')][ 'latest']) < versioning.parse(fields.get('version')): index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get( 'version') - yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper) + ruamel.yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper) self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path))) else: mkdir(destination + '/{}/'.format(package_type) + fields.get('id')) mkdir(final_path) copyfile(path, final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz') - yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) - index = yaml.load(open(destination + '/index.yaml')) + ruamel.yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) + index = ruamel.yaml.load(open(destination + '/index.yaml'), Loader=ruamel.yaml.Loader) index['{}_packages'.format(package_type)][fields.get('id')] = {fields.get('version'): data_ind} index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get('version') - yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) + ruamel.yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path))) def current_datatime(self): @@ -376,4 +376,4 @@ class OSMRepo(Repo): index_data = {'apiVersion': 'v1', 'generated': self.current_datatime(), 'vnf_packages': {}, 'ns_packages': {}} with open(join(destination, 'index.yaml'), 'w') as outfile: - yaml.dump(index_data, outfile, default_flow_style=False) + ruamel.yaml.dump(index_data, outfile, default_flow_style=False)