Fix repo-index

Fully changes over to ruamel as the mixing of yaml and
ruamel packages was causing exceptions.

Report exceptions instead of silently ignoring them.

Change-Id: Iede687b558901340a9aa30670f3474c776a690a2
Signed-off-by: beierlm <mark.beierl@canonical.com>

diff --git a/osmclient/sol005/osmrepo.py b/osmclient/sol005/osmrepo.py
index b4d49cf..c2bb0c9 100644 (file)
--- a/osmclient/sol005/osmrepo.py
+++ b/osmclient/sol005/osmrepo.py
@@ -15,22 +15,22 @@
 """
 OSM Repo API handling
 """
-from osmclient.common.exceptions import ClientException
-from osmclient.sol005.repo import Repo
-from osmclient.common.package_tool import PackageTool
-import requests
+import glob
+import hashlib
 import logging
-import tempfile
+from os import listdir, mkdir, getcwd, remove
+from os.path import isfile, isdir, join, abspath
 from shutil import copyfile, rmtree
-import yaml
 import tarfile
-import glob
-from packaging import version as versioning
+import tempfile
 import time
-from os import listdir, mkdir, getcwd, remove
-from os.path import isfile, isdir, join, abspath
-import hashlib
+
 from osm_im.validation import Validation as validation_im
+from osmclient.common.exceptions import ClientException
+from osmclient.common.package_tool import PackageTool
+from osmclient.sol005.repo import Repo
+from packaging import version as versioning
+import requests
 import ruamel.yaml
 
 
@@ -64,7 +64,7 @@ class OSMRepo(Repo):
                 r = requests.get('{}/index.yaml'.format(repository.get('url')))
 
                 if r.status_code == 200:
-                    repo_list = yaml.safe_load(r.text)
+                    repo_list = ruamel.yaml.safe_load(r.text)
                     vnf_packages = repo_list.get('{}_packages'.format(pkgtype))
                     for repo in vnf_packages:
                         versions = vnf_packages.get(repo)
@@ -127,7 +127,7 @@ class OSMRepo(Repo):
                         f.write(r.raw.read())
                         f_name = f.name
                     if not f_name:
-                        raise ClientException("{} {} not found at repo {}".format(pkgtype,name, repo))
+                        raise ClientException("{} {} not found at repo {}".format(pkgtype, name, repo))
         return f_name
 
     def pkg_get(self, pkgtype, name, repo, version, filter):
@@ -137,7 +137,7 @@ class OSMRepo(Repo):
             raise ClientException('Package not found')
         folder, descriptor = self.zip_extraction(pkg_name)
         with open(descriptor) as pkg:
-            pkg_descriptor = yaml.safe_load(pkg)
+            pkg_descriptor = ruamel.yaml.safe_load(pkg)
         rmtree(folder, ignore_errors=False)
         if ((pkgtype == 'vnf' and (pkg_descriptor.get('vnfd') or pkg_descriptor.get('vnfd:vnfd_catalog'))) or
                 (pkgtype == 'ns' and (pkg_descriptor.get('nsd') or pkg_descriptor.get('nsd:nsd_catalog')))):
@@ -224,8 +224,8 @@ class OSMRepo(Repo):
         fields['description'] = aux_dict.get('description')
         fields['vendor'] = aux_dict.get('vendor')
         fields['version'] = aux_dict.get('version', '1.0')
-        fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'), \
-                          fields.get('version'))
+        fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'),
+                                                       fields.get('version'))
         return fields
 
     def zip_extraction(self, file_name):
@@ -306,7 +306,7 @@ class OSMRepo(Repo):
                 self.indexation(destination, path, package_type, fields)
 
         except Exception as e:
-            self._logger.debug("Error registering artifact in Repository: {}".format(e))
+            self._logger.exception("Error registering artifact in Repository: {}".format(e))
 
         finally:
             if source == 'directory' and compresed:
@@ -331,28 +331,28 @@ class OSMRepo(Repo):
                 mkdir(final_path)
                 copyfile(path,
                          final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz')
-                yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'),
-                          Dumper=ruamel.yaml.RoundTripDumper)
-                index = yaml.load(open(destination + '/index.yaml'))
+                ruamel.yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'),
+                                 Dumper=ruamel.yaml.RoundTripDumper)
+                index = ruamel.yaml.load(open(destination + '/index.yaml'), Loader=ruamel.yaml.Loader)
 
                 index['{}_packages'.format(package_type)][fields.get('id')][fields.get('version')] = data_ind
                 if versioning.parse(index['{}_packages'.format(package_type)][fields.get('id')][
                                     'latest']) < versioning.parse(fields.get('version')):
                     index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get(
                         'version')
-                yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper)
+                ruamel.yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper)
                 self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path)))
         else:
             mkdir(destination + '/{}/'.format(package_type) + fields.get('id'))
             mkdir(final_path)
             copyfile(path,
                      final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz')
-            yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
-            index = yaml.load(open(destination + '/index.yaml'))
+            ruamel.yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
+            index = ruamel.yaml.load(open(destination + '/index.yaml'), Loader=ruamel.yaml.Loader)
 
             index['{}_packages'.format(package_type)][fields.get('id')] = {fields.get('version'): data_ind}
             index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get('version')
-            yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
+            ruamel.yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
             self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path)))
 
     def current_datatime(self):
@@ -376,4 +376,4 @@ class OSMRepo(Repo):
             index_data = {'apiVersion': 'v1', 'generated': self.current_datatime(), 'vnf_packages': {},
                           'ns_packages': {}}
             with open(join(destination, 'index.yaml'), 'w') as outfile:
-                yaml.dump(index_data, outfile, default_flow_style=False)
+                ruamel.yaml.dump(index_data, outfile, default_flow_style=False)