osmclient/sol005/osmrepo.py

   1 #
   2 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   3 #    not use this file except in compliance with the License. You may obtain
   4 #    a copy of the License at
   5 #
   6 #         http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 #    Unless required by applicable law or agreed to in writing, software
   9 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11 #    License for the specific language governing permissions and limitations
  12 #    under the License.
  13 #
  14
  15 """
  16 OSM Repo API handling
  17 """
  18 from osmclient.common.exceptions import ClientException
  19 from osmclient.sol005.repo import Repo
  20 from osmclient.common.package_tool import PackageTool
  21 import requests
  22 import logging
  23 import tempfile
  24 from shutil import copyfile, rmtree
  25 import yaml
  26 import tarfile
  27 import glob
  28 from packaging import version as versioning
  29 import time
  30 from os import listdir, mkdir, getcwd, remove
  31 from os.path import isfile, isdir, join, abspath
  32 import hashlib
  33 from osm_im.validation import Validation as validation_im
  34 import ruamel.yaml
  35
  36
  37 class OSMRepo(Repo):
  38     def __init__(self, http=None, client=None):
  39         self._http = http
  40         self._client = client
  41         self._apiName = '/admin'
  42         self._apiVersion = '/v1'
  43         self._apiResource = '/osmrepos'
  44         self._logger = logging.getLogger('osmclient')
  45         self._apiBase = '{}{}{}'.format(self._apiName,
  46                                         self._apiVersion, self._apiResource)
  47
  48     def pkg_list(self, pkgtype, filter=None, repo=None):
  49         """
  50             Returns a repo based on name or id
  51         """
  52         self._logger.debug("")
  53         self._client.get_token()
  54         # Get OSM registered repository list
  55         repositories = self.list()
  56         if repo:
  57             repositories = [r for r in repositories if r["name"] == repo]
  58         if not repositories:
  59             raise ClientException('Not repository found')
  60
  61         vnf_repos = []
  62         for repository in repositories:
  63             try:
  64                 r = requests.get('{}/index.yaml'.format(repository.get('url')))
  65
  66                 if r.status_code == 200:
  67                     repo_list = yaml.safe_load(r.text)
  68                     vnf_packages = repo_list.get('{}_packages'.format(pkgtype))
  69                     for repo in vnf_packages:
  70                         versions = vnf_packages.get(repo)
  71                         latest = versions.get('latest')
  72                         del versions['latest']
  73                         for version in versions:
  74                             latest_version = False
  75                             if version == latest:
  76                                 latest_version = True
  77                             vnf_repos.append({'vendor': versions[version].get("vendor"),
  78                                               'name': versions[version].get("name"),
  79                                               'version': version,
  80                                               'description': versions[version].get("description"),
  81                                               'location': versions[version].get("path"),
  82                                               'repository': repository.get('name'),
  83                                               'repourl': repository.get('url'),
  84                                               'latest': latest_version
  85                                               })
  86                 else:
  87                     raise Exception('repository in url {} unreachable'.format(repository.get('url')))
  88             except Exception as e:
  89                 logging.error("Error cannot read from repository {} '{}': {}".format(repository['name'], repository['url'], e))
  90                 continue
  91
  92         vnf_repos_filtered = []
  93         if filter:
  94             for vnf_repo in vnf_repos:
  95                 for k, v in vnf_repo.items():
  96                     if v:
  97                         kf, vf = filter.split('=')
  98                         if k == kf and vf in v:
  99                             vnf_repos_filtered.append(vnf_repo)
 100                             break
 101             vnf_repos = vnf_repos_filtered
 102         return vnf_repos
 103
 104     def get_pkg(self, pkgtype, name, repo, filter, version):
 105         """
 106             Returns the filename of the PKG downloaded to disk
 107         """
 108         self._logger.debug("")
 109         self._client.get_token()
 110         f = None
 111         f_name = None
 112         # Get OSM registered repository list
 113         pkgs = self.pkg_list(pkgtype, filter, repo)
 114         for pkg in pkgs:
 115             if pkg.get('repository') == repo and pkg.get('name') == name:
 116                 if 'latest' in version:
 117                     if not pkg.get('latest'):
 118                         continue
 119                     else:
 120                         version = pkg.get('version')
 121                 if pkg.get('version') == version:
 122                     r = requests.get('{}{}'.format(pkg.get('repourl'), pkg.get('location')), stream=True)
 123                     if r.status_code != 200:
 124                         raise ClientException("Package not found")
 125
 126                     with tempfile.NamedTemporaryFile(delete=False) as f:
 127                         f.write(r.raw.read())
 128                         f_name = f.name
 129                     if not f_name:
 130                         raise ClientException("{} {} not found at repo {}".format(pkgtype,name, repo))
 131         return f_name
 132
 133     def pkg_get(self, pkgtype, name, repo, version, filter):
 134
 135         pkg_name = self.get_pkg(pkgtype, name, repo, filter, version)
 136         if not pkg_name:
 137             raise ClientException('Package not found')
 138         folder, descriptor = self.zip_extraction(pkg_name)
 139         with open(descriptor) as pkg:
 140             pkg_descriptor = yaml.safe_load(pkg)
 141         rmtree(folder, ignore_errors=False)
 142         if ((pkgtype == 'vnf' and (pkg_descriptor.get('vnfd') or pkg_descriptor.get('vnfd:vnfd_catalog'))) or
 143                 (pkgtype == 'ns' and (pkg_descriptor.get('nsd') or pkg_descriptor.get('nsd:nsd_catalog')))):
 144             raise ClientException('Wrong Package type')
 145         return pkg_descriptor
 146
 147     def repo_index(self, origin=".", destination='.'):
 148         """
 149             Repo Index main function
 150             :param origin: origin directory for getting all the artifacts
 151             :param destination: destination folder for create and index the valid artifacts
 152         """
 153         if destination == '.':
 154             if origin == destination:
 155                 destination = 'repository'
 156
 157         destination = abspath(destination)
 158         origin = abspath(origin)
 159
 160         if origin[0] != '/':
 161             origin = join(getcwd(), origin)
 162         if destination[0] != '/':
 163             destination = join(getcwd(), destination)
 164
 165         self.init_directory(destination)
 166         artifacts = [f for f in listdir(origin) if isfile(join(origin, f))]
 167         directories = [f for f in listdir(origin) if isdir(join(origin, f))]
 168         for artifact in artifacts:
 169             self.register_artifact_in_repository(join(origin, artifact), destination, source='file')
 170         for artifact in directories:
 171             self.register_artifact_in_repository(join(origin, artifact), destination, source='directory')
 172         print("\nFinal Results: ")
 173         print("VNF Packages Indexed: " + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml"))))
 174         print("NS Packages Indexed: " + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml"))))
 175
 176     def md5(self, fname):
 177         """
 178             Checksum generator
 179             :param fname: file path
 180             :return: checksum string
 181         """
 182         hash_md5 = hashlib.md5()
 183         with open(fname, "rb") as f:
 184             for chunk in iter(lambda: f.read(4096), b""):
 185                 hash_md5.update(chunk)
 186         return hash_md5.hexdigest()
 187
 188     def fields_building(self, descriptor_json, file, package_type):
 189         """
 190             From an artifact descriptor, obtain the fields required for indexing
 191             :param descriptor_json: artifact description
 192             :param file: artifact package
 193             :param package_type: type of artifact (vnf or ns)
 194             :return: fields
 195         """
 196         fields = {}
 197         base_path = '/{}/'.format(package_type)
 198         aux_dict = {}
 199         if package_type == "vnf":
 200             if descriptor_json.get('vnfd-catalog', False):
 201                 aux_dict = descriptor_json.get('vnfd-catalog', {}).get('vnfd', [{}])[0]
 202             else:
 203                 aux_dict = descriptor_json.get('vnfd:vnfd-catalog', {}).get('vnfd', [{}])[0]
 204
 205             images = []
 206             for vdu in aux_dict.get('vdu', ()):
 207                 images.append(vdu.get('image'))
 208             fields['images'] = images
 209         if package_type == "ns":
 210             if descriptor_json.get('nsd-catalog', False):
 211                 aux_dict = descriptor_json.get('nsd-catalog', {}).get('nsd', [{}])[0]
 212             else:
 213                 aux_dict = descriptor_json.get('nsd:nsd-catalog', {}).get('nsd', [{}])[0]
 214
 215             vnfs = []
 216
 217             for vnf in aux_dict.get('constituent-vnfd', ()):
 218                 vnfs.append(vnf.get('vnfd-id-ref'))
 219             self._logger.debug('Used VNFS in the NSD: ' + str(vnfs))
 220             fields['vnfd-id-ref'] = vnfs
 221
 222         fields['name'] = aux_dict.get('name')
 223         fields['id'] = aux_dict.get('id')
 224         fields['description'] = aux_dict.get('description')
 225         fields['vendor'] = aux_dict.get('vendor')
 226         fields['version'] = aux_dict.get('version', '1.0')
 227         fields['path'] = "{}{}/{}/{}-{}.tar.gz".format(base_path, fields['id'], fields['version'], fields.get('id'), \
 228                           fields.get('version'))
 229         return fields
 230
 231     def zip_extraction(self, file_name):
 232         """
 233             Validation of artifact.
 234             :param file: file path
 235             :return: status details, status, fields, package_type
 236         """
 237         self._logger.debug("Decompressing package file")
 238         temp_file = '/tmp/{}'.format(file_name.split('/')[-1])
 239         if file_name != temp_file:
 240             copyfile(file_name, temp_file)
 241         with tarfile.open(temp_file, "r:gz") as tar:
 242             folder = tar.getnames()[0].split('/')[0]
 243             tar.extractall()
 244
 245         remove(temp_file)
 246         descriptor_file = glob.glob('{}/*.y*ml'.format(folder))[0]
 247         return folder, descriptor_file
 248
 249     def validate_artifact(self, path, source):
 250         """
 251             Validation of artifact.
 252             :param path: file path
 253             :return: status details, status, fields, package_type
 254         """
 255         package_type = ''
 256         folder = ''
 257         try:
 258             if source == 'directory':
 259                 descriptor_file = glob.glob('{}/*.y*ml'.format(path))[0]
 260             else:
 261                 folder, descriptor_file = self.zip_extraction(path)
 262
 263             self._logger.debug("Opening descriptor file: {}".format(descriptor_file))
 264
 265             with open(descriptor_file, 'r') as f:
 266                 descriptor_data = f.read()
 267             validation = validation_im()
 268             desc_type, descriptor_data = validation.yaml_validation(descriptor_data)
 269             validation_im.pyangbind_validation(self, desc_type, descriptor_data)
 270             if 'vnf' in list(descriptor_data.keys())[0]:
 271                 package_type = 'vnf'
 272             else:
 273                 # raise ClientException("Not VNF package")
 274                 package_type = 'ns'
 275
 276             self._logger.debug("Descriptor: {}".format(descriptor_data))
 277             fields = self.fields_building(descriptor_data, path, package_type)
 278             self._logger.debug("Descriptor sucessfully validated")
 279             return {"detail": "{}D successfully validated".format(package_type.upper()),
 280                     "code": "OK"}, True, fields, package_type
 281         except Exception as e:
 282             # Delete the folder we just created
 283             return {"detail": str(e)}, False, {}, package_type
 284         finally:
 285             if folder:
 286                 rmtree(folder, ignore_errors=True)
 287
 288     def register_artifact_in_repository(self, path, destination, source):
 289         """
 290             Registration of one artifact in a repository
 291             file: VNF or NS
 292             destination: path for index creation
 293         """
 294         pt = PackageTool()
 295         compresed = False
 296         try:
 297             fields = {}
 298             _, valid, fields, package_type = self.validate_artifact(path, source)
 299             if not valid:
 300                 raise Exception('{} {} Not well configured.'.format(package_type.upper(), str(path)))
 301             else:
 302                 if source == 'directory':
 303                     path = pt.build(path)
 304                     compresed = True
 305                 fields['checksum'] = self.md5(path)
 306                 self.indexation(destination, path, package_type, fields)
 307
 308         except Exception as e:
 309             self._logger.debug("Error registering artifact in Repository: {}".format(e))
 310
 311         finally:
 312             if source == 'directory' and compresed:
 313                 remove(path)
 314
 315     def indexation(self, destination, path, package_type, fields):
 316         """
 317             Process for index packages
 318             :param destination: index repository path
 319             :param path: path of the package
 320             :param package_type: package type (vnf, ns)
 321             :param fields: dict with the required values
 322         """
 323         data_ind = {'name': fields.get('name'), 'description': fields.get('description'),
 324                     'vendor': fields.get('vendor'), 'path': fields.get('path')}
 325
 326         final_path = join(destination, package_type, fields.get('id'), fields.get('version'))
 327         if isdir(join(destination, package_type, fields.get('id'))):
 328             if isdir(final_path):
 329                 self._logger.warning('{} {} already exists'.format(package_type.upper(), str(path)))
 330             else:
 331                 mkdir(final_path)
 332                 copyfile(path,
 333                          final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz')
 334                 yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'),
 335                           Dumper=ruamel.yaml.RoundTripDumper)
 336                 index = yaml.load(open(destination + '/index.yaml'))
 337
 338                 index['{}_packages'.format(package_type)][fields.get('id')][fields.get('version')] = data_ind
 339                 if versioning.parse(index['{}_packages'.format(package_type)][fields.get('id')][
 340                                     'latest']) < versioning.parse(fields.get('version')):
 341                     index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get(
 342                         'version')
 343                 yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper)
 344                 self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path)))
 345         else:
 346             mkdir(destination + '/{}/'.format(package_type) + fields.get('id'))
 347             mkdir(final_path)
 348             copyfile(path,
 349                      final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz')
 350             yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
 351             index = yaml.load(open(destination + '/index.yaml'))
 352
 353             index['{}_packages'.format(package_type)][fields.get('id')] = {fields.get('version'): data_ind}
 354             index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get('version')
 355             yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper)
 356             self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path)))
 357
 358     def current_datatime(self):
 359         """
 360             Datetime Generator
 361             :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z"
 362         """
 363         return time.strftime('%Y-%m-%dT%H:%M:%S.%sZ')
 364
 365     def init_directory(self, destination):
 366         """
 367             Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns
 368             :param destination:
 369             :return:
 370         """
 371         if not isdir(destination):
 372             mkdir(destination)
 373         if not isfile(join(destination, 'index.yaml')):
 374             mkdir(join(destination, 'vnf'))
 375             mkdir(join(destination, 'ns'))
 376             index_data = {'apiVersion': 'v1', 'generated': self.current_datatime(), 'vnf_packages': {},
 377                           'ns_packages': {}}
 378             with open(join(destination, 'index.yaml'), 'w') as outfile:
 379                 yaml.dump(index_data, outfile, default_flow_style=False)