From: gomezl Date: Wed, 6 May 2020 07:44:54 +0000 (+0200) Subject: Feature 8178 VNF Repositories X-Git-Tag: v7.1.0rc5^2^2 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fosmclient.git;a=commitdiff_plain;h=bcb78331f0e8a321585fc7b84e9b7857a90ecb6c Feature 8178 VNF Repositories Change-Id: I040da8dd9d5696f9029cf7ecf82aa1eff26bb22a Signed-off-by: gomezl --- diff --git a/.gitignore b/.gitignore index afae2d8..b4c9ff5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,17 @@ +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. *.pyc .cache deb_dist/ @@ -17,3 +31,4 @@ stage/ .tox/ snap/.snapcraft/ .vscode +build/ diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index 3da96d5..c2e1995 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -289,7 +289,7 @@ class PackageTool(object): if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)): print('Building charm {}/charms/layers/{}'.format(package_folder, charmName)) self.charm_build(package_folder, charmName) - print('Charm built {}'.format(charmName)) + print('Charm built: {}'.format(charmName)) else: if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)): raise ClientException ('The charm: {} referenced in the descriptor file ' diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py index 46ceb91..402a09b 100755 --- a/osmclient/scripts/osm.py +++ b/osmclient/scripts/osm.py @@ -20,7 +20,7 @@ OSM shell/cli import click from osmclient import client -from osmclient.common.exceptions import ClientException +from osmclient.common.exceptions import ClientException, NotFound from prettytable import PrettyTable import yaml import json @@ -466,6 +466,26 @@ def nsd_list2(ctx, filter, long): nsd_list(ctx, filter, long) +def pkg_repo_list(ctx, pkgtype, filter, repo, long): + resp = ctx.obj.osmrepo.pkg_list(pkgtype, filter, repo) + if long: + table = PrettyTable(['nfpkg name', 'vendor', 'version', 'latest', 'description', 'repository']) + else: + table = PrettyTable(['nfpkg name', 'repository']) + for vnfd in resp: + name = vnfd.get('name', '-') + repository = vnfd.get('repository') + if long: + vendor = vnfd.get('vendor') + version = vnfd.get('version') + description = vnfd.get('description') + latest = vnfd.get('latest') + table.add_row([name, vendor, version, latest, description, repository]) + else: + table.add_row([name, repository]) + table.align = 'l' + print(table) + def vnfd_list(ctx, nf_type, filter, long): logger.debug("") if nf_type: @@ -493,7 +513,7 @@ def vnfd_list(ctx, nf_type, filter, long): fullclassname = ctx.obj.__module__ + "." + ctx.obj.__class__.__name__ if fullclassname == 'osmclient.sol005.client.Client': if long: - table = PrettyTable(['nfpkg name', 'id', 'onboarding state', 'operational state', + table = PrettyTable(['nfpkg name', 'id', 'vendor', 'version', 'onboarding state', 'operational state', 'usage state', 'date', 'last update']) else: table = PrettyTable(['nfpkg name', 'id']) @@ -502,10 +522,12 @@ def vnfd_list(ctx, nf_type, filter, long): if long: onb_state = vnfd['_admin'].get('onboardingState','-') op_state = vnfd['_admin'].get('operationalState','-') + vendor = vnfd.get('vendor') + version = vnfd.get('version') usage_state = vnfd['_admin'].get('usageState','-') date = datetime.fromtimestamp(vnfd['_admin']['created']).strftime("%Y-%m-%dT%H:%M:%S") last_update = datetime.fromtimestamp(vnfd['_admin']['modified']).strftime("%Y-%m-%dT%H:%M:%S") - table.add_row([name, vnfd['_id'], onb_state, op_state, usage_state, date, last_update]) + table.add_row([name, vnfd['_id'], vendor, version, onb_state, op_state, usage_state, date, last_update]) else: table.add_row([name, vnfd['_id']]) else: @@ -539,6 +561,17 @@ def vnfd_list2(ctx, nf_type, filter, long): logger.debug("") vnfd_list(ctx, nf_type, filter, long) +@cli_osm.command(name='vnfpkg-repo-list', short_help='list all xNF from OSM repositories') +@click.option('--filter', default=None, + help='restricts the list to the NFpkg matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def vnfd_list3(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'vnf' + pkg_repo_list(ctx, pkgtype, filter, repo, long) @cli_osm.command(name='nfpkg-list', short_help='list all xNF packages (VNF, HNF, PNF)') @click.option('--nf_type', help='type of NF (vnf, pnf, hnf)') @@ -556,6 +589,17 @@ def nfpkg_list(ctx, nf_type, filter, long): # print(str(e)) # exit(1) +@cli_osm.command(name='nfpkg-repo-list', short_help='list all xNF from OSM repositories') +@click.option('--filter', default=None, + help='restricts the list to the NFpkg matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def vnfd_list4(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'vnf' + pkg_repo_list(ctx, pkgtype, filter, repo, long) def vnf_list(ctx, ns, filter, long): # try: @@ -619,6 +663,29 @@ def vnf_list1(ctx, ns, filter, long): logger.debug("") vnf_list(ctx, ns, filter, long) +@cli_osm.command(name='nsd-repo-list', short_help='list all NS from OSM repositories') +@click.option('--filter', default=None, + help='restricts the list to the NS matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def nsd_list3(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'ns' + pkg_repo_list(ctx, pkgtype, filter, repo, long) + +@cli_osm.command(name='nspkg-repo-list', short_help='list all NS from OSM repositories') +@click.option('--filter', default=None, + help='restricts the list to the NS matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def nspkg_list(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'ns' + pkg_repo_list(ctx, pkgtype, filter, repo, long) @cli_osm.command(name='nf-list', short_help='list all NF instances') @click.option('--ns', default=None, help='NS instance id or name to restrict the NF list') @@ -675,7 +742,7 @@ def nf_list(ctx, ns, filter, long): --filter vnfd-ref=,vdur.ip-address= """ logger.debug("") - vnf_list(ctx, ns, filter) + vnf_list(ctx, ns, filter, long) @cli_osm.command(name='ns-op-list', short_help='shows the history of operations over a NS instance') @@ -983,6 +1050,28 @@ def vnfd_show(ctx, name, literal): print(table) +def pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal): + logger.debug("") + # try: + resp = ctx.obj.osmrepo.pkg_get(pkgtype, name, repo, version, filter) + + if literal: + print(yaml.safe_dump(resp)) + return + pkgtype += 'd' + catalog = pkgtype + '-catalog' + full_catalog = pkgtype + ':' + catalog + if resp.get(catalog): + resp = resp.pop(catalog)[pkgtype][0] + elif resp.get(full_catalog): + resp = resp.pop(full_catalog)[pkgtype][0] + + table = PrettyTable(['field', 'value']) + for k, v in list(resp.items()): + table.add_row([k, wrap_text(text=json.dumps(v, indent=2), width=100)]) + table.align = 'l' + print(table) + @cli_osm.command(name='vnfd-show', short_help='shows the content of a VNFD') @click.option('--literal', is_flag=True, help='print literally, no pretty table') @@ -1010,6 +1099,69 @@ def vnfd_show2(ctx, name, literal): logger.debug("") vnfd_show(ctx, name, literal) +@cli_osm.command(name='vnfpkg-repo-show', short_help='shows the content of a VNFD') +@click.option('--literal', is_flag=True, + help='print literally, no pretty table') +@click.option('--repo', + required=True, + help='Repository name') +@click.argument('name') +@click.option('--filter', + help='filter by fields') +@click.option('--version', + default='latest', + help='package version') +@click.pass_context +def vnfd_show3(ctx, name, repo, version, literal=None, filter=None): + """shows the content of a VNFD in a repository + + NAME: name or ID of the VNFD/VNFpkg + """ + pkgtype = 'vnf' + pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal) + + +@cli_osm.command(name='nsd-repo-show', short_help='shows the content of a NSD') +@click.option('--literal', is_flag=True, + help='print literally, no pretty table') +@click.option('--repo', + required=True, + help='Repository name') +@click.argument('name') +@click.option('--filter', + help='filter by fields') +@click.option('--version', + default='latest', + help='package version') +@click.pass_context +def nsd_repo_show(ctx, name, repo, version, literal=None, filter=None): + """shows the content of a VNFD in a repository + + NAME: name or ID of the VNFD/VNFpkg + """ + pkgtype = 'ns' + pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal) + +@cli_osm.command(name='nspkg-repo-show', short_help='shows the content of a NSD') +@click.option('--literal', is_flag=True, + help='print literally, no pretty table') +@click.option('--repo', + required=True, + help='Repository name') +@click.argument('name') +@click.option('--filter', + help='filter by fields') +@click.option('--version', + default='latest', + help='package version') +@click.pass_context +def nsd_repo_show2(ctx, name, repo, version, literal=None, filter=None): + """shows the content of a VNFD in a repository + + NAME: name or ID of the VNFD/VNFpkg + """ + pkgtype = 'ns' + pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal) @cli_osm.command(name='nfpkg-show', short_help='shows the content of a NF Descriptor') @click.option('--literal', is_flag=True, @@ -1025,6 +1177,28 @@ def nfpkg_show(ctx, name, literal): vnfd_show(ctx, name, literal) +@cli_osm.command(name='nfpkg-repo-show', short_help='shows the content of a VNFD') +@click.option('--literal', is_flag=True, + help='print literally, no pretty table') +@click.option('--repo', + required=True, + help='Repository name') +@click.argument('name') +@click.option('--filter', + help='filter by fields') +@click.option('--version', + default='latest', + help='package version') +@click.pass_context +def vnfd_show4(ctx, name, repo, version, literal=None, filter=None): + """shows the content of a VNFD in a repository + + NAME: name or ID of the VNFD/VNFpkg + """ + pkgtype = 'vnf' + pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal) + + @cli_osm.command(name='ns-show', short_help='shows the info of a NS instance') @click.argument('name') @click.option('--literal', is_flag=True, @@ -1407,10 +1581,12 @@ def pdu_show(ctx, name, literal, filter): # CREATE operations #################### -def nsd_create(ctx, filename, overwrite, skip_charm_build): +def nsd_create(ctx, filename, overwrite, skip_charm_build, repo, vendor, version): logger.debug("") # try: check_client_version(ctx.obj, ctx.command.name) + if repo: + filename = ctx.obj.osmrepo.get_pkg('ns', filename, repo, vendor, version) ctx.obj.nsd.create(filename, overwrite=overwrite, skip_charm_build=skip_charm_build) # except ClientException as e: # print(str(e)) @@ -1426,8 +1602,14 @@ def nsd_create(ctx, filename, overwrite, skip_charm_build): '"key1.key2...=value[;key3...=value;...]"') @click.option('--skip-charm-build', default=False, is_flag=True, help='The charm will not be compiled, it is assumed to already exist') +@click.option('--repo', default=None, + help='[repository]: Repository name') +@click.option('--vendor', default=None, + help='[repository]: filter by vendor]') +@click.option('--version', default='latest', + help='[repository]: filter by version. Default: latest') @click.pass_context -def nsd_create1(ctx, filename, overwrite, skip_charm_build): +def nsd_create1(ctx, filename, overwrite, skip_charm_build, repo, vendor, version): """onboards a new NSpkg (alias of nspkg-create) (TO BE DEPRECATED) \b @@ -1436,7 +1618,8 @@ def nsd_create1(ctx, filename, overwrite, skip_charm_build): If FILENAME is an NF Package folder, it is built and then onboarded. """ logger.debug("") - nsd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) + nsd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build, repo=repo, vendor=vendor, + version=version) @cli_osm.command(name='nspkg-create', short_help='creates a new NSD/NSpkg') @@ -1448,23 +1631,32 @@ def nsd_create1(ctx, filename, overwrite, skip_charm_build): '"key1.key2...=value[;key3...=value;...]"') @click.option('--skip-charm-build', default=False, is_flag=True, help='The charm will not be compiled, it is assumed to already exist') +@click.option('--repo', default=None, + help='[repository]: Repository name') +@click.option('--vendor', default=None, + help='[repository]: filter by vendor]') +@click.option('--version', default='latest', + help='[repository]: filter by version. Default: latest') @click.pass_context -def nsd_create2(ctx, filename, overwrite, skip_charm_build): +def nsd_pkg_create(ctx, filename, overwrite, skip_charm_build, repo, vendor, version): """onboards a new NSpkg - \b FILENAME: NF Package tar.gz file, NF Descriptor YAML file or NF Package folder If FILENAME is a file (NF Package tar.gz or NF Descriptor YAML), it is onboarded. If FILENAME is an NF Package folder, it is built and then onboarded. """ logger.debug("") - nsd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) + nsd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build, repo=repo, vendor=vendor, + version=version) -def vnfd_create(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt): +def vnfd_create(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt, + repo, vendor, version): logger.debug("") # try: check_client_version(ctx.obj, ctx.command.name) + if repo: + filename = ctx.obj.osmrepo.get_pkg('vnf', filename, repo, vendor, version) ctx.obj.vnfd.create(filename, overwrite=overwrite, skip_charm_build=skip_charm_build, override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt) @@ -1488,10 +1680,16 @@ def vnfd_create(ctx, filename, overwrite, skip_charm_build, override_epa, overri help='removes all guest-epa parameters from all VDU') @click.option('--override-paravirt', required=False, default=False, is_flag=True, help='overrides all VDU interfaces to PARAVIRT') +@click.option('--repo', default=None, + help='[repository]: Repository name') +@click.option('--vendor', default=None, + help='[repository]: filter by vendor]') +@click.option('--version', default='latest', + help='[repository]: filter by version. Default: latest') @click.pass_context -def vnfd_create1(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt): - """onboards a new NFpkg (alias of nfpkg-create) (TO BE DEPRECATED) - +def vnfd_create1(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt, + repo,vendor, version): + """creates a new VNFD/VNFpkg \b FILENAME: NF Package tar.gz file, NF Descriptor YAML file or NF Package folder If FILENAME is a file (NF Package tar.gz or NF Descriptor YAML), it is onboarded. @@ -1499,7 +1697,8 @@ def vnfd_create1(ctx, filename, overwrite, skip_charm_build, override_epa, overr """ logger.debug("") vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build, - override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt) + override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt, + repo=repo, vendor=vendor, version=version) @cli_osm.command(name='vnfpkg-create', short_help='creates a new VNFD/VNFpkg') @@ -1517,10 +1716,16 @@ def vnfd_create1(ctx, filename, overwrite, skip_charm_build, override_epa, overr help='removes all guest-epa parameters from all VDU') @click.option('--override-paravirt', required=False, default=False, is_flag=True, help='overrides all VDU interfaces to PARAVIRT') +@click.option('--repo', default=None, + help='[repository]: Repository name') +@click.option('--vendor', default=None, + help='[repository]: filter by vendor]') +@click.option('--version', default='latest', + help='[repository]: filter by version. Default: latest') @click.pass_context -def vnfd_create2(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt): - """onboards a new NFpkg (alias of nfpkg-create) - +def vnfd_create2(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt, + repo, vendor, version): + """creates a new VNFD/VNFpkg \b FILENAME: NF Package tar.gz file, NF Descriptor YAML file or NF Package folder If FILENAME is a file (NF Package tar.gz or NF Descriptor YAML), it is onboarded. @@ -1528,8 +1733,8 @@ def vnfd_create2(ctx, filename, overwrite, skip_charm_build, override_epa, overr """ logger.debug("") vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build, - override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt) - + override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt, + repo=repo, vendor=vendor, version=version) @cli_osm.command(name='nfpkg-create', short_help='creates a new NFpkg') @click.argument('filename') @@ -1546,9 +1751,16 @@ def vnfd_create2(ctx, filename, overwrite, skip_charm_build, override_epa, overr help='removes all guest-epa parameters from all VDU') @click.option('--override-paravirt', required=False, default=False, is_flag=True, help='overrides all VDU interfaces to PARAVIRT') +@click.option('--repo', default=None, + help='[repository]: Repository name') +@click.option('--vendor', default=None, + help='[repository]: filter by vendor]') +@click.option('--version', default='latest', + help='[repository]: filter by version. Default: latest') @click.pass_context -def nfpkg_create(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt): - """onboards a new NFpkg (alias of nfpkg-create) +def nfpkg_create(ctx, filename, overwrite, skip_charm_build, override_epa, override_nonepa, override_paravirt, + repo, vendor, version): + """creates a new NFpkg \b FILENAME: NF Package tar.gz file, NF Descriptor YAML file or NF Package folder @@ -1557,7 +1769,8 @@ def nfpkg_create(ctx, filename, overwrite, skip_charm_build, override_epa, overr """ logger.debug("") vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build, - override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt) + override_epa=override_epa, override_nonepa=override_nonepa, override_paravirt=override_paravirt, + repo=repo, vendor=vendor, version=version) @cli_osm.command(name='ns-create', short_help='creates a new Network Service instance') @@ -2926,35 +3139,36 @@ def k8scluster_show(ctx, name, literal): @click.argument('name') @click.argument('uri') @click.option('--type', - type=click.Choice(['helm-chart', 'juju-bundle']), - prompt=True, - help='type of repo (helm-chart for Helm Charts, juju-bundle for Juju Bundles)') + type=click.Choice(['helm-chart', 'juju-bundle', 'osm']), + default='osm', + help='type of repo (helm-chart for Helm Charts, juju-bundle for Juju Bundles, osm for OSM Repositories)') @click.option('--description', default=None, help='human readable description') +@click.option('--user', + default=None, + help='OSM repository: The username of the OSM repository') +@click.option('--password', + default=None, + help='OSM repository: The password of the OSM repository') #@click.option('--wait', # is_flag=True, # help='do not return the control immediately, but keep it until the operation is completed, or timeout') @click.pass_context -def repo_add(ctx, - name, - uri, - type, - description): +def repo_add(ctx, **kwargs): """adds a repo to OSM NAME: name of the repo URI: URI of the repo """ # try: - check_client_version(ctx.obj, ctx.command.name) - repo = {} - repo['name'] = name - repo['url'] = uri - repo['type'] = type - if description: - repo['description'] = description - ctx.obj.repo.create(name, repo) + kwargs = {k: v for k, v in kwargs.items() if v is not None} + repo = kwargs + repo["url"] = repo.pop("uri") + if repo["type"] in ['helm-chart', 'juju-bundle']: + ctx.obj.repo.create(repo['name'], repo) + else: + ctx.obj.osmrepo.create(repo['name'], repo) # except ClientException as e: # print(str(e)) # exit(1) @@ -2964,8 +3178,6 @@ def repo_add(ctx, @click.argument('name') @click.option('--newname', help='New name for the repo') @click.option('--uri', help='URI of the repo') -@click.option('--type', type=click.Choice(['helm-chart', 'juju-bundle']), - help='type of repo (helm-chart for Helm Charts, juju-bundle for Juju Bundles)') @click.option('--description', help='human readable description') #@click.option('--wait', # is_flag=True, @@ -2975,7 +3187,6 @@ def repo_update(ctx, name, newname, uri, - type, description): """updates a repo in OSM @@ -2984,16 +3195,34 @@ def repo_update(ctx, # try: check_client_version(ctx.obj, ctx.command.name) repo = {} - if newname: repo['name'] = newname - if uri: repo['uri'] = uri - if type: repo['type'] = type + if newname: + repo['name'] = newname + if uri: + repo['uri'] = uri if description: repo['description'] = description - ctx.obj.repo.update(name, repo) + try: + ctx.obj.repo.update(name, repo) + except NotFound: + ctx.obj.osmrepo.update(name, repo) + # except ClientException as e: # print(str(e)) # exit(1) +@cli_osm.command(name='repo-index', short_help='Index a repository from a folder with artifacts') +@click.option('--origin', default='.', help='origin path where the artifacts are located') +@click.option('--destination', default='.', help='destination path where the index is deployed') +@click.pass_context +def repo_index(ctx, origin, destination): + """Index a repository + + NAME: name or ID of the repo to be deleted + """ + check_client_version(ctx.obj, ctx.command.name) + ctx.obj.osmrepo.repo_index(origin, destination) + + @cli_osm.command(name='repo-delete', short_help='deletes a repo') @click.argument('name') @click.option('--force', is_flag=True, help='forces the deletion from the DB (not recommended)') @@ -3006,9 +3235,11 @@ def repo_delete(ctx, name, force): NAME: name or ID of the repo to be deleted """ - # try: - check_client_version(ctx.obj, ctx.command.name) - ctx.obj.repo.delete(name, force=force) + logger.debug("") + try: + ctx.obj.repo.delete(name, force=force) + except NotFound: + ctx.obj.osmrepo.delete(name, force=force) # except ClientException as e: # print(str(e)) # exit(1) @@ -3023,8 +3254,10 @@ def repo_delete(ctx, name, force): def repo_list(ctx, filter, literal): """list all repos""" # try: + # K8s Repositories check_client_version(ctx.obj, ctx.command.name) resp = ctx.obj.repo.list(filter) + resp += ctx.obj.osmrepo.list(filter) if literal: print(yaml.safe_dump(resp)) return @@ -3034,6 +3267,7 @@ def repo_list(ctx, filter, literal): table.add_row([repo['name'], repo['_id'], repo['type'], repo['url'], trunc_text(repo.get('description',''),40)]) table.align = 'l' print(table) + # except ClientException as e: # print(str(e)) # exit(1) @@ -3049,14 +3283,20 @@ def repo_show(ctx, name, literal): NAME: name or ID of the repo """ - # try: - resp = ctx.obj.repo.get(name) + try: + resp = ctx.obj.repo.get(name) + except NotFound: + resp = ctx.obj.osmrepo.get(name) + if literal: - print(yaml.safe_dump(resp)) + if resp: + print(yaml.safe_dump(resp)) return table = PrettyTable(['key', 'attribute']) - for k, v in list(resp.items()): - table.add_row([k, json.dumps(v, indent=2)]) + if resp: + for k, v in list(resp.items()): + table.add_row([k, json.dumps(v, indent=2)]) + table.align = 'l' print(table) # except ClientException as e: diff --git a/osmclient/sol005/client.py b/osmclient/sol005/client.py index 5e28797..00c50cb 100644 --- a/osmclient/sol005/client.py +++ b/osmclient/sol005/client.py @@ -36,6 +36,7 @@ from osmclient.sol005 import role from osmclient.sol005 import pdud from osmclient.sol005 import k8scluster from osmclient.sol005 import repo +from osmclient.sol005 import osmrepo from osmclient.common import package_tool import json import logging @@ -93,6 +94,7 @@ class Client(object): self.pdu = pdud.Pdu(self._http_client, client=self) self.k8scluster = k8scluster.K8scluster(self._http_client, client=self) self.repo = repo.Repo(self._http_client, client=self) + self.osmrepo = osmrepo.OSMRepo(self._http_client, client=self) self.package_tool = package_tool.PackageTool(client=self) ''' self.vca = vca.Vca(http_client, client=self, **kwargs) diff --git a/osmclient/sol005/osmrepo.py b/osmclient/sol005/osmrepo.py new file mode 100644 index 0000000..4e37603 --- /dev/null +++ b/osmclient/sol005/osmrepo.py @@ -0,0 +1,390 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +""" +OSM Repo API handling +""" +from osmclient.common.exceptions import ClientException +from osmclient.sol005.repo import Repo +import requests +import logging +import tempfile +from shutil import copyfile, rmtree +import yaml +import tarfile +import glob +from packaging import version as versioning +import time +from os import listdir, mkdir, getcwd, remove +from os.path import isfile, isdir, join, abspath +import hashlib +from osm_im.validation import Validation as validation_im +import ruamel.yaml + + +class OSMRepo(Repo): + def __init__(self, http=None, client=None): + self._http = http + self._client = client + self._apiName = '/admin' + self._apiVersion = '/v1' + self._apiResource = '/osmrepos' + self._logger = logging.getLogger('osmclient') + self._apiBase = '{}{}{}'.format(self._apiName, + self._apiVersion, self._apiResource) + + def pkg_list(self, pkgtype, filter=None, repo=None): + """ + Returns a repo based on name or id + """ + self._logger.debug("") + self._client.get_token() + # Get OSM registered repository list + repositories = self.list() + if repo: + repositories = [r for r in repositories if r["name"] == repo] + if not repositories: + raise ClientException('Not repository found') + + vnf_repos = [] + for repository in repositories: + try: + r = requests.get('{}/index.yaml'.format(repository.get('url'))) + + if r.status_code == 200: + repo_list = yaml.safe_load(r.text) + vnf_packages = repo_list.get('{}_packages'.format(pkgtype)) + for repo in vnf_packages: + versions = vnf_packages.get(repo) + latest = versions.get('latest') + del versions['latest'] + for version in versions: + latest_version = False + if version == latest: + latest_version = True + vnf_repos.append({'vendor': versions[version].get("vendor"), + 'name': versions[version].get("name"), + 'version': version, + 'description': versions[version].get("description"), + 'location': versions[version].get("path"), + 'repository': repository.get('name'), + 'repourl': repository.get('url'), + 'latest': latest_version + }) + else: + raise Exception('repository in url {} unreachable'.format(repository.get('url'))) + except Exception as e: + logging.error("Error cannot read from repository {} '{}': {}".format(repository['name'], repository['url'], e)) + continue + + vnf_repos_filtered = [] + if filter: + for vnf_repo in vnf_repos: + for k, v in vnf_repo.items(): + if v: + kf, vf = filter.split('=') + if k == kf and vf in v: + vnf_repos_filtered.append(vnf_repo) + break + vnf_repos = vnf_repos_filtered + return vnf_repos + + def get_pkg(self, pkgtype, name, repo, filter, version): + """ + Returns the filename of the PKG downloaded to disk + """ + self._logger.debug("") + self._client.get_token() + f = None + f_name = None + # Get OSM registered repository list + pkgs = self.pkg_list(pkgtype, filter, repo) + for pkg in pkgs: + if pkg.get('repository') == repo and pkg.get('name') == name: + if 'latest' in version: + if not pkg.get('latest'): + continue + else: + version = pkg.get('version') + if pkg.get('version') == version: + r = requests.get('{}{}'.format(pkg.get('repourl'), pkg.get('location')), stream=True) + if r.status_code != 200: + raise ClientException("Package not found") + + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(r.raw.read()) + f_name = f.name + if not f_name: + raise ClientException("{} {} not found at repo {}".format(pkgtype,name, repo)) + return f_name + + def pkg_get(self, pkgtype, name, repo, version, filter): + + pkg_name = self.get_pkg(pkgtype, name, repo, filter, version) + if not pkg_name: + raise ClientException('Package not found') + folder, descriptor = self.zip_extraction(pkg_name) + with open(descriptor) as pkg: + pkg_descriptor = yaml.safe_load(pkg) + rmtree(folder, ignore_errors=False) + if ((pkgtype == 'vnf' and (pkg_descriptor.get('vnfd') or pkg_descriptor.get('vnfd:vnfd_catalog'))) or + (pkgtype == 'ns' and (pkg_descriptor.get('nsd') or pkg_descriptor.get('nsd:nsd_catalog')))): + raise ClientException('Wrong Package type') + return pkg_descriptor + + def repo_index(self, origin=".", destination='.'): + """ + Repo Index main function + :param origin: origin directory for getting all the artifacts + :param destination: destination folder for create and index the valid artifacts + """ + if destination == '.': + if origin == destination: + destination = 'repository' + + destination = abspath(destination) + origin = abspath(origin) + + if origin[0] != '/': + origin = join(getcwd(), origin) + if destination[0] != '/': + destination = join(getcwd(), destination) + + self.init_directory(destination) + artifacts = [f for f in listdir(origin) if isfile(join(origin, f))] + directories = [f for f in listdir(origin) if isdir(join(origin, f))] + for artifact in artifacts: + self.register_artifact_in_repository(join(origin, artifact), destination, source='file') + for artifact in directories: + self.register_artifact_in_repository(join(origin, artifact), destination, source='directory') + print("\nFinal Results: ") + print("VNF Packages Indexed: " + str(len(glob.glob(destination + "/vnf/*/*/metadata.yaml")))) + print("NS Packages Indexed: " + str(len(glob.glob(destination + "/ns/*/*/metadata.yaml")))) + + def md5(self, fname): + """ + Checksum generator + :param fname: file path + :return: checksum string + """ + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + def fields_building(self, descriptor_json, file, package_type): + """ + From an artifact descriptor, obtain the fields required for indexing + :param descriptor_json: artifact description + :param file: artifact package + :param package_type: type of artifact (vnf or ns) + :return: fields + """ + fields = {} + base_path = '/' + package_type + '/' + if package_type == "vnf": + if descriptor_json.get('vnfd-catalog', False): + aux_dict = descriptor_json.get('vnfd-catalog', {}).get('vnfd', [{}])[0] + else: + aux_dict = descriptor_json.get('vnfd:vnfd-catalog', {}).get('vnfd', [{}])[0] + + images = [] + for vdu in aux_dict.get('vdu', ()): + images.append(vdu.get('image')) + fields['images'] = images + if package_type == "ns": + if descriptor_json.get('nsd-catalog', False): + aux_dict = descriptor_json.get('nsd-catalog', {}).get('nsd', [{}])[0] + else: + aux_dict = descriptor_json.get('nsd:nsd-catalog', {}).get('nsd', [{}])[0] + + vnfs = [] + + for vnf in aux_dict.get('constituent-vnfd', ()): + vnfs.append(vnf.get('vnfd-id-ref')) + self._logger.debug('Used VNFS in the NSD: ' + str(vnfs)) + fields['vnfd-id-ref'] = vnfs + + fields['name'] = aux_dict.get('name') + fields['id'] = aux_dict.get('id') + fields['description'] = aux_dict.get('description') + fields['vendor'] = aux_dict.get('vendor') + fields['version'] = aux_dict.get('version', '1.0') + fields['path'] = base_path + fields['id'] + '/' + fields['version'] + '/' + fields.get('id') + "-" + \ + fields.get('version') + '.tar.gz' + return fields + + def zip_extraction(self, file): + """ + Validation of artifact. + :param file: file path + :return: status details, status, fields, package_type + """ + self._logger.debug("Decompressing package file") + temp_file = '/tmp/' + file.split('/')[-1] + if file != temp_file: + copyfile(file, temp_file) + with tarfile.open(temp_file, "r:gz") as tar: + folder = tar.getnames()[0].split('/')[0] + tar.extractall() + + remove(temp_file) + descriptor_file = glob.glob(folder + "/*.y*ml")[0] + return folder, descriptor_file + + def validate_artifact(self, path, source): + """ + Validation of artifact. + :param path: file path + :return: status details, status, fields, package_type + """ + try: + package_type = '' + folder = '' + if source == 'directory': + descriptor_file = glob.glob(path + "/*.y*ml")[0] + else: + folder, descriptor_file = self.zip_extraction(path) + + self._logger.debug("Opening descriptor file: {}".format(descriptor_file)) + + with open(descriptor_file, 'r') as f: + descriptor_data = f.read() + validation = validation_im() + desc_type, descriptor_data = validation.yaml_validation(descriptor_data) + validation_im.pyangbind_validation(self, desc_type, descriptor_data) + if 'vnf' in list(descriptor_data.keys())[0]: + package_type = 'vnf' + else: + # raise ClientException("Not VNF package") + package_type = 'ns' + + self._logger.debug("Descriptor: {}".format(descriptor_data)) + fields = self.fields_building(descriptor_data, path, package_type) + self._logger.debug("Descriptor sucessfully validated") + return {"detail": "{}D successfully validated".format(package_type.upper()), + "code": "OK"}, True, fields, package_type + except Exception as e: + # Delete the folder we just created + return {"detail": str(e)}, False, {}, package_type + finally: + if folder: + rmtree(folder, ignore_errors=True) + + def compress_artifact(self, path): + """ + Compress a directory for building an artifact + :param path: path of the directory + :return: file path + """ + if path[-1] == '/': + path = path[:-1] + file = path + '.tar.gz' + with tarfile.open(file, "w:gz") as tar: + tar.add(path) + + return file + + def register_artifact_in_repository(self, path, destination, source): + """ + Registration of one artifact in a repository + file: VNF or NS + destination: path for index creation + """ + try: + compresed = False + fields = {} + res, valid, fields, package_type = self.validate_artifact(path, source) + if not valid: + raise Exception('{} {} Not well configured.'.format(package_type.upper(), str(path))) + else: + if source == 'directory': + path = self.compress_artifact(path) + compresed = True + fields['checksum'] = self.md5(path) + self.indexation(destination, path, package_type, fields) + + except Exception as e: + self._logger.debug(str(e)) + + finally: + if source == 'directory' and compresed: + remove(path) + + def indexation(self, destination, path, package_type, fields): + """ + Process for index packages + :param destination: index repository path + :param path: path of the package + :param package_type: package type (vnf, ns) + :param fields: dict with the required values + """ + data_ind = {'name': fields.get('name'), 'description': fields.get('description'), + 'vendor': fields.get('vendor'), 'path': fields.get('path')} + + final_path = join(destination, package_type, fields.get('id'), fields.get('version')) + if isdir(join(destination, package_type, fields.get('id'))): + if isdir(final_path): + self._logger.warning('{} {} already exists'.format(package_type.upper(), str(path))) + else: + mkdir(final_path) + copyfile(path, + final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz') + yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w'), + Dumper=ruamel.yaml.RoundTripDumper) + index = yaml.load(open(destination + '/index.yaml')) + + index['{}_packages'.format(package_type)][fields.get('id')][fields.get('version')] = data_ind + if versioning.parse(index['{}_packages'.format(package_type)][fields.get('id')][ + 'latest']) < versioning.parse(fields.get('version')): + index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get( + 'version') + yaml.dump(index, open(destination + '/index.yaml', 'w'), Dumper=ruamel.yaml.RoundTripDumper) + self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path))) + else: + mkdir(destination + '/{}/'.format(package_type) + fields.get('id')) + mkdir(final_path) + copyfile(path, + final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz') + yaml.dump(fields, open(join(final_path, 'metadata.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) + index = yaml.load(open(destination + '/index.yaml')) + + index['{}_packages'.format(package_type)][fields.get('id')] = {fields.get('version'): data_ind} + index['{}_packages'.format(package_type)][fields.get('id')]['latest'] = fields.get('version') + yaml.dump(index, open(join(destination, 'index.yaml'), 'w'), Dumper=ruamel.yaml.RoundTripDumper) + self._logger.info('{} {} added in the repository'.format(package_type.upper(), str(path))) + + def current_datatime(self): + """ + Datetime Generator + :return: Datetime as string with the following structure "2020-04-29T08:41:07.681653Z" + """ + return time.strftime('%Y-%m-%dT%H:%M:%S.%sZ') + + def init_directory(self, destination): + """ + Initialize the index directory. Creation of index.yaml, and the directories for vnf and ns + :param destination: + :return: + """ + if not isdir(destination): + mkdir(destination) + if not isfile(join(destination, 'index.yaml')): + mkdir(join(destination, 'vnf')) + mkdir(join(destination, 'ns')) + index_data = {'apiVersion': 'v1', 'generated': self.current_datatime(), 'vnf_packages': {}, + 'ns_packages': {}} + with open(join(destination, 'index.yaml'), 'w') as outfile: + yaml.dump(index_data, outfile, default_flow_style=False) diff --git a/requirements.txt b/requirements.txt index 96b6e69..a2a4e5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,4 +21,7 @@ pycurl python-magic jinja2 verboselogs +packaging +ruamel.yaml +requests git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im diff --git a/setup.py b/setup.py index 1a60771..f5b9e15 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,8 @@ setup( license='Apache 2', install_requires=[ 'Click', 'prettytable', 'pyyaml', 'pycurl', 'python-magic', - 'jinja2', 'osm-im', 'verboselogs' + 'jinja2', 'osm-im', 'verboselogs', 'packaging', 'ruamel.yaml', + 'requests' ], dependency_links=[ 'git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im',