From 15b8b64b9b3134b13f9b229329f0cc1f508aad38 Mon Sep 17 00:00:00 2001 From: garciaale Date: Fri, 27 Nov 2020 15:17:17 -0300 Subject: [PATCH] Merges branch sol006 into master Change-Id: Idb5eb93eeab7e8696e097ee0c2e1cf79ea054f20 Signed-off-by: garciaale --- osmclient/common/package_tool.py | 78 +++++++++++---------- osmclient/common/utils.py | 36 +++++----- osmclient/scripts/osm.py | 116 ++++++++++++++----------------- osmclient/sol005/vnfd.py | 89 +++++++++++++++++------- setup.py | 8 +-- snap/snapcraft.yaml | 2 + 6 files changed, 184 insertions(+), 145 deletions(-) diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index ec0a6d4..ac0421f 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -27,7 +27,6 @@ import subprocess import shutil import yaml import logging -import pathlib class PackageTool(object): @@ -440,42 +439,47 @@ class PackageTool(object): def charms_search(self, descriptor_file, desc_type): self._logger.debug("") - - descriptor_dict = {} - charm_list = [] - bundle_list = [] - with open("{}".format(descriptor_file)) as yaml_desc: descriptor_dict = yaml.safe_load(yaml_desc) - for _, v1 in descriptor_dict.items(): - for _, v2 in v1.items(): - for entry in v2: - if "{}-configuration".format(desc_type) in entry: - name = entry["{}-configuration".format(desc_type)] - for _, v3 in name.items(): - if "charm" in v3: - charm_list.append((v3["charm"])) - if "vdu" in entry: - name = entry["vdu"] - for vdu in name: - if "vdu-configuration" in vdu: - for _, v4 in vdu["vdu-configuration"].items(): - if "charm" in v4: - charm_list.append((v4["charm"])) - if "kdu" in entry: - name = entry["kdu"] - for kdu in name: - if "juju-bundle" in kdu: - bundle_list.append(kdu["juju-bundle"]) - - path = pathlib.Path(descriptor_file).parent - for bundle in bundle_list: - descriptor_dict = {} - with open("{}/juju-bundles/{}".format(path, bundle)) as yaml_desc: - descriptor_dict = yaml.safe_load(yaml_desc) - if "applications" in descriptor_dict: - for _, v in descriptor_dict["applications"].items(): - charm_list.append(pathlib.Path(v["charm"]).name) - - return charm_list + if "catalog" in descriptor_dict: # Match OSM-IM vnfd-catalog and nsd-catalog + charms_list = self._charms_search_on_osm_im_dict(descriptor_dict, desc_type) + else: + charms_list = self._charms_search_on_sol006_dict(descriptor_dict, desc_type) + + return charms_list + + def _charms_search_on_osm_im_dict(self, osm_im_dict, desc_type): + charms_list = [] + for k1, v1 in osm_im_dict.items(): + for k2, v2 in v1.items(): + for entry in v2: + if '{}-configuration'.format(desc_type) in entry: + vnf_config = entry['{}-configuration'.format(desc_type)] + for k3, v3 in vnf_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + if 'vdu' in entry: + vdus = entry['vdu'] + for vdu in vdus: + if 'vdu-configuration' in vdu: + for k4, v4 in vdu['vdu-configuration'].items(): + if 'charm' in v4: + charms_list.append((v4['charm'])) + return charms_list + + def _charms_search_on_sol006_dict(self, sol006_dict, desc_type): + charms_list = [] + for k1, v1 in sol006_dict.items(): + for k2, v2 in v1.items(): + if '{}-configuration'.format(desc_type) in k2: + for vnf_config in v2: + for k3, v3 in vnf_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + if 'vdu-configuration' in k2: + for vdu_config in v2: + for k3, v3 in vdu_config.items(): + if 'charm' in v3: + charms_list.append((v3['charm'])) + return charms_list diff --git a/osmclient/common/utils.py b/osmclient/common/utils.py index ec0e0b0..94bd72f 100644 --- a/osmclient/common/utils.py +++ b/osmclient/common/utils.py @@ -68,23 +68,27 @@ def get_key_val_from_pkg(descriptor_file): dict = yaml.safe_load(tar.extractfile(yamlfile)) result = {} - for k1, v1 in list(dict.items()): - if not k1.endswith('-catalog'): - continue - for k2, v2 in v1.items(): - if not k2.endswith('nsd') and not k2.endswith('vnfd'): + for k in dict: + if 'nsd' in k: + result['type'] = 'nsd' + else: + result['type'] = 'vnfd' + if 'type' not in result: + for k1, v1 in list(dict.items()): + if not k1.endswith('-catalog'): continue + for k2, v2 in v1.items(): + if not k2.endswith('nsd') and not k2.endswith('vnfd'): + continue + if 'nsd' in k2: + result['type'] = 'nsd' + else: + result['type'] = 'vnfd' + for entry in v2: + for k3, v3 in list(entry.items()): + # strip off preceeding chars before : + key_name = k3.split(':').pop() + result[key_name] = v3 - if 'nsd' in k2: - result['type'] = 'nsd' - else: - result['type'] = 'vnfd' - - for entry in v2: - for k3, v3 in list(entry.items()): - # strip off preceeding chars before : - key_name = k3.split(':').pop() - - result[key_name] = v3 tar.close() return result diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py index 054bbd1..227e429 100755 --- a/osmclient/scripts/osm.py +++ b/osmclient/scripts/osm.py @@ -380,6 +380,7 @@ def ns_list(ctx, filter, long): project = project_name vim_id = nsr.get('datacenter') vim_name = get_vim_name(vim_list, vim_id) + #vim = '{} ({})'.format(vim_name, vim_id) vim = vim_name if 'currentOperation' in nsr: @@ -447,7 +448,7 @@ def nsd_list(ctx, filter, long): else: table = PrettyTable(['nsd name', 'id']) for nsd in resp: - name = nsd.get('name','-') + name = nsd.get('id', '-') if long: onb_state = nsd['_admin'].get('onboardingState','-') op_state = nsd['_admin'].get('operationalState','-') @@ -487,6 +488,26 @@ def nsd_list2(ctx, filter, long): nsd_list(ctx, filter, long) +def pkg_repo_list(ctx, pkgtype, filter, repo, long): + resp = ctx.obj.osmrepo.pkg_list(pkgtype, filter, repo) + if long: + table = PrettyTable(['nfpkg name', 'vendor', 'version', 'latest', 'description', 'repository']) + else: + table = PrettyTable(['nfpkg name', 'repository']) + for vnfd in resp: + name = vnfd.get('id', vnfd.get('name','-')) + repository = vnfd.get('repository') + if long: + vendor = vnfd.get('provider', vnfd.get('vendor')) + version = vnfd.get('version') + description = vnfd.get('description') + latest = vnfd.get('latest') + table.add_row([name, vendor, version, latest, description, repository]) + else: + table.add_row([name, repository]) + table.align = 'l' + print(table) + def vnfd_list(ctx, nf_type, filter, long): logger.debug("") if nf_type: @@ -516,23 +537,24 @@ def vnfd_list(ctx, nf_type, filter, long): fullclassname = ctx.obj.__module__ + "." + ctx.obj.__class__.__name__ if fullclassname == 'osmclient.sol005.client.Client': if long: - table = PrettyTable(['nfpkg name', 'id', 'vendor', 'version', 'onboarding state', 'operational state', + table = PrettyTable(['nfpkg name', 'id', 'desc type', 'vendor', 'version', 'onboarding state', 'operational state', 'usage state', 'date', 'last update']) else: - table = PrettyTable(['nfpkg name', 'id']) + table = PrettyTable(['nfpkg name', 'id', 'desc type']) for vnfd in resp: - name = vnfd['name'] if 'name' in vnfd else '-' + name = vnfd.get('id', vnfd.get('name','-')) + descriptor_type = 'sol005' if 'product-name' in vnfd else 'rel8' if long: onb_state = vnfd['_admin'].get('onboardingState','-') op_state = vnfd['_admin'].get('operationalState','-') - vendor = vnfd.get('vendor') + vendor = vnfd.get('provider', vnfd.get('vendor')) version = vnfd.get('version') usage_state = vnfd['_admin'].get('usageState','-') date = datetime.fromtimestamp(vnfd['_admin']['created']).strftime("%Y-%m-%dT%H:%M:%S") last_update = datetime.fromtimestamp(vnfd['_admin']['modified']).strftime("%Y-%m-%dT%H:%M:%S") - table.add_row([name, vnfd['_id'], vendor, version, onb_state, op_state, usage_state, date, last_update]) + table.add_row([name, vnfd['_id'], descriptor_type, vendor, version, onb_state, op_state, usage_state, date, last_update]) else: - table.add_row([name, vnfd['_id']]) + table.add_row([name, vnfd['_id'], descriptor_type]) else: table = PrettyTable(['nfpkg name', 'id']) for vnfd in resp: @@ -582,31 +604,8 @@ def nfpkg_list(ctx, nf_type, filter, long): # exit(1) -def pkg_repo_list(ctx, pkgtype, filter, repo, long): - if filter: - filter='&'.join(filter) - resp = ctx.obj.osmrepo.pkg_list(pkgtype, filter, repo) - if long: - table = PrettyTable(['nfpkg name', 'vendor', 'version', 'latest', 'description', 'repository']) - else: - table = PrettyTable(['nfpkg name', 'repository']) - for vnfd in resp: - name = vnfd.get('name', '-') - repository = vnfd.get('repository') - if long: - vendor = vnfd.get('vendor') - version = vnfd.get('version') - description = vnfd.get('description') - latest = vnfd.get('latest') - table.add_row([name, vendor, version, latest, description, repository]) - else: - table.add_row([name, repository]) - table.align = 'l' - print(table) - - @cli_osm.command(name='vnfpkg-repo-list', short_help='list all xNF from OSM repositories') -@click.option('--filter', default=None, multiple=True, +@click.option('--filter', default=None, help='restricts the list to the NFpkg matching the filter') @click.option('--repo', default=None, help='restricts the list to a particular OSM repository') @@ -617,9 +616,8 @@ def nfpkg_repo_list1(ctx, filter, repo, long): pkgtype = 'vnf' pkg_repo_list(ctx, pkgtype, filter, repo, long) - @cli_osm.command(name='nfpkg-repo-list', short_help='list all xNF from OSM repositories') -@click.option('--filter', default=None, multiple=True, +@click.option('--filter', default=None, help='restricts the list to the NFpkg matching the filter') @click.option('--repo', default=None, help='restricts the list to a particular OSM repository') @@ -631,32 +629,6 @@ def nfpkg_repo_list2(ctx, filter, repo, long): pkg_repo_list(ctx, pkgtype, filter, repo, long) -@cli_osm.command(name='nsd-repo-list', short_help='list all NS from OSM repositories') -@click.option('--filter', default=None, multiple=True, - help='restricts the list to the NS matching the filter') -@click.option('--repo', default=None, - help='restricts the list to a particular OSM repository') -@click.option('--long', is_flag=True, help='get more details') -@click.pass_context -def nspkg_repo_list(ctx, filter, repo, long): - """list xNF packages from OSM repositories""" - pkgtype = 'ns' - pkg_repo_list(ctx, pkgtype, filter, repo, long) - - -@cli_osm.command(name='nspkg-repo-list', short_help='list all NS from OSM repositories') -@click.option('--filter', default=None, multiple=True, - help='restricts the list to the NS matching the filter') -@click.option('--repo', default=None, - help='restricts the list to a particular OSM repository') -@click.option('--long', is_flag=True, help='get more details') -@click.pass_context -def nspkg_repo_list2(ctx, filter, repo, long): - """list xNF packages from OSM repositories""" - pkgtype = 'ns' - pkg_repo_list(ctx, pkgtype, filter, repo, long) - - def vnf_list(ctx, ns, filter, long): # try: if ns or filter: @@ -720,6 +692,29 @@ def vnf_list1(ctx, ns, filter, long): logger.debug("") vnf_list(ctx, ns, filter, long) +@cli_osm.command(name='nsd-repo-list', short_help='list all NS from OSM repositories') +@click.option('--filter', default=None, multiple=True, + help='restricts the list to the NS matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def nspkg_repo_list(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'ns' + pkg_repo_list(ctx, pkgtype, filter, repo, long) + +@cli_osm.command(name='nspkg-repo-list', short_help='list all NS from OSM repositories') +@click.option('--filter', default=None, multiple=True, + help='restricts the list to the NS matching the filter') +@click.option('--repo', default=None, + help='restricts the list to a particular OSM repository') +@click.option('--long', is_flag=True, help='get more details') +@click.pass_context +def nspkg_repo_list2(ctx, filter, repo, long): + """list xNF packages from OSM repositories""" + pkgtype = 'ns' + pkg_repo_list(ctx, pkgtype, filter, repo, long) @cli_osm.command(name='nf-list', short_help='list all NF instances') @click.option('--ns', default=None, help='NS instance id or name to restrict the NF list') @@ -1142,7 +1137,6 @@ def vnfd_show2(ctx, name, literal): logger.debug("") vnfd_show(ctx, name, literal) - @cli_osm.command(name='vnfpkg-repo-show', short_help='shows the details of a NF package in an OSM repository') @click.option('--literal', is_flag=True, help='print literally, no pretty table') @@ -1186,7 +1180,6 @@ def nsd_repo_show(ctx, name, repo, version, literal=None, filter=None): pkgtype = 'ns' pkg_repo_show(ctx, pkgtype, name, repo, version, filter, literal) - @cli_osm.command(name='nspkg-repo-show', short_help='shows the details of a NS package in an OSM repository') @click.option('--literal', is_flag=True, help='print literally, no pretty table') @@ -1360,7 +1353,6 @@ def vnf_show(ctx, name, literal, filter, kdu): return table = PrettyTable(['field', 'value']) - for k, v in list(resp.items()): if not filter or k in filter: table.add_row([k, wrap_text(text=json.dumps(v,indent=2),width=100)]) diff --git a/osmclient/sol005/vnfd.py b/osmclient/sol005/vnfd.py index 159205a..e32ac83 100644 --- a/osmclient/sol005/vnfd.py +++ b/osmclient/sol005/vnfd.py @@ -51,6 +51,7 @@ class Vnfd(object): if filter: filter_string = '?{}'.format(filter) _, resp = self._http.get2_cmd('{}{}'.format(self._apiBase, filter_string)) + if resp: return json.loads(resp) return list() @@ -64,7 +65,9 @@ class Vnfd(object): return vnfd else: for vnfd in self.list(): - if 'name' in vnfd and name == vnfd['name']: + if 'product-name' in vnfd and name == vnfd['product-name']: + return vnfd + elif 'name' in vnfd and name == vnfd['name']: return vnfd raise NotFound("vnfd {} not found".format(name)) @@ -128,6 +131,7 @@ class Vnfd(object): filename = filename.rstrip('/') filename = self._client.package_tool.build(filename, skip_validation=False, skip_charm_build=skip_charm_build) + print('Uploading package {}'.format(filename)) self.create(filename, overwrite=overwrite, update_endpoint=update_endpoint, override_epa=override_epa, override_nonepa=override_nonepa, @@ -151,9 +155,10 @@ class Vnfd(object): "Unexpected MIME type for file {}: MIME type {}".format( filename, mime_type) ) - special_ow_string = '' + + special_override_string = '' if override_epa or override_nonepa or override_paravirt: - # If override for EPA, non-EPA or paravirt is required, get the descriptor data + # If override for EPA, non-EPA or paravirt is required, get the descriptor data descriptor_data = None if mime_type in ['application/yaml', 'text/plain', 'application/json']: with open(filename) as df: @@ -176,47 +181,81 @@ class Vnfd(object): raise ClientException('Descriptor could not be read') desc_type, vnfd = validation_im().yaml_validation(descriptor_data) validation_im().pyangbind_validation(desc_type, vnfd) + vnfd = yaml.safe_load(descriptor_data) + vcd_list = [] vdu_list = [] for k in vnfd: # Get only the first descriptor in case there are many in the yaml file - # k can be vnfd:vnfd-catalog or vnfd-catalog. This check is skipped - first_vnfd = vnfd[k]['vnfd'][0] + # k can be vnfd or etsi-nfv-vnfd:vnfd. This check is skipped + first_vnfd = vnfd.get(k, {}) + vcd_list = first_vnfd.get('virtual-compute-desc', []) vdu_list = first_vnfd.get('vdu', []) break - for vdu_number, vdu in enumerate(vdu_list): + + for vcd_number, vcd in enumerate(vcd_list): if override_epa: - guest_epa = {} - guest_epa["mempage-size"] = "LARGE" - guest_epa["cpu-pinning-policy"] = "DEDICATED" - guest_epa["cpu-thread-pinning-policy"] = "PREFER" - guest_epa["numa-node-policy"] = {} - guest_epa["numa-node-policy"]["node-cnt"] = 1 - guest_epa["numa-node-policy"]["mem-policy"] = "STRICT" - special_ow_string = "{}vdu.{}.guest-epa={};".format(special_ow_string, vdu_number, - quote(yaml.safe_dump(guest_epa))) - headers['Query-String-Format'] = 'yaml' + virtual_memory = vcd["virtual-memory"] + virtual_memory["mempage-size"] = "LARGE" + virtual_memory["numa-enabled"] = True + virtual_memory["numa-node-policy"] = { + "node-cnt": 1, + "mem-policy": "STRICT" + } + virtual_cpu = vcd["virtual-cpu"] + virtual_cpu["pinning"] = { + "policy": "static", + "thread-policy": "PREFER" + } + + cpu_override_string = "virtual-compute-desc.{}.virtual-cpu={};"\ + .format(vcd_number, quote(yaml.safe_dump(virtual_cpu))) + memory_override_string = "virtual-compute-desc.{}.virtual-memory={};"\ + .format(vcd_number, quote(yaml.safe_dump(virtual_memory))) + special_override_string = "{}{}{}".format(special_override_string, + cpu_override_string, memory_override_string) + + headers['Query-String-Format'] = 'yaml' if override_nonepa: - special_ow_string = "{}vdu.{}.guest-epa=;".format(special_ow_string, vdu_number) - if override_paravirt: - for iface_number in range(len(vdu['interface'])): - special_ow_string = "{}vdu.{}.interface.{}.virtual-interface.type=PARAVIRT;".format( - special_ow_string, vdu_number, iface_number) - special_ow_string = special_ow_string.rstrip(";") + virtual_memory = vcd["virtual-memory"] + virtual_memory["mempage-size"] = "" + virtual_memory["numa-enabled"] = "" + virtual_memory["numa-node-policy"] = {} + virtual_cpu = vcd["virtual-cpu"] + virtual_cpu["pinning"] = {} + + cpu_override_string = "virtual-compute-desc.{}.virtual-cpu={};"\ + .format(vcd_number, quote(yaml.safe_dump(virtual_cpu))) + memory_override_string = "virtual-compute-desc.{}.virtual-memory={};"\ + .format(vcd_number, quote(yaml.safe_dump(virtual_memory))) + special_override_string = "{}{}{}".format(special_override_string, + cpu_override_string, memory_override_string) + + if override_paravirt: + for vdu_number, vdu in enumerate(vdu_list): + for cpd_number, cpd in enumerate(vdu["int-cpd"]): + for vnir_number, vnir in enumerate(cpd['virtual-network-interface-requirement']): + special_override_string = "{}vdu.{}.int-cpd.{}.virtual-network-interface-" \ + "requirement.{}.virtual-interface.type=PARAVIRT;"\ + .format(special_override_string, vdu_number, cpd_number, vnir_number) + + special_override_string = special_override_string.rstrip(";") headers["Content-File-MD5"] = utils.md5(filename) http_header = ['{}: {}'.format(key, val) for (key, val) in list(headers.items())] + self._http.set_http_header(http_header) if update_endpoint: http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) else: ow_string = '' - if special_ow_string: + if special_override_string: if overwrite: - overwrite = "{};{}".format(overwrite, special_ow_string) + overwrite = "{};{}".format(overwrite, special_override_string) else: - overwrite = special_ow_string + overwrite = special_override_string + if overwrite: ow_string = '?{}'.format(overwrite) self._apiResource = '/vnf_packages_content' diff --git a/setup.py b/setup.py index 6bf023e..2140468 100644 --- a/setup.py +++ b/setup.py @@ -28,12 +28,10 @@ setup( description=_description, license='Apache 2', install_requires=[ - 'Click', 'prettytable<=1.0.1', 'pyyaml', 'pycurl', 'python-magic', + 'Click', 'prettytable', 'pyyaml', 'pycurl', 'python-magic', 'jinja2', 'osm-im', 'verboselogs', 'packaging', 'ruamel.yaml', - 'requests' - ], - dependency_links=[ - 'git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im', + 'requests', + 'osm-im @ git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im', ], setup_requires=['setuptools-version-command'], test_suite='nose.collector', diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 805f1b6..84635ae 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -50,6 +50,8 @@ parts: build-packages: - gcc - git + - make + - wget stage-packages: - libmagic1 - python3 -- 2.25.1