From 566a8c34fbf1d676a0acd725f6f30fce8baf8c0a Mon Sep 17 00:00:00 2001 From: pinoa Date: Mon, 25 Nov 2019 15:25:18 +0100 Subject: [PATCH] Capability to upload a package from a source folder Change-Id: I26d695e724175bef07c25736e1eec907603f72c2 Signed-off-by: pinoa --- osmclient/common/package_tool.py | 186 ++++++++++++++++++++++++++++--- osmclient/common/utils.py | 2 +- osmclient/scripts/osm.py | 78 ++++++++----- osmclient/sol005/nsd.py | 110 +++++++++--------- osmclient/sol005/nst.py | 119 +++++++++++--------- osmclient/sol005/package.py | 84 +++++++------- osmclient/sol005/vnfd.py | 116 ++++++++++--------- 7 files changed, 454 insertions(+), 241 deletions(-) diff --git a/osmclient/common/package_tool.py b/osmclient/common/package_tool.py index 7cc10be..72d7d3d 100644 --- a/osmclient/common/package_tool.py +++ b/osmclient/common/package_tool.py @@ -23,11 +23,15 @@ import tarfile import hashlib from osm_im.validation import Validation as validation_im from jinja2 import Environment, PackageLoader - +import subprocess +import shutil +import yaml +import logging class PackageTool(object): def __init__(self, client=None): self._client = client + self._logger = logging.getLogger('osmclient') def create(self, package_type, base_directory, package_name, override, image, vdus, vcpu, memory, storage, interfaces, vendor, detailed, netslice_subnets, netslice_vlds): @@ -50,7 +54,7 @@ class PackageTool(object): :return: status """ - + self._logger.debug("") # print("location: {}".format(osmclient.__path__)) file_loader = PackageLoader("osmclient") env = Environment(loader=file_loader) @@ -80,7 +84,7 @@ class PackageTool(object): self.create_files(structure["files"], output, package_type) return "Created" - def validate(self, base_directory): + def validate(self, base_directory, recursive=True): """ **Validate OSM Descriptors given a path** @@ -89,8 +93,12 @@ class PackageTool(object): :return: List of dict of validated descriptors. keys: type, path, valid, error """ + self._logger.debug("") table = [] - descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=True)] + if recursive: + descriptors_paths = [f for f in glob.glob(base_directory + "/**/*.yaml", recursive=recursive)] + else: + descriptors_paths = [f for f in glob.glob(base_directory + "/*.yaml", recursive=recursive)] print("Base directory: {}".format(base_directory)) print("{} Descriptors found to validate".format(len(descriptors_paths))) for desc_path in descriptors_paths: @@ -105,7 +113,7 @@ class PackageTool(object): table.append({"type": desc_type, "path": desc_path, "valid": "ERROR", "error": str(e)}) return table - def build(self, package_folder, skip_validation=True): + def build(self, package_folder, skip_validation=False, skip_charm_build=False): """ **Creates a .tar.gz file given a package_folder** @@ -115,20 +123,21 @@ class PackageTool(object): :returns: message result for the build process """ - + self._logger.debug("") + package_folder = package_folder.rstrip('/') if not os.path.exists("{}".format(package_folder)): return "Fail, package is not in the specified route" if not skip_validation: - results = self.validate(package_folder) - for result in results: - if result["valid"] != "OK": - return("There was an error validating the file: {} with error: {}".format(result["path"], - result["error"])) - self.calculate_checksum(package_folder) - with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive: - print("Adding File: {}".format(package_folder)) - archive.add('{}'.format(package_folder), recursive=True) - return "Created {}.tar.gz".format(package_folder) + results = self.validate(package_folder, recursive=False) + if results: + for result in results: + if result["valid"] != "OK": + raise ClientException("There was an error validating the file: {} with error: {}" + .format(result["path"], result["error"])) + else: + raise ClientException("No descriptor file found in: {}".format(package_folder)) + charm_list = self.build_all_charms(package_folder, skip_charm_build) + return self.build_tarfile(package_folder, charm_list) def calculate_checksum(self, package_folder): """ @@ -138,10 +147,11 @@ class PackageTool(object): - package_folder: is the folder where we have the files to calculate the checksum :returns: None """ + self._logger.debug("") files = [f for f in glob.glob(package_folder + "/**/*.*", recursive=True)] - checksum = open("{}/checksum.txt".format(package_folder), "w+") + checksum = open("{}/checksums.txt".format(package_folder), "w+") for file_item in files: - if "checksum.txt" in file_item: + if "checksums.txt" in file_item: continue # from https://www.quickprogrammingtips.com/python/how-to-calculate-md5-hash-of-a-file-in-python.html md5_hash = hashlib.md5() @@ -215,6 +225,7 @@ class PackageTool(object): :return: None """ + self._logger.debug("") for file_item, file_package, file_type in files: if package_type == file_package: if file_type == "descriptor": @@ -234,6 +245,7 @@ class PackageTool(object): :return: Missing paths Dict """ + self._logger.debug("") missing_paths = {} folders = [] files = [] @@ -249,6 +261,38 @@ class PackageTool(object): return missing_paths + def build_all_charms(self, package_folder, skip_charm_build): + """ + **Read the descriptor file, check that the charms referenced are in the folder and compiles them** + + :params: + - packet_folder: is the location of the package + :return: Files and Folders not found. In case of override, it will return all file list + """ + listCharms = [] + self._logger.debug("") + descriptor_file = False + descriptors_paths = [f for f in glob.glob(package_folder + "/*.yaml")] + for file in descriptors_paths: + if 'nfd.yaml' in file: + descriptor_file = True + listCharms = self.charms_search(file, 'vnf') + if 'nsd.yaml' in file: + descriptor_file = True + listCharms = self.charms_search(file, 'ns') + if not descriptor_file: + raise ClientException ('descriptor name is not correct in: {}'.format(package_folder)) + if listCharms and not skip_charm_build: + for charmName in listCharms: + if os.path.isdir('{}/charms/layers/{}'.format(package_folder,charmName)): + self.charm_build(package_folder, charmName) + else: + if not os.path.isdir('{}/charms/{}'.format(package_folder,charmName)): + raise ClientException ('The charm: {} referenced in the descriptor file ' + 'is not present either in {}/charms or in {}/charms/layers'. + format(charmName, package_folder,package_folder)) + return listCharms + def discover_folder_structure(self, base_directory, name, override): """ **Discover files and folders structure for OSM descriptors given a base_directory and name** @@ -259,6 +303,7 @@ class PackageTool(object): - override: is the flag used to indicate the creation of the list even if the file exist to override it :return: Files and Folders not found. In case of override, it will return all file list """ + self._logger.debug("") prefix = "{}/{}".format(base_directory, name) files_folders = {"folders": [("{}_ns".format(prefix), "ns"), ("{}_ns/icons".format(prefix), "ns"), @@ -284,3 +329,108 @@ class PackageTool(object): missing_files_folders = self.check_files_folders(files_folders, override) # print("Missing files and folders: {}".format(missing_files_folders)) return missing_files_folders + + def charm_build(self, charms_folder, build_name): + """ + Build the charms inside the package. + params: package_folder is the name of the folder where is the charms to compile. + build_name is the name of the layer or interface + """ + self._logger.debug("") + os.environ['JUJU_REPOSITORY'] = "{}/charms".format(charms_folder) + os.environ['CHARM_LAYERS_DIR'] = "{}/layers".format(os.environ['JUJU_REPOSITORY']) + os.environ['CHARM_INTERFACES_DIR'] = "{}/interfaces".format(os.environ['JUJU_REPOSITORY']) + os.environ['CHARM_BUILD_DIR'] = "{}/charms/builds".format(charms_folder) + src_folder = '{}/{}'.format(os.environ['CHARM_LAYERS_DIR'], build_name) + result = subprocess.run(["charm", "build", "{}".format(src_folder)]) + if result.returncode == 1: + raise ClientException("failed to build the charm: {}".format(src_folder)) + self._logger.verbose("charm: {} compiled".format(src_folder)) + + def build_tarfile(self, package_folder, charm_list=None): + """ + Creates a .tar.gz file given a package_folder + params: package_folder is the name of the folder to be packaged + returns: .tar.gz name + """ + self._logger.debug("") + ignore_patterns = "'*layers*', '*interfaces*'" + try: + directory_name = self.create_temp_dir(package_folder, ignore_patterns, charm_list) + cwd = os.getcwd() + os.chdir(directory_name) + self.calculate_checksum(package_folder) + with tarfile.open("{}.tar.gz".format(package_folder), mode='w:gz') as archive: + print("Adding File: {}".format(package_folder)) + archive.add('{}'.format(package_folder), recursive=True) + #return "Created {}.tar.gz".format(package_folder) + #self.build("{}".format(os.path.basename(package_folder))) + os.chdir(cwd) + except: + shutil.rmtree(os.path.join(package_folder, "tmp")) + raise ClientException('failure to manipulate the result of the compilation') + os.rename("{}/{}.tar.gz".format(directory_name, os.path.basename(package_folder)), + "{}.tar.gz".format(os.path.basename(package_folder))) + os.rename("{}/{}/checksums.txt".format(directory_name, os.path.basename(package_folder)), + "{}/checksums.txt".format(package_folder)) + shutil.rmtree(os.path.join(package_folder, "tmp")) + self._logger.verbose("package created: {}.tar.gz".format(os.path.basename(package_folder))) + return "{}.tar.gz".format(package_folder) + + def create_temp_dir(self, package_folder, ignore_patterns=None, charm_list=None): + """ + Method to create a temporary folder where we can move the files in package_folder, which do not + meet the pattern defined in ignore_patterns + """ + self._logger.debug("") + ignore = shutil.ignore_patterns(ignore_patterns) + os.makedirs("{}/tmp".format(package_folder), exist_ok=True) + directory_name = os.path.abspath("{}/tmp".format(package_folder)) + os.makedirs("{}/{}".format(directory_name, os.path.basename(package_folder),exist_ok=True)) + for item in os.listdir(package_folder): + if item != "tmp": + s = os.path.join(package_folder, item) + d = os.path.join(os.path.join(directory_name, os.path.basename(package_folder)), item) + if os.path.isdir(s): + if item == "charms": + s = os.path.join(s, "builds") + if not os.path.exists(s): + os.makedirs(s) + for i in os.listdir(s): + if i in charm_list: + s_charm = os.path.join(s, i) + # d_charm = os.path.join(package_folder, item, i) + d_temp = os.path.join(d, i) + # if os.path.exists(d_charm): + # shutil.rmtree(d_charm) + shutil.copytree(s_charm, d_temp, symlinks = True, ignore = ignore) + # shutil.copytree(s_charm, d_charm, symlinks = True, ignore = ignore) + else: + shutil.copytree(s, d, symlinks = True, ignore = ignore) + else: + shutil.copy2(s, d) + return directory_name + + def charms_search(self, descriptor_file, desc_type): + self._logger.debug("") + dict = {} + list = [] + with open("{}".format(descriptor_file)) as yaml_desc: + dict = yaml.safe_load(yaml_desc) + for k1, v1 in dict.items(): + for k2, v2 in v1.items(): + for entry in v2: + if '{}-configuration'.format(desc_type) in entry: + name = entry['{}-configuration'.format(desc_type)] + for k3, v3 in name.items(): + if 'charm' in v3: + list.append((v3['charm'])) + if 'vdu' in entry: + name = entry['vdu'] + for vdu in name: + if 'vdu-configuration' in vdu: + for k4, v4 in vdu['vdu-configuration'].items(): + if 'charm' in v4: + list.append((v4['charm'])) + return list + diff --git a/osmclient/common/utils.py b/osmclient/common/utils.py index 43287e9..ec0e0b0 100644 --- a/osmclient/common/utils.py +++ b/osmclient/common/utils.py @@ -71,7 +71,7 @@ def get_key_val_from_pkg(descriptor_file): for k1, v1 in list(dict.items()): if not k1.endswith('-catalog'): continue - for k2, v2 in list(v1.items()): + for k2, v2 in v1.items(): if not k2.endswith('nsd') and not k2.endswith('vnfd'): continue diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py index 280264b..e3e2a28 100755 --- a/osmclient/scripts/osm.py +++ b/osmclient/scripts/osm.py @@ -1343,11 +1343,11 @@ def pdu_show(ctx, name, literal, filter): # CREATE operations #################### -def nsd_create(ctx, filename, overwrite): +def nsd_create(ctx, filename, overwrite, skip_charm_build): logger.debug("") # try: check_client_version(ctx.obj, ctx.command.name) - ctx.obj.nsd.create(filename, overwrite) + ctx.obj.nsd.create(filename, overwrite=overwrite, skip_charm_build=skip_charm_build) # except ClientException as e: # print(str(e)) # exit(1) @@ -1360,14 +1360,16 @@ def nsd_create(ctx, filename, overwrite): @click.option('--override', 'overwrite', default=None, help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='The charm will not be compiled, it is assumed to already exist') @click.pass_context -def nsd_create1(ctx, filename, overwrite): +def nsd_create1(ctx, filename, overwrite, skip_charm_build): """creates a new NSD/NSpkg FILENAME: NSD yaml file or NSpkg tar.gz file """ logger.debug("") - nsd_create(ctx, filename, overwrite) + nsd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) @cli_osm.command(name='nspkg-create', short_help='creates a new NSD/NSpkg') @@ -1377,21 +1379,23 @@ def nsd_create1(ctx, filename, overwrite): @click.option('--override', 'overwrite', default=None, help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='The charm will not be compiled, it is assumed to already exist') @click.pass_context -def nsd_create2(ctx, filename, overwrite): +def nsd_create2(ctx, charm_folder, overwrite, skip_charm_build): """creates a new NSD/NSpkg - FILENAME: NSD yaml file or NSpkg tar.gz file + FILENAME: NSD folder, NSD yaml file or NSpkg tar.gz file """ logger.debug("") - nsd_create(ctx, filename, overwrite) + nsd_create(ctx, charm_folder, overwrite=overwrite, skip_charm_build=skip_charm_build) -def vnfd_create(ctx, filename, overwrite): +def vnfd_create(ctx, filename, overwrite, skip_charm_build): logger.debug("") # try: check_client_version(ctx.obj, ctx.command.name) - ctx.obj.vnfd.create(filename, overwrite) + ctx.obj.vnfd.create(filename, overwrite=overwrite, skip_charm_build=skip_charm_build) # except ClientException as e: # print(str(e)) # exit(1) @@ -1404,14 +1408,16 @@ def vnfd_create(ctx, filename, overwrite): @click.option('--override', 'overwrite', default=None, help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='The charm will not be compiled, it is assumed to already exist') @click.pass_context -def vnfd_create1(ctx, filename, overwrite): +def vnfd_create1(ctx, filename, overwrite, skip_charm_build): """creates a new VNFD/VNFpkg FILENAME: VNFD yaml file or VNFpkg tar.gz file """ logger.debug("") - vnfd_create(ctx, filename, overwrite) + vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) @cli_osm.command(name='vnfpkg-create', short_help='creates a new VNFD/VNFpkg') @@ -1421,14 +1427,16 @@ def vnfd_create1(ctx, filename, overwrite): @click.option('--override', 'overwrite', default=None, help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='The charm will not be compiled, it is assumed to already exist') @click.pass_context -def vnfd_create2(ctx, filename, overwrite): +def vnfd_create2(ctx, filename, overwrite, skip_charm_build): """creates a new VNFD/VNFpkg - FILENAME: VNFD yaml file or VNFpkg tar.gz file + FILENAME: NF Package Folder, NF Descriptor yaml file or NFpkg tar.gz file """ logger.debug("") - vnfd_create(ctx, filename, overwrite) + vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) @cli_osm.command(name='nfpkg-create', short_help='creates a new NFpkg') @@ -1438,14 +1446,16 @@ def vnfd_create2(ctx, filename, overwrite): @click.option('--override', 'overwrite', default=None, help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='The charm will not be compiled, it is assumed to already exist') @click.pass_context -def nfpkg_create(ctx, filename, overwrite): +def nfpkg_create(ctx, filename, overwrite, skip_charm_build): """creates a new NFpkg - FILENAME: NF Descriptor yaml file or NFpkg tar.gz file + FILENAME: NF Package Folder, NF Descriptor yaml file or NFpkg tar.gz filems to build """ logger.debug("") - vnfd_create(ctx, filename, overwrite) + vnfd_create(ctx, filename, overwrite=overwrite, skip_charm_build=skip_charm_build) @cli_osm.command(name='ns-create', short_help='creates a new Network Service instance') @@ -1522,13 +1532,13 @@ def nst_create(ctx, filename, overwrite): help='overrides fields in descriptor, format: ' '"key1.key2...=value[;key3...=value;...]"') @click.pass_context -def nst_create1(ctx, filename, overwrite): +def nst_create1(ctx, charm_folder, overwrite): """creates a new Network Slice Template (NST) - FILENAME: NST yaml file or NSTpkg tar.gz file + FILENAME: NST package folder, NST yaml file or NSTpkg tar.gz file """ logger.debug("") - nst_create(ctx, filename, overwrite) + nst_create(ctx, charm_folder, overwrite) @cli_osm.command(name='netslice-template-create', short_help='creates a new Network Slice Template (NST)') @@ -3319,15 +3329,17 @@ def get_version(ctx): @cli_osm.command(name='upload-package', short_help='uploads a VNF package or NS package') @click.argument('filename') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='the charm will not be compiled, it is assumed to already exist') @click.pass_context -def upload_package(ctx, filename): - """uploads a VNF package or NS package +def upload_package(ctx, filename, skip_charm_build): + """uploads a vnf package or ns package - FILENAME: VNF or NS package file (tar.gz) + filename: vnf or ns package folder, or vnf or ns package file (tar.gz) """ logger.debug("") # try: - ctx.obj.package.upload(filename) + ctx.obj.package.upload(filename, skip_charm_build=skip_charm_build) fullclassname = ctx.obj.__module__ + "." + ctx.obj.__class__.__name__ if fullclassname != 'osmclient.sol005.client.Client': ctx.obj.package.wait_for_upload(filename) @@ -3794,9 +3806,14 @@ def package_create(ctx, @click.argument('base-directory', default=".", required=False) +@click.option('--recursive/--no-recursive', + default=True, + help='The activated recursive option will validate the yaml files' + ' within the indicated directory and in its subdirectories') @click.pass_context def package_validate(ctx, - base_directory): + base_directory, + recursive): """ Validate descriptors given a base directory. @@ -3805,7 +3822,7 @@ def package_validate(ctx, """ # try: check_client_version(ctx.obj, ctx.command.name) - results = ctx.obj.package_tool.validate(base_directory) + results = ctx.obj.package_tool.validate(base_directory, recursive) table = PrettyTable() table.field_names = ["TYPE", "PATH", "VALID", "ERROR"] # Print the dictionary generated by the validation function @@ -3827,10 +3844,13 @@ def package_validate(ctx, default=False, is_flag=True, help='skip package validation') +@click.option('--skip-charm-build', default=False, is_flag=True, + help='the charm will not be compiled, it is assumed to already exist') @click.pass_context def package_build(ctx, package_folder, - skip_validation): + skip_validation, + skip_charm_build): """ Build the package NS, VNF given the package_folder. @@ -3839,7 +3859,9 @@ def package_build(ctx, """ # try: check_client_version(ctx.obj, ctx.command.name) - results = ctx.obj.package_tool.build(package_folder, skip_validation) + results = ctx.obj.package_tool.build(package_folder, + skip_validation=skip_validation, + skip_charm_build=skip_charm_build) print(results) # except ClientException as inst: # print("ERROR: {}".format(inst)) diff --git a/osmclient/sol005/nsd.py b/osmclient/sol005/nsd.py index 5979bc0..9aca930 100644 --- a/osmclient/sol005/nsd.py +++ b/osmclient/sol005/nsd.py @@ -25,6 +25,7 @@ import json import magic from os.path import basename import logging +import os.path #from os import stat @@ -138,62 +139,69 @@ class Nsd(object): # msg = resp raise ClientException("failed to delete nsd {} - {}".format(name, msg)) - def create(self, filename, overwrite=None, update_endpoint=None): + def create(self, filename, overwrite=None, update_endpoint=None, skip_charm_build=False): self._logger.debug("") - self._client.get_token() - mime_type = magic.from_file(filename, mime=True) - if mime_type is None: - raise ClientException( - "failed to guess MIME type for file '{}'".format(filename)) - headers= self._client._headers - headers['Content-Filename'] = basename(filename) - if mime_type in ['application/yaml', 'text/plain', 'application/json']: - headers['Content-Type'] = 'text/plain' - elif mime_type in ['application/gzip', 'application/x-gzip']: - headers['Content-Type'] = 'application/gzip' - #headers['Content-Type'] = 'application/binary' - # Next three lines are to be removed in next version - #headers['Content-Filename'] = basename(filename) - #file_size = stat(filename).st_size - #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) + if os.path.isdir(filename): + filename = filename.rstrip('/') + filename = self._client.package_tool.build(filename, skip_validation=False, skip_charm_build=skip_charm_build) + self.create(filename, overwrite=overwrite, update_endpoint=update_endpoint) else: - raise ClientException( + self._client.get_token() + mime_type = magic.from_file(filename, mime=True) + if mime_type is None: + raise ClientException( + "Unexpected MIME type for file {}: MIME type {}".format( + filename, mime_type) + ) + headers= self._client._headers + headers['Content-Filename'] = basename(filename) + if mime_type in ['application/yaml', 'text/plain', 'application/json']: + headers['Content-Type'] = 'text/plain' + elif mime_type in ['application/gzip', 'application/x-gzip']: + headers['Content-Type'] = 'application/gzip' + #headers['Content-Type'] = 'application/binary' + # Next three lines are to be removed in next version + #headers['Content-Filename'] = basename(filename) + #file_size = stat(filename).st_size + #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) + else: + raise ClientException( "Unexpected MIME type for file {}: MIME type {}".format( filename, mime_type) ) - headers["Content-File-MD5"] = utils.md5(filename) - http_header = ['{}: {}'.format(key,val) - for (key,val) in list(headers.items())] - self._http.set_http_header(http_header) - if update_endpoint: - http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) - else: - ow_string = '' - if overwrite: - ow_string = '?{}'.format(overwrite) - self._apiResource = '/ns_descriptors_content' - self._apiBase = '{}{}{}'.format(self._apiName, - self._apiVersion, self._apiResource) - endpoint = '{}{}'.format(self._apiBase,ow_string) - http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) - #print('HTTP CODE: {}'.format(http_code)) - #print('RESP: {}'.format(resp)) - if http_code in (200, 201, 202): - if resp: - resp = json.loads(resp) - if not resp or 'id' not in resp: - raise ClientException('unexpected response from server - {}'.format(resp)) - print(resp['id']) - elif http_code == 204: - print('Updated') - # else: - # msg = "Error {}".format(http_code) - # if resp: - # try: - # msg = "{} - {}".format(msg, json.loads(resp)) - # except ValueError: - # msg = "{} - {}".format(msg, resp) - # raise ClientException("failed to create/update nsd - {}".format(msg)) + headers["Content-File-MD5"] = utils.md5(filename) + http_header = ['{}: {}'.format(key,val) + for (key,val) in list(headers.items())] + self._http.set_http_header(http_header) + if update_endpoint: + http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) + else: + ow_string = '' + if overwrite: + ow_string = '?{}'.format(overwrite) + self._apiResource = '/ns_descriptors_content' + self._apiBase = '{}{}{}'.format(self._apiName, + self._apiVersion, self._apiResource) + endpoint = '{}{}'.format(self._apiBase,ow_string) + http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) + #print('HTTP CODE: {}'.format(http_code)) + #print('RESP: {}'.format(resp)) + if http_code in (200, 201, 202): + if resp: + resp = json.loads(resp) + if not resp or 'id' not in resp: + raise ClientException('unexpected response from server - {}'.format(resp)) + print(resp['id']) + elif http_code == 204: + print('Updated') + # else: + # msg = "Error {}".format(http_code) + # if resp: + # try: + # msg = "{} - {}".format(msg, json.loads(resp)) + # except ValueError: + # msg = "{} - {}".format(msg, resp) + # raise ClientException("failed to create/update nsd - {}".format(msg)) def update(self, name, filename): self._logger.debug("") diff --git a/osmclient/sol005/nst.py b/osmclient/sol005/nst.py index 3b15e96..214a79d 100644 --- a/osmclient/sol005/nst.py +++ b/osmclient/sol005/nst.py @@ -24,6 +24,7 @@ from osmclient.common import utils import json import magic import logging +import os.path #from os import stat #from os.path import basename @@ -139,57 +140,75 @@ class Nst(object): def create(self, filename, overwrite=None, update_endpoint=None): self._logger.debug("") - self._client.get_token() - mime_type = magic.from_file(filename, mime=True) - if mime_type is None: - raise ClientException( - "failed to guess MIME type for file '{}'".format(filename)) - headers= self._client._headers - if mime_type in ['application/yaml', 'text/plain']: - headers['Content-Type'] = 'application/yaml' - elif mime_type in ['application/gzip', 'application/x-gzip']: - headers['Content-Type'] = 'application/gzip' - #headers['Content-Type'] = 'application/binary' - # Next three lines are to be removed in next version - #headers['Content-Filename'] = basename(filename) - #file_size = stat(filename).st_size - #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) - else: - raise ClientException( - "Unexpected MIME type for file {}: MIME type {}".format( - filename, mime_type) - ) - headers["Content-File-MD5"] = utils.md5(filename) - http_header = ['{}: {}'.format(key,val) - for (key,val) in list(headers.items())] - self._http.set_http_header(http_header) - if update_endpoint: - http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) + if os.path.isdir(filename): + charm_folder = filename.rstrip('/') + for files in os.listdir(charm_folder): + if "nst.yaml" in files: + results = self._client.package_tool.validate(charm_folder, recursive=False) + for result in results: + if result["valid"] != "OK": + raise ClientException('There was an error validating the file: {} ' + 'with error: {}'.format(result["path"], result["error"])) + result = self._client.package_tool.build(charm_folder) + if 'Created' in result: + filename = "{}.tar.gz".format(charm_folder) + else: + raise ClientException('Failed in {}tar.gz creation'.format(charm_folder)) + self.create(filename, overwrite, update_endpoint) else: - ow_string = '' - if overwrite: - ow_string = '?{}'.format(overwrite) - self._apiResource = '/netslice_templates_content' - self._apiBase = '{}{}{}'.format(self._apiName, - self._apiVersion, self._apiResource) - endpoint = '{}{}'.format(self._apiBase,ow_string) - http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) - #print('HTTP CODE: {}'.format(http_code)) - #print('RESP: {}'.format(resp)) - # if http_code in (200, 201, 202, 204): - if resp: - resp = json.loads(resp) - if not resp or 'id' not in resp: - raise ClientException('unexpected response from server - {}'.format(resp)) - print(resp['id']) - # else: - # msg = "Error {}".format(http_code) - # if resp: - # try: - # msg = "{} - {}".format(msg, json.loads(resp)) - # except ValueError: - # msg = "{} - {}".format(msg, resp) - # raise ClientException("failed to create/update nst - {}".format(msg)) + self._client.get_token() + mime_type = magic.from_file(filename, mime=True) + if mime_type is None: + raise ClientException( + "Unexpected MIME type for file {}: MIME type {}".format( + filename, mime_type) + ) + headers= self._client._headers + if mime_type in ['application/yaml', 'text/plain']: + headers['Content-Type'] = 'application/yaml' + elif mime_type in ['application/gzip', 'application/x-gzip']: + headers['Content-Type'] = 'application/gzip' + #headers['Content-Type'] = 'application/binary' + # Next three lines are to be removed in next version + #headers['Content-Filename'] = basename(filename) + #file_size = stat(filename).st_size + #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) + else: + raise ClientException( + "Unexpected MIME type for file {}: MIME type {}".format( + filename, mime_type) + ) + headers["Content-File-MD5"] = utils.md5(filename) + http_header = ['{}: {}'.format(key,val) + for (key,val) in list(headers.items())] + self._http.set_http_header(http_header) + if update_endpoint: + http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) + else: + ow_string = '' + if overwrite: + ow_string = '?{}'.format(overwrite) + self._apiResource = '/netslice_templates_content' + self._apiBase = '{}{}{}'.format(self._apiName, + self._apiVersion, self._apiResource) + endpoint = '{}{}'.format(self._apiBase,ow_string) + http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) + #print('HTTP CODE: {}'.format(http_code)) + #print('RESP: {}'.format(resp)) + # if http_code in (200, 201, 202, 204): + if resp: + resp = json.loads(resp) + if not resp or 'id' not in resp: + raise ClientException('unexpected response from server - {}'.format(resp)) + print(resp['id']) + # else: + # msg = "Error {}".format(http_code) + # if resp: + # try: + # msg = "{} - {}".format(msg, json.loads(resp)) + # except ValueError: + # msg = "{} - {}".format(msg, resp) + # raise ClientException("failed to create/update nst - {}".format(msg)) def update(self, name, filename): self._logger.debug("") diff --git a/osmclient/sol005/package.py b/osmclient/sol005/package.py index 1ca3864..622fb86 100644 --- a/osmclient/sol005/package.py +++ b/osmclient/sol005/package.py @@ -25,6 +25,7 @@ from osmclient.common.exceptions import NotFound from osmclient.common import utils import json import logging +import os.path class Package(object): @@ -74,44 +75,49 @@ class Package(object): raise ClientException("package {} failed to upload" .format(filename)) - def upload(self, filename): + def upload(self, filename, skip_charm_build=False): self._logger.debug("") - self._client.get_token() - pkg_type = utils.get_key_val_from_pkg(filename) - if pkg_type is None: - raise ClientException("Cannot determine package type") - if pkg_type['type'] == 'nsd': - endpoint = '/nsd/v1/ns_descriptors_content' + if os.path.isdir(filename): + filename = filename.rstrip('/') + filename = self._client.package_tool.build(filename, skip_validation=False, skip_charm_build=skip_charm_build) + self.upload(filename) else: - endpoint = '/vnfpkgm/v1/vnf_packages_content' - #endpoint = '/nsds' if pkg_type['type'] == 'nsd' else '/vnfds' - #print('Endpoint: {}'.format(endpoint)) - headers = self._client._headers - headers['Content-Type'] = 'application/gzip' - #headers['Content-Type'] = 'application/binary' - # Next three lines are to be removed in next version - #headers['Content-Filename'] = basename(filename) - #file_size = stat(filename).st_size - #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) - headers["Content-File-MD5"] = utils.md5(filename) - http_header = ['{}: {}'.format(key,val) - for (key,val) in list(headers.items())] - self._http.set_http_header(http_header) - http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) - #print('HTTP CODE: {}'.format(http_code)) - #print('RESP: {}'.format(resp)) - #if http_code in (200, 201, 202, 204): - if resp: - resp = json.loads(resp) - if not resp or 'id' not in resp: - raise ClientException('unexpected response from server - {}'.format( - resp)) - print(resp['id']) - # else: - # msg = "" - # if resp: - # try: - # msg = json.loads(resp) - # except ValueError: - # msg = resp - # raise ClientException("failed to upload package - {}".format(msg)) + self._client.get_token() + pkg_type = utils.get_key_val_from_pkg(filename) + if pkg_type is None: + raise ClientException("Cannot determine package type") + if pkg_type['type'] == 'nsd': + endpoint = '/nsd/v1/ns_descriptors_content' + else: + endpoint = '/vnfpkgm/v1/vnf_packages_content' + #endpoint = '/nsds' if pkg_type['type'] == 'nsd' else '/vnfds' + #print('Endpoint: {}'.format(endpoint)) + headers = self._client._headers + headers['Content-Type'] = 'application/gzip' + #headers['Content-Type'] = 'application/binary' + # Next three lines are to be removed in next version + #headers['Content-Filename'] = basename(filename) + #file_size = stat(filename).st_size + #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) + headers["Content-File-MD5"] = utils.md5(filename) + http_header = ['{}: {}'.format(key,val) + for (key,val) in list(headers.items())] + self._http.set_http_header(http_header) + http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) + #print('HTTP CODE: {}'.format(http_code)) + #print('RESP: {}'.format(resp)) + #if http_code in (200, 201, 202, 204): + if resp: + resp = json.loads(resp) + if not resp or 'id' not in resp: + raise ClientException('unexpected response from server - {}'.format( + resp)) + print(resp['id']) + # else: + # msg = "" + # if resp: + # try: + # msg = json.loads(resp) + # except ValueError: + # msg = resp + # raise ClientException("failed to upload package - {}".format(msg)) diff --git a/osmclient/sol005/vnfd.py b/osmclient/sol005/vnfd.py index 8bf3552..cb2e90d 100644 --- a/osmclient/sol005/vnfd.py +++ b/osmclient/sol005/vnfd.py @@ -25,6 +25,7 @@ import json import magic from os.path import basename import logging +import os.path #from os import stat @@ -136,62 +137,69 @@ class Vnfd(object): # msg = resp raise ClientException("failed to delete vnfd {} - {}".format(name, msg)) - def create(self, filename, overwrite=None, update_endpoint=None): + def create(self, filename, overwrite=None, update_endpoint=None, skip_charm_build=False): self._logger.debug("") - self._client.get_token() - mime_type = magic.from_file(filename, mime=True) - if mime_type is None: - raise ClientException( - "failed to guess MIME type for file '{}'".format(filename)) - headers= self._client._headers - headers['Content-Filename'] = basename(filename) - if mime_type in ['application/yaml', 'text/plain', 'application/json']: - headers['Content-Type'] = 'text/plain' - elif mime_type in ['application/gzip', 'application/x-gzip']: - headers['Content-Type'] = 'application/gzip' - #headers['Content-Type'] = 'application/binary' - # Next three lines are to be removed in next version - #headers['Content-Filename'] = basename(filename) - #file_size = stat(filename).st_size - #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) - else: - raise ClientException( - "Unexpected MIME type for file {}: MIME type {}".format( - filename, mime_type) - ) - headers["Content-File-MD5"] = utils.md5(filename) - http_header = ['{}: {}'.format(key,val) - for (key,val) in list(headers.items())] - self._http.set_http_header(http_header) - if update_endpoint: - http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) + if os.path.isdir(filename): + filename = filename.rstrip('/') + filename = self._client.package_tool.build(filename, skip_validation=False, skip_charm_build=skip_charm_build) + self.create(filename, overwrite=overwrite, update_endpoint=update_endpoint) else: - ow_string = '' - if overwrite: - ow_string = '?{}'.format(overwrite) - self._apiResource = '/vnf_packages_content' - self._apiBase = '{}{}{}'.format(self._apiName, - self._apiVersion, self._apiResource) - endpoint = '{}{}'.format(self._apiBase,ow_string) - http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) - #print('HTTP CODE: {}'.format(http_code)) - #print('RESP: {}'.format(resp)) - if http_code in (200, 201, 202): - if resp: - resp = json.loads(resp) - if not resp or 'id' not in resp: - raise ClientException('unexpected response from server: '.format(resp)) - print(resp['id']) - elif http_code == 204: - print('Updated') - # else: - # msg = "Error {}".format(http_code) - # if resp: - # try: - # msg = "{} - {}".format(msg, json.loads(resp)) - # except ValueError: - # msg = "{} - {}".format(msg, resp) - # raise ClientException("failed to create/update vnfd - {}".format(msg)) + self._client.get_token() + mime_type = magic.from_file(filename, mime=True) + if mime_type is None: + raise ClientException( + "Unexpected MIME type for file {}: MIME type {}".format( + filename, mime_type) + ) + headers= self._client._headers + headers['Content-Filename'] = basename(filename) + if mime_type in ['application/yaml', 'text/plain', 'application/json']: + headers['Content-Type'] = 'text/plain' + elif mime_type in ['application/gzip', 'application/x-gzip']: + headers['Content-Type'] = 'application/gzip' + #headers['Content-Type'] = 'application/binary' + # Next three lines are to be removed in next version + #headers['Content-Filename'] = basename(filename) + #file_size = stat(filename).st_size + #headers['Content-Range'] = 'bytes 0-{}/{}'.format(file_size - 1, file_size) + else: + raise ClientException( + "Unexpected MIME type for file {}: MIME type {}".format( + filename, mime_type) + ) + headers["Content-File-MD5"] = utils.md5(filename) + http_header = ['{}: {}'.format(key,val) + for (key,val) in list(headers.items())] + self._http.set_http_header(http_header) + if update_endpoint: + http_code, resp = self._http.put_cmd(endpoint=update_endpoint, filename=filename) + else: + ow_string = '' + if overwrite: + ow_string = '?{}'.format(overwrite) + self._apiResource = '/vnf_packages_content' + self._apiBase = '{}{}{}'.format(self._apiName, + self._apiVersion, self._apiResource) + endpoint = '{}{}'.format(self._apiBase,ow_string) + http_code, resp = self._http.post_cmd(endpoint=endpoint, filename=filename) + #print('HTTP CODE: {}'.format(http_code)) + #print('RESP: {}'.format(resp)) + if http_code in (200, 201, 202): + if resp: + resp = json.loads(resp) + if not resp or 'id' not in resp: + raise ClientException('unexpected response from server: '.format(resp)) + print(resp['id']) + elif http_code == 204: + print('Updated') + # else: + # msg = "Error {}".format(http_code) + # if resp: + # try: + # msg = "{} - {}".format(msg, json.loads(resp)) + # except ValueError: + # msg = "{} - {}".format(msg, resp) + # raise ClientException("failed to create/update vnfd - {}".format(msg)) def update(self, name, filename): self._logger.debug("") -- 2.17.1