From: tierno Date: Mon, 12 Mar 2018 16:08:42 +0000 (+0100) Subject: sol005 packages upload implementation X-Git-Tag: v4.0.0~21 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=f27c79b67671934005fa1691158c363e2b686e77;p=osm%2FNBI.git sol005 packages upload implementation Change-Id: I7bba91831d0e29e1cd874d28a566de18eb113994 Signed-off-by: tierno --- diff --git a/.gitignore-common b/.gitignore-common index 77f6798..62bb57e 100644 --- a/.gitignore-common +++ b/.gitignore-common @@ -1,24 +1,3 @@ -## -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. -# This file is part of openmano -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: nfvlabs@tid.es -## - # This is a template with common files to be igonored, after clone make a copy to .gitignore # cp .gitignore-common .gitignore @@ -28,7 +7,7 @@ #auto-ignore .gitignore -#logs of openmano +#logs logs #pycharm @@ -41,17 +20,12 @@ logs #local stuff files that end in ".local" or folders called "local" *.local -vnfs/*.local -test/*.local -scenarios/*.local -instance-scenarios/*.local -database_utils/*.local -scripts/*.local -local -vnfs/local -test/local -scenarios/local -instance-scenarios/local -database_utils/local -scripts/local +osm_nbi/local +osm_nbi/test/local + +#local stuff files that end in ".temp" or folders called "temp" +*.temp +osm_nbi/temp +osm_nbi/test/temp + diff --git a/osm_nbi/dbmongo.py b/osm_nbi/dbmongo.py index ffab5c7..a8ea1ca 100644 --- a/osm_nbi/dbmongo.py +++ b/osm_nbi/dbmongo.py @@ -21,6 +21,7 @@ __author__ = "Alfonso Tierno " # sleep(retry) # return _retry_mongocall + class DbMongo(DbBase): conn_initial_timout = 120 conn_timout = 10 @@ -63,7 +64,7 @@ class DbMongo(DbBase): "ncont", "neq"): operator = "$" + query_k[dot_index+1:] if operator == "$neq": - operator = "$nq" + operator = "$ne" k = query_k[:dot_index] else: operator = "$eq" @@ -97,7 +98,6 @@ class DbMongo(DbBase): raise DbException("Invalid query string filter at {}:{}. Error: {}".format(query_k, v, e), http_code=HTTPStatus.BAD_REQUEST) - def get_list(self, table, filter={}): try: l = [] @@ -121,11 +121,12 @@ class DbMongo(DbBase): rows = collection.find(filter) if rows.count() == 0: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + HTTPStatus.NOT_FOUND) return None elif rows.count() > 1: if fail_on_more: - raise DbException("Found more than one entry with filter='{}'".format(filter), + raise DbException("Found more than one {} with filter='{}'".format(table[:-1], filter), HTTPStatus.CONFLICT) return rows[0] except Exception as e: # TODO refine @@ -147,7 +148,8 @@ class DbMongo(DbBase): rows = collection.delete_one(self._format_filter(filter)) if rows.deleted_count == 0: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + HTTPStatus.NOT_FOUND) return None return {"deleted": rows.deleted_count} except Exception as e: # TODO refine @@ -167,7 +169,8 @@ class DbMongo(DbBase): rows = collection.update_one(self._format_filter(filter), {"$set": update_dict}) if rows.updated_count == 0: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + HTTPStatus.NOT_FOUND) return None return {"deleted": rows.deleted_count} except Exception as e: # TODO refine @@ -179,7 +182,8 @@ class DbMongo(DbBase): rows = collection.replace_one({"_id": id}, indata) if rows.modified_count == 0: if fail_on_empty: - raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND) + raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter), + HTTPStatus.NOT_FOUND) return None return {"replace": rows.modified_count} except Exception as e: # TODO refine diff --git a/osm_nbi/engine.py b/osm_nbi/engine.py index c35617b..a15420f 100644 --- a/osm_nbi/engine.py +++ b/osm_nbi/engine.py @@ -29,6 +29,29 @@ class EngineException(Exception): Exception.__init__(self, message) +def _deep_update(dict_to_change, dict_reference): + """ + Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396 + :param dict_to_change: Ends modified + :param dict_reference: reference + :return: none + """ + + for k in dict_reference: + if dict_reference[k] is None: # None->Anything + if k in dict_to_change: + del dict_to_change[k] + elif not isinstance(dict_reference[k], dict): # NotDict->Anything + dict_to_change[k] = dict_reference[k] + elif k not in dict_to_change: # Dict->Empty + dict_to_change[k] = deepcopy(dict_reference[k]) + _deep_update(dict_to_change[k], dict_reference[k]) + elif isinstance(dict_to_change[k], dict): # Dict->Dict + _deep_update(dict_to_change[k], dict_reference[k]) + else: # Dict->NotDict + dict_to_change[k] = deepcopy(dict_reference[k]) + _deep_update(dict_to_change[k], dict_reference[k]) + class Engine(object): def __init__(self): @@ -189,7 +212,7 @@ class Engine(object): """ Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the vnfd or nsd content - :param item: can be vnfds, nsds, users, projects, + :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds :param indata: Content to be inspected :return: the useful part of indata """ @@ -214,9 +237,12 @@ class Engine(object): if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1: raise EngineException("'nsd' must be a list only one element") clean_indata = clean_indata['nsd'][0] + elif item == "userDefinedData": + if "userDefinedData" in indata: + clean_indata = clean_indata['userDefinedData'] return clean_indata - def _validate_new_data(self, session, item, indata): + def _validate_new_data(self, session, item, indata, id=None): if item == "users": if not indata.get("username"): raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY) @@ -233,8 +259,10 @@ class Engine(object): # check name not exist if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False): raise EngineException("name '{}' exist".format(indata["name"]), HTTPStatus.CONFLICT) - elif item == "vnfds" or item == "nsds": + elif item in ("vnfds", "nsds"): filter = {"id": indata["id"]} + if id: + filter["_id.neq"] = id # TODO add admin to filter, validate rights self._add_read_filter(session, item, filter) if self.db.get_one(item, filter, fail_on_empty=False): @@ -242,6 +270,10 @@ class Engine(object): HTTPStatus.CONFLICT) # TODO validate with pyangbind + elif item == "userDefinedData": + # TODO validate userDefinedData is a keypair values + pass + elif item == "nsrs": pass @@ -254,7 +286,7 @@ class Engine(object): if item == "users": _id = indata["username"] salt = uuid4().hex - indata["_admin"]["salt"] = salt + indata["_admin"]["salt"] = salt indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest() elif item == "projects": _id = indata["name"] @@ -266,36 +298,44 @@ class Engine(object): storage = admin.get("storage") if not _id: _id = str(uuid4()) - if item == "vnfds" or item == "nsds": + if item in ("vnfds", "nsds"): if not indata["_admin"].get("projects_read"): indata["_admin"]["projects_read"] = [session["project_id"]] if not indata["_admin"].get("projects_write"): indata["_admin"]["projects_write"] = [session["project_id"]] + indata["_admin"]["onboardingState"] = "CREATED" + indata["_admin"]["operationalState"] = "DISABLED" + indata["_admin"]["usageSate"] = "NOT_IN_USE" if storage: indata["_admin"]["storage"] = storage indata["_id"] = _id - def _new_item_partial(self, session, item, indata, headers): + def upload_content(self, session, item, _id, indata, kwargs, headers): """ - Used for recieve content by chunks (with a transaction_id header and/or gzip file. It will store and extract + Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract) :param session: session - :param item: + :param item: can be nsds or vnfds + :param _id : the nsd,vnfd is already created, this is the id :param indata: http body request + :param kwargs: user query string to override parameters. NOT USED :param headers: http request headers - :return: a dict with:: - _id: - storage: : where it is saving - desc: : descriptor: Only present when all the content is received, extracted and read the descriptor + :return: True package has is completely uploaded or False if partial content has been uplodaed. + Raise exception on error """ + # Check that _id exist and it is valid + current_desc = self.get_item(session, item, _id) + content_range_text = headers.get("Content-Range") - transaction_id = headers.get("Transaction-Id") - filename = headers.get("Content-Filename", "pkg") - # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266 expected_md5 = headers.get("Content-File-MD5") compressed = None - if "application/gzip" in headers.get("Content-Type") or "application/x-gzip" in headers.get("Content-Type") or \ - "application/zip" in headers.get("Content-Type"): + content_type = headers.get("Content-Type") + if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \ + "application/zip" in content_type: compressed = "gzip" + filename = headers.get("Content-Filename") + if not filename: + filename = "package.tar.gz" if compressed else "package" + # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266 file_pkg = None error_text = "" try: @@ -306,41 +346,48 @@ class Engine(object): start = int(content_range[1]) end = int(content_range[2]) + 1 total = int(content_range[3]) - if len(indata) != end-start: - raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format( - start, end-1, len(indata)), HTTPStatus.BAD_REQUEST) else: start = 0 - total = end = len(indata) - if not transaction_id: - # generate transaction - transaction_id = str(uuid4()) - self.fs.mkdir(transaction_id) - # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'wb') - # control = {"received": 0} - elif not self.fs.file_exists(transaction_id): - raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND) + + if start: + if not self.fs.file_exists(_id, 'dir'): + raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND) else: - pass - # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'rw') - # control = yaml.load(control_file) - # control_file.seek(0, 0) + self.fs.file_delete(_id, ignore_non_exist=True) + self.fs.mkdir(_id) + storage = self.fs.get_params() - storage["folder"] = transaction_id - storage["file"] = filename + storage["folder"] = _id - file_path = (transaction_id, filename) - if self.fs.file_exists(file_path): + file_path = (_id, filename) + if self.fs.file_exists(file_path, 'file'): file_size = self.fs.file_size(file_path) else: file_size = 0 if file_size != start: - raise EngineException("invalid upload transaction sequence, expected '{}' but received '{}'".format( - file_size, start), HTTPStatus.BAD_REQUEST) + raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format( + file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) file_pkg = self.fs.file_open(file_path, 'a+b') - file_pkg.write(indata) - if end != total: - return {"_id": transaction_id, "storage": storage} + if isinstance(indata, dict): + indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False) + file_pkg.write(indata_text.encode(encoding="utf-8")) + else: + indata_len = 0 + while True: + indata_text = indata.read(4096) + indata_len += len(indata_text) + if not indata_text: + break + file_pkg.write(indata_text) + if content_range_text: + if indata_len != end-start: + raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format( + start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) + if end != total: + # TODO update to UPLOADING + return False + + # PACKAGE UPLOADED if expected_md5: file_pkg.seek(0, 0) file_md5 = md5() @@ -352,8 +399,6 @@ class Engine(object): raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT) file_pkg.seek(0, 0) if compressed == "gzip": - # TODO unzip, - storage["tarfile"] = filename tar = tarfile.open(mode='r', fileobj=file_pkg) descriptor_file_name = None for tarinfo in tar: @@ -364,32 +409,43 @@ class Engine(object): if len(tarname_path) == 1 and not tarinfo.isdir(): raise EngineException("All files must be inside a dir for package descriptor tar.gz") if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"): - storage["file"] = tarname_path[0] + storage["pkg-dir"] = tarname_path[0] if len(tarname_path) == 2: if descriptor_file_name: raise EngineException("Found more than one descriptor file at package descriptor tar.gz") descriptor_file_name = tarname if not descriptor_file_name: raise EngineException("Not found any descriptor file at package descriptor tar.gz") - self.fs.file_extract(tar, transaction_id) - with self.fs.file_open((transaction_id, descriptor_file_name), "r") as descriptor_file: + storage["descriptor"] = descriptor_file_name + storage["zipfile"] = filename + self.fs.file_extract(tar, _id) + with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file: content = descriptor_file.read() else: content = file_pkg.read() - tarname = "" + storage["descriptor"] = descriptor_file_name = filename - if tarname.endswith(".json"): + if descriptor_file_name.endswith(".json"): error_text = "Invalid json format " indata = json.load(content) else: error_text = "Invalid yaml format " indata = yaml.load(content) - return {"_id": transaction_id, "storage": storage, "desc": indata} + + current_desc["_admin"]["storage"] = storage + current_desc["_admin"]["onboardingState"] = "ONBOARDED" + current_desc["_admin"]["operationalState"] = "ENABLED" + + self._edit_item(session, item, _id, current_desc, indata, kwargs) + # TODO if descriptor has changed because kwargs update content and remove cached zip + # TODO if zip is not present creates one + return True + except EngineException: raise except IndexError: raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'", - HTTPStatus.BAD_REQUEST) + HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE) except IOError as e: raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST) except (ValueError, yaml.YAMLError) as e: @@ -441,9 +497,10 @@ class Engine(object): def new_item(self, session, item, indata={}, kwargs=None, headers={}): """ - Creates a new entry into database + Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED entry, + that must be completed with a call to method upload_content :param session: contains the used login username and working project - :param item: it can be: users, projects, vnfds, nsds, ... + :param item: it can be: users, projects, nsrs, nsds, vnfds :param indata: data to be inserted :param kwargs: used to override the indata descriptor :param headers: http request headers @@ -453,16 +510,20 @@ class Engine(object): # TODO add admin and status transaction = None - if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \ - "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type"): - if not indata: - raise EngineException("Empty payload") - transaction = self._new_item_partial(session, item, indata, headers) - if "desc" not in transaction: - return transaction["_id"], False - indata = transaction["desc"] - - content = self._remove_envelop(item, indata) + # if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \ + # "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type") or \ + # "text/plain" in headers.get("Content-Type"): + # if not indata: + # raise EngineException("Empty payload") + # transaction = self._new_item_partial(session, item, indata, headers) + # if "desc" not in transaction: + # return transaction["_id"], False + # indata = transaction["desc"] + + item_envelop = item + if item in ("nsds", "vnfds"): + item_envelop = "userDefinedData" + content = self._remove_envelop(item_envelop, indata) # Override descriptor with query string kwargs if kwargs: @@ -491,7 +552,7 @@ class Engine(object): except IndexError: raise EngineException( "Invalid query string '{}'. Index '{}' out of range".format(k, kitem_old)) - if not indata: + if not indata and item not in ("nsds", "vnfds"): raise EngineException("Empty payload") if item == "nsrs": @@ -500,12 +561,14 @@ class Engine(object): content, _id = self.new_nsr(session, ns_request) transaction = {"_id": _id} - self._validate_new_data(session, item, content) + self._validate_new_data(session, item_envelop, content) + if item in ("nsds", "vnfds"): + content = {"_admin": {"userDefinedData": content}} self._format_new_data(session, item, content, transaction) _id = self.db.create(item, content) if item == "nsrs": self.msg.write("ns", "create", _id) - return _id, True + return _id def _add_read_filter(self, session, item, filter): if session["project_id"] == "admin": # allows all @@ -527,6 +590,62 @@ class Engine(object): elif item in ("vnfds", "nsds") and session["project_id"] != "admin": filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]] + def get_file(self, session, item, _id, path=None, accept_header=None): + """ + Return the file content of a vnfd or nsd + :param session: contains the used login username and working project + :param item: it can be vnfds or nsds + :param _id: Identity of the vnfd, ndsd + :param path: artifact path or "$DESCRIPTOR" or None + :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain + :return: opened file or raises an exception + """ + accept_text = accept_zip = False + if accept_header: + if 'text/plain' in accept_header or '*/*' in accept_header: + accept_text = True + if 'application/zip' in accept_header or '*/*' in accept_header: + accept_zip = True + if not accept_text and not accept_zip: + raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'", + http_code=HTTPStatus.NOT_ACCEPTABLE) + + content = self.get_item(session, item, _id) + if content["_admin"]["onboardingState"] != "ONBOARDED": + raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. " + "onboardingState is {}".format(content["_admin"]["onboardingState"]), + http_code=HTTPStatus.CONFLICT) + storage = content["_admin"]["storage"] + if path is not None and path != "$DESCRIPTOR": # artifacts + if not storage.get('pkg-dir'): + raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST) + if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'): + folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path)) + return folder_content, "text/plain" + # TODO manage folders in http + else: + return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \ + "application/octet-stream" + + # pkgtype accept ZIP TEXT -> result + # manyfiles yes X -> zip + # no yes -> error + # onefile yes no -> zip + # X yes -> text + + if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"): + return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain" + elif storage.get('pkg-dir') and not accept_zip: + raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'" + "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE) + else: + if not storage.get('zipfile'): + # TODO generate zipfile if not present + raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in future versions" + "", http_code=HTTPStatus.NOT_ACCEPTABLE) + return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip" + + def get_item_list(self, session, item, filter={}): """ Get a list of items @@ -536,6 +655,9 @@ class Engine(object): :return: The list, it can be empty if no one match the filter. """ # TODO add admin to filter, validate rights + # TODO transform data for SOL005 URL requests. Transform filtering + # TODO implement "field-type" query string SOL005 + self._add_read_filter(session, item, filter) return self.db.get_list(item, filter) @@ -543,12 +665,14 @@ class Engine(object): """ Get complete information on an items :param session: contains the used login username and working project - :param item: it can be: users, projects, vnfds, nsds, ... + :param item: it can be: users, projects, vnfds, nsds, :param _id: server id of the item :return: dictionary, raise exception if not found. """ + database_item = item filter = {"_id": _id} # TODO add admin to filter, validate rights + # TODO transform data for SOL005 URL requests self._add_read_filter(session, item, filter) return self.db.get_one(item, filter) @@ -611,27 +735,15 @@ class Engine(object): _id = self.db.create("users", indata) return _id - def edit_item(self, session, item, id, indata={}, kwargs=None): - """ - Update an existing entry at database - :param session: contains the used login username and working project - :param item: it can be: users, projects, vnfds, nsds, ... - :param id: identity of entry to be updated - :param indata: data to be inserted - :param kwargs: used to override the indata descriptor - :return: dictionary, raise exception if not found. - """ - - content = self.get_item(session, item, id) + def _edit_item(self, session, item, id, content, indata={}, kwargs=None): if indata: indata = self._remove_envelop(item, indata) - # TODO update content with with a deep-update # Override descriptor with query string kwargs if kwargs: try: for k, v in kwargs.items(): - update_content = content + update_content = indata kitem_old = None klist = k.split(".") for kitem in klist: @@ -655,9 +767,25 @@ class Engine(object): raise EngineException( "Invalid query string '{}'. Index '{}' out of range".format(k, kitem_old)) - self._validate_new_data(session, item, content) + _deep_update(content, indata) + self._validate_new_data(session, item, content, id) # self._format_new_data(session, item, content) self.db.replace(item, id, content) return id + def edit_item(self, session, item, _id, indata={}, kwargs=None): + """ + Update an existing entry at database + :param session: contains the used login username and working project + :param item: it can be: users, projects, vnfds, nsds, ... + :param _id: identifier to be updated + :param indata: data to be inserted + :param kwargs: used to override the indata descriptor + :return: dictionary, raise exception if not found. + """ + + content = self.get_item(session, item, _id) + return self._edit_item(session, item, _id, content, indata, kwargs) + + diff --git a/osm_nbi/fslocal.py b/osm_nbi/fslocal.py index 10ddf73..b7dd839 100644 --- a/osm_nbi/fslocal.py +++ b/osm_nbi/fslocal.py @@ -46,17 +46,23 @@ class FsLocal(FsBase): except Exception as e: raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR) - def file_exists(self, storage): + def file_exists(self, storage, mode=None): """ Indicates if "storage" file exist :param storage: can be a str or a str list + :param mode: can be 'file' exist as a regular file; 'dir' exists as a directory or; 'None' just exists :return: True, False """ if isinstance(storage, str): f = storage else: f = "/".join(storage) - return os.path.exists(self.path + f) + if os.path.exists(self.path + f): + if mode == "file" and os.path.isfile(self.path + f): + return True + if mode == "dir" and os.path.isdir(self.path + f): + return True + return False def file_size(self, storage): """ @@ -90,11 +96,33 @@ class FsLocal(FsBase): :param mode: file mode :return: file object """ - if isinstance(storage, str): - f = storage - else: - f = "/".join(storage) - return open(self.path + f, mode) + try: + if isinstance(storage, str): + f = storage + else: + f = "/".join(storage) + return open(self.path + f, mode) + except FileNotFoundError: + raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND) + except IOError: + raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST) + + def dir_ls(self, storage): + """ + return folder content + :param storage: can be a str or list of str + :return: folder content + """ + try: + if isinstance(storage, str): + f = storage + else: + f = "/".join(storage) + return os.listdir(self.path + f) + except NotADirectoryError: + raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND) + except IOError: + raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST) def file_delete(self, storage, ignore_non_exist=False): """ @@ -111,4 +139,4 @@ class FsLocal(FsBase): if os.path.exists(f): rmtree(f) elif not ignore_non_exist: - raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.BAD_REQUEST) + raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.NOT_FOUND) diff --git a/osm_nbi/html_out.py b/osm_nbi/html_out.py index f6f92d9..f37b5dd 100644 --- a/osm_nbi/html_out.py +++ b/osm_nbi/html_out.py @@ -19,13 +19,13 @@ html_start = """ """ @@ -61,7 +61,7 @@ html_auth2 = """

Sign in to OSM

-
+ @@ -108,12 +108,15 @@ def format(data, request, response, session): if response.status and response.status > 202: body += html_body_error.format(yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False)) elif isinstance(data, (list, tuple)): - if request.path_info == "/vnfpkgm/v1/vnf_packages": + if request.path_info == "/vnfpkgm/v1/vnf_packages_content": body += html_upload_body.format("VNFD", request.path_info) - elif request.path_info == "/nsd/v1/ns_descriptors": + elif request.path_info == "/nsd/v1/ns_descriptors_content": body += html_upload_body.format("NSD", request.path_info) for k in data: - data_id = k.pop("_id", None) + if isinstance(k, dict): + data_id = k.pop("_id", None) + elif isinstance(k, str): + data_id = k body += '

{id}: {t}

'.format(url=request.path_info, id=data_id, t=k) elif isinstance(data, dict): if "Location" in response.headers: diff --git a/osm_nbi/nbi.py b/osm_nbi/nbi.py index 9cdb409..99f90aa 100644 --- a/osm_nbi/nbi.py +++ b/osm_nbi/nbi.py @@ -9,49 +9,63 @@ import html_out as html import logging from engine import Engine, EngineException from dbbase import DbException +from fsbase import FsException from base64 import standard_b64decode -from os import getenv +#from os import getenv from http import HTTPStatus -from http.client import responses as http_responses +#from http.client import responses as http_responses from codecs import getreader from os import environ __author__ = "Alfonso Tierno " -__version__ = "0.1" -version_date = "Feb 2018" +__version__ = "0.2" +version_date = "Mar 2018" """ -North Bound Interface (O: OSM; S: SOL5 +North Bound Interface (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented) URL: /osm GET POST PUT DELETE PATCH - /nsd/v1 + /nsd/v1 O O + /ns_descriptors_content O O + / O O O O /ns_descriptors O5 O5 / O5 O5 5 /nsd_content O5 O5 + /nsd O + /artifacts[/] O /pnf_descriptors 5 5 / 5 5 5 /pnfd_content 5 5 - /subcriptions 5 5 - / 5 X + /subscriptions 5 5 + / 5 X /vnfpkgm/v1 /vnf_packages O5 O5 / O5 O5 5 - /vnfd O5 O /package_content O5 O5 /upload_from_uri X - /artifacts/ X X + /vnfd O5 + /artifacts[/] O5 + /subscriptions X X + / X X /nslcm/v1 - /ns_instances O5 O5 - / O5 O5 + /ns_instances_content O O + / O O + /ns_instances 5 5 + / 5 5 TO BE COMPLETED /ns_lcm_op_occs 5 5 / 5 5 5 TO BE COMPLETED 5 5 - /subcriptions 5 5 - / 5 X + /subscriptions 5 5 + / 5 X + /admin/v1 + /tokens O O + / O O + /users O O + / O O + /projects O O + / O O query string. [....]*[.]=[,...]&... @@ -106,6 +120,74 @@ class Server(object): def __init__(self): self.instance += 1 self.engine = Engine() + self.valid_methods = { # contains allowed URL and methods + "admin": { + "v1": { + "tokens": { "METHODS": ("GET", "POST", "DELETE"), + "": { "METHODS": ("GET", "DELETE")} + }, + "users": { "METHODS": ("GET", "POST"), + "": {"METHODS": ("GET", "POST", "DELETE")} + }, + "projects": { "METHODS": ("GET", "POST"), + "": {"METHODS": ("GET", "POST", "DELETE")} + }, + } + }, + "nsd": { + "v1": { + "ns_descriptors_content": { "METHODS": ("GET", "POST"), + "": {"METHODS": ("GET", "PUT", "DELETE")} + }, + "ns_descriptors": { "METHODS": ("GET", "POST"), + "": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH", + "nsd_content": { "METHODS": ("GET", "PUT")}, + "nsd": {"METHODS": "GET"}, # descriptor inside package + "artifacts": {"*": {"METHODS": "GET"}} + } + + }, + "pnf_descriptors": {"TODO": ("GET", "POST"), + "": {"TODO": ("GET", "DELETE", "PATCH"), + "pnfd_content": {"TODO": ("GET", "PUT")} + } + }, + "subscriptions": {"TODO": ("GET", "POST"), + "": {"TODO": ("GET", "DELETE"),} + }, + } + }, + "vnfpkgm": { + "v1": { + "vnf_packages_content": { "METHODS": ("GET", "POST"), + "": {"METHODS": ("GET", "PUT", "DELETE")} + }, + "vnf_packages": { "METHODS": ("GET", "POST"), + "": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH", # GET: vnfPkgInfo + "package_content": { "METHODS": ("GET", "PUT"), # package + "upload_from_uri": {"TODO": "POST"} + }, + "vnfd": {"METHODS": "GET"}, # descriptor inside package + "artifacts": {"*": {"METHODS": "GET"}} + } + + }, + "subscriptions": {"TODO": ("GET", "POST"), + "": {"TODO": ("GET", "DELETE"),} + }, + } + }, + "nslcm": { + "v1": { + "ns_instances_content": {"METHODS": ("GET", "POST"), + "": {"METHODS": ("GET", "DELETE")} + }, + "ns_instances": {"TODO": ("GET", "POST"), + "": {"TODO": ("GET", "DELETE")} + } + } + }, + } def _authorization(self): token = None @@ -164,14 +246,15 @@ class Server(object): indata = yaml.load(cherrypy.request.body) elif "application/binary" in cherrypy.request.headers["Content-Type"] or \ "application/gzip" in cherrypy.request.headers["Content-Type"] or \ - "application/zip" in cherrypy.request.headers["Content-Type"]: - indata = cherrypy.request.body.read() + "application/zip" in cherrypy.request.headers["Content-Type"] or \ + "text/plain" in cherrypy.request.headers["Content-Type"]: + indata = cherrypy.request.body # .read() elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]: if "descriptor_file" in kwargs: filecontent = kwargs.pop("descriptor_file") if not filecontent.file: raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST) - indata = filecontent.file.read() + indata = filecontent.file # .read() if filecontent.content_type.value: cherrypy.request.headers["Content-Type"] = filecontent.content_type.value else: @@ -186,10 +269,6 @@ class Server(object): if not indata: indata = {} - if "METHOD" in kwargs: - method = kwargs.pop("METHOD") - else: - method = cherrypy.request.method format_yaml = False if cherrypy.request.headers.get("Query-String-Format") == "yaml": format_yaml = True @@ -223,20 +302,33 @@ class Server(object): except: pass - return indata, method + return indata except (ValueError, yaml.YAMLError) as exc: raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST) except KeyError as exc: raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST) @staticmethod - def _format_out(data, session=None): + def _format_out(data, session=None, _format=None): """ return string of dictionary data according to requested json, yaml, xml. By default json - :param data: response to be sent. Can be a dict or text + :param data: response to be sent. Can be a dict, text or file :param session: + :param _format: The format to be set as Content-Type ir data is a file :return: None """ + if data is None: + cherrypy.response.status = HTTPStatus.NO_CONTENT.value + return + elif hasattr(data, "read"): # file object + if _format: + cherrypy.response.headers["Content-Type"] = _format + elif "b" in data.mode: # binariy asssumig zip + cherrypy.response.headers["Content-Type"] = 'application/zip' + else: + cherrypy.response.headers["Content-Type"] = 'text/plain' + # TODO check that cherrypy close file. If not implement pending things to close per thread next + return data if "Accept" in cherrypy.request.headers: accept = cherrypy.request.headers["Accept"] if "application/json" in accept: @@ -246,7 +338,7 @@ class Server(object): elif "text/html" in accept: return html.format(data, cherrypy.request, cherrypy.response, session) - elif "application/yaml" in accept or "*/*" in accept: + elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept: pass else: raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value, @@ -275,20 +367,17 @@ class Server(object): return self._format_out("Welcome to OSM!", session) @cherrypy.expose - def token(self, *args, **kwargs): - if not args: - raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED) - version = args[0] - if version != 'v1': - raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED) + def token(self, method, token_id=None, kwargs=None): session = None # self.engine.load_dbase(cherrypy.request.app.config) + indata = self._format_in(kwargs) + if not isinstance(indata, dict): + raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST) try: - indata, method = self._format_in(kwargs) if method == "GET": session = self._authorization() - if len(args) >= 2: - outdata = self.engine.get_token(session, args[1]) + if token_id: + outdata = self.engine.get_token(session, token_id) else: outdata = self.engine.get_token_list(session) elif method == "POST": @@ -304,11 +393,9 @@ class Server(object): # cherrypy.response.cookie["Authorization"] = outdata["id"] # cherrypy.response.cookie["Authorization"]['expires'] = 3600 elif method == "DELETE": - if len(args) >= 2 and "logout" not in args: - token_id = args[1] - elif "id" in kwargs: + if not token_id and "id" in kwargs: token_id = kwargs["id"] - else: + elif not token_id: session = self._authorization() token_id = session["_id"] outdata = self.engine.del_token(token_id) @@ -329,10 +416,21 @@ class Server(object): } return self._format_out(problem_details, session) + @cherrypy.expose + def test2(self, args0=None, args1=None, args2=None, args3=None, *args, **kwargs): + return_text = ( + "
\n{} {} {} {} {} {} \n".format(args0, args1, args2, args3, args, kwargs))
+        return_text += "
" + return return_text + @cherrypy.expose def test(self, *args, **kwargs): thread_info = None - if args and args[0] == "init": + if args and args[0] == "help": + return "
\ninit\nfile/  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
+                    "sleep/
" + + elif args and args[0] == "init": try: # self.engine.load_dbase(cherrypy.request.app.config) self.engine.create_admin() @@ -340,6 +438,17 @@ class Server(object): except Exception: cherrypy.response.status = HTTPStatus.FORBIDDEN.value return self._format_out("Database already initialized") + elif args and args[0] == "file": + return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1], + "text/plain", "attachment") + elif args and args[0] == "file2": + f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1] + f = open(f_path, "r") + cherrypy.response.headers["Content-type"] = "text/plain" + + return f + elif len(args) == 2 and args[0] == "db-clear": + return self.engine.del_item_list({"project_id": "admin"}, args[1], {}) elif args and args[0] == "prune": return self.engine.prune() elif args and args[0] == "login": @@ -390,83 +499,148 @@ class Server(object): return_text += "" return return_text + def _check_valid_url_method(self, method, *args): + if len(args) < 3: + raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED) + + reference = self.valid_methods + for arg in args: + if arg is None: + break + if not isinstance(reference, dict): + raise NbiException("URL contains unexpected extra items '{}'".format(arg), + HTTPStatus.METHOD_NOT_ALLOWED) + + if arg in reference: + reference = reference[arg] + elif "" in reference: + reference = reference[""] + elif "*" in reference: + reference = reference["*"] + break + else: + raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED) + if "TODO" in reference and method in reference["TODO"]: + raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED) + elif "METHODS" in reference and not method in reference["METHODS"]: + raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED) + return + + @staticmethod + def _set_location_header(topic, version, item, id): + """ + Insert response header Location with the URL of created item base on URL params + :param topic: + :param version: + :param item: + :param id: + :return: None + """ + # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT + cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(topic, version, item, id) + return + @cherrypy.expose - def default(self, *args, **kwargs): + def default(self, topic=None, version=None, item=None, _id=None, item2=None, *args, **kwargs): session = None + outdata = None + _format = None try: - if not args or len(args) < 2: - raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED) - item = args[0] - version = args[1] - if item not in ("token", "user", "project", "vnfpkgm", "nsd", "nslcm"): - raise NbiException("URL item '{}' not supported".format(item), HTTPStatus.METHOD_NOT_ALLOWED) + if not topic or not version or not item: + raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED) + if topic not in ("admin", "vnfpkgm", "nsd", "nslcm"): + raise NbiException("URL topic '{}' not supported".format(topic), HTTPStatus.METHOD_NOT_ALLOWED) if version != 'v1': raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED) + if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"): + method = kwargs.pop("METHOD") + else: + method = cherrypy.request.method + + self._check_valid_url_method(method, topic, version, item, _id, item2, *args) + + if topic == "admin" and item == "tokens": + return self.token(method, _id, kwargs) + # self.engine.load_dbase(cherrypy.request.app.config) session = self._authorization() - indata, method = self._format_in(kwargs) - _id = None - - if item == "nsd": - item = "nsds" - if len(args) < 3 or args[2] != "ns_descriptors": - raise NbiException("only ns_descriptors is allowed", HTTPStatus.METHOD_NOT_ALLOWED) - if len(args) > 3: - _id = args[3] - if len(args) > 4 and args[4] != "nsd_content": - raise NbiException("only nsd_content is allowed", HTTPStatus.METHOD_NOT_ALLOWED) - elif item == "vnfpkgm": - item = "vnfds" - if len(args) < 3 or args[2] != "vnf_packages": - raise NbiException("only vnf_packages is allowed", HTTPStatus.METHOD_NOT_ALLOWED) - if len(args) > 3: - _id = args[3] - if len(args) > 4 and args[4] not in ("vnfd", "package_content"): - raise NbiException("only vnfd or package_content are allowed", HTTPStatus.METHOD_NOT_ALLOWED) - elif item == "nslcm": - item = "nsrs" - if len(args) < 3 or args[2] != "ns_instances": - raise NbiException("only ns_instances is allowed", HTTPStatus.METHOD_NOT_ALLOWED) - if len(args) > 3: - _id = args[3] - if len(args) > 4: - raise NbiException("This feature is not implemented", HTTPStatus.METHOD_NOT_ALLOWED) - else: - if len(args) >= 3: - _id = args[2] - item += "s" + indata = self._format_in(kwargs) + engine_item = item + if item == "subscriptions": + engine_item = topic + "_" + item + if item2: + engine_item = item2 + + if topic == "nsd": + engine_item = "nsds" + elif topic == "vnfpkgm": + engine_item = "vnfds" + elif topic == "nslcm": + engine_item = "nsrs" if method == "GET": - if not _id: - outdata = self.engine.get_item_list(session, item, kwargs) - else: # len(args) > 1 - outdata = self.engine.get_item(session, item, _id) + if item2 in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd"): + if item2 in ("vnfd", "nsd"): + path = "$DESCRIPTOR" + elif args: + path = args + elif item2 == "artifacts": + path = () + else: + path = None + file, _format = self.engine.get_file(session, engine_item, _id, path, + cherrypy.request.headers.get("Accept")) + outdata = file + elif not _id: + outdata = self.engine.get_item_list(session, engine_item, kwargs) + else: + outdata = self.engine.get_item(session, engine_item, _id) elif method == "POST": - id, completed = self.engine.new_item(session, item, indata, kwargs, cherrypy.request.headers) - if not completed: - cherrypy.response.headers["Transaction-Id"] = id - cherrypy.response.status = HTTPStatus.CREATED.value + if item in ("ns_descriptors_content", "vnf_packages_content"): + _id = cherrypy.request.headers.get("Transaction-Id") + if not _id: + _id = self.engine.new_item(session, engine_item, {}, None, cherrypy.request.headers) + completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers) + if completed: + self._set_location_header(topic, version, item, _id) + else: + cherrypy.response.headers["Transaction-Id"] = _id + outdata = {"id": _id} + elif item in ("ns_descriptors", "vnf_packages"): + _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers) + self._set_location_header(topic, version, item, _id) + #TODO form NsdInfo + outdata = {"id": _id} else: - cherrypy.response.headers["Location"] = cherrypy.request.base + "/osm/" + "/".join(args[0:3]) + "/" + id - outdata = {"id": id} + _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers) + self._set_location_header(topic, version, item, _id) + outdata = {"id": _id} + cherrypy.response.status = HTTPStatus.CREATED.value elif method == "DELETE": if not _id: - outdata = self.engine.del_item_list(session, item, kwargs) + outdata = self.engine.del_item_list(session, engine_item, kwargs) else: # len(args) > 1 - outdata = self.engine.del_item(session, item, _id) + outdata = self.engine.del_item(session, engine_item, _id) + if item in ("ns_descriptors", "vnf_packages"): # SOL005 + outdata = None elif method == "PUT": - if not _id: - raise NbiException("Missing '/' at the URL to identify item to be updated", - HTTPStatus.METHOD_NOT_ALLOWED) - elif not indata and not kwargs: + if not indata and not kwargs: raise NbiException("Nothing to update. Provide payload and/or query string", HTTPStatus.BAD_REQUEST) - outdata = {"id": self.engine.edit_item(session, item, args[1], indata, kwargs)} + if item2 in ("nsd_content", "package_content"): + completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers) + if not completed: + cherrypy.response.headers["Transaction-Id"] = id + outdata = None + else: + outdata = {"id": self.engine.edit_item(session, engine_item, args[1], indata, kwargs)} else: raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED) - return self._format_out(outdata, session) - except (NbiException, EngineException, DbException) as e: + return self._format_out(outdata, session, _format) + except (NbiException, EngineException, DbException, FsException) as e: + if hasattr(outdata, "close"): # is an open file + outdata.close() cherrypy.log("Exception {}".format(e)) cherrypy.response.status = e.http_code.value problem_details = { diff --git a/osm_nbi/test/cirros_ns/cirros_nsd.yaml b/osm_nbi/test/cirros_ns/cirros_nsd.yaml new file mode 100644 index 0000000..5c4e214 --- /dev/null +++ b/osm_nbi/test/cirros_ns/cirros_nsd.yaml @@ -0,0 +1,49 @@ +nsd-catalog: + nsd: + - id: cirros_nsd + name: cirros_ns + short-name: cirros_ns + description: Generated by OSM pacakage generator + vendor: OSM + version: '1.0' + + # Place the logo as png in icons directory and provide the name here + logo: osm_2x.png + + # Specify the VNFDs that are part of this NSD + constituent-vnfd: + # The member-vnf-index needs to be unique, starting from 1 + # vnfd-id-ref is the id of the VNFD + # Multiple constituent VNFDs can be specified + - member-vnf-index: 1 + vnfd-id-ref: cirros_vnfd + scaling-group-descriptor: + - name: "scaling_cirros" + vnfd-member: + - count: 1 + member-vnf-index-ref: 1 + min-instance-count: 0 + max-instance-count: 10 + scaling-policy: + - scaling-type: "manual" + cooldown-time: 10 + threshold-time: 10 + name: manual_scale + vld: + # Networks for the VNFs + - id: cirros_nsd_vld1 + name: cirros_nsd_vld1 + type: ELAN + mgmt-network: 'true' + # vim-network-name: + # provider-network: + # segmentation_id: + vnfd-connection-point-ref: + # Specify the constituent VNFs + # member-vnf-index-ref - entry from constituent vnf + # vnfd-id-ref - VNFD id + # vnfd-connection-point-ref - connection point name in the VNFD + - member-vnf-index-ref: 1 + vnfd-id-ref: cirros_vnfd + # NOTE: Validate the entry below + vnfd-connection-point-ref: eth0 diff --git a/osm_nbi/test/cirros_ns/icons/osm_2x.png b/osm_nbi/test/cirros_ns/icons/osm_2x.png new file mode 100644 index 0000000..62012d2 Binary files /dev/null and b/osm_nbi/test/cirros_ns/icons/osm_2x.png differ diff --git a/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml b/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml new file mode 100644 index 0000000..94fa5f1 --- /dev/null +++ b/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml @@ -0,0 +1,48 @@ +vnfd-catalog: + vnfd: + - id: cirros_vnfd + name: cirros_vnf + short-name: cirros_vnf + description: Simple VNF example with a cirros + vendor: OSM + version: '1.0' + + # Place the logo as png in icons directory and provide the name here + logo: cirros-64.png + + # Management interface + mgmt-interface: + cp: eth0 + + # Atleast one VDU need to be specified + vdu: + - id: cirros_vnfd-VM + name: cirros_vnfd-VM + description: cirros_vnfd-VM + count: 1 + + # Flavour of the VM to be instantiated for the VDU + # flavor below can fit into m1.micro + vm-flavor: + vcpu-count: 1 + memory-mb: 256 + storage-gb: 2 + + # Image/checksum or image including the full path + image: 'cirros034' + #checksum: + + interface: + # Specify the external interfaces + # There can be multiple interfaces defined + - name: eth0 + type: EXTERNAL + virtual-interface: + type: VIRTIO + bandwidth: '0' + vpci: 0000:00:0a.0 + external-connection-point-ref: eth0 + + connection-point: + - name: eth0 + type: VPORT diff --git a/osm_nbi/test/cirros_vnf/icons/cirros-64.png b/osm_nbi/test/cirros_vnf/icons/cirros-64.png new file mode 100644 index 0000000..5725d29 Binary files /dev/null and b/osm_nbi/test/cirros_vnf/icons/cirros-64.png differ diff --git a/osm_nbi/test/create-ping-pong.sh b/osm_nbi/test/create-ping-pong.sh index 4d96d47..7ef1cf7 100755 --- a/osm_nbi/test/create-ping-pong.sh +++ b/osm_nbi/test/create-ping-pong.sh @@ -25,19 +25,23 @@ NSD3=${DESCRIPTORS}/cirros_nsd.yaml [ -f "$NSD3" ] || ! echo "not found cirros_nsd.yaml. Set DESCRIPTORS variable to a proper location" || exit 1 #get token -TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}'`; +TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" \ + --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens \ + 2>/dev/null | awk '($1=="id:"){print $2}'`; echo token: $TOKEN - - # VNFD ######### #insert PKG -VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN" --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'` +VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \ + -H "Authorization: Bearer $TOKEN" --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \ + 2>/dev/null | awk '($1=="id:"){print $2}'` echo ping_vnfd: $VNFD1_ID -VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN" --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'` +VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \ + -H "Authorization: Bearer $TOKEN" --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \ + 2>/dev/null | awk '($1=="id:"){print $2}'` echo pong_vnfd: $VNFD2_ID @@ -45,19 +49,25 @@ echo pong_vnfd: $VNFD2_ID # NSD ######### #insert PKG -NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN" --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors 2>/dev/null | awk '($1=="id:"){print $2}'` +NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \ + -H "Authorization: Bearer $TOKEN" --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors_content \ + 2>/dev/null | awk '($1=="id:"){print $2}'` echo ping_pong_nsd: $NSD1_ID # NSRS ############## #add nsr -NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN" --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }" ${NBI_URL}/nslcm/v1/ns_instances 2>/dev/null | awk '($1=="id:"){print $2}'` ; +NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" \ + -H "Authorization: Bearer $TOKEN" --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, \ + ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }" \ + ${NBI_URL}/nslcm/v1/ns_instances_content 2>/dev/null | awk '($1=="id:"){print $2}'` ; echo ping_pong_nsr: $NSR1_ID -echo ' -curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer '$TOKEN'" '${NBI_URL}'/nslcm/v1/ns_instances/'$NSR1_ID' 2>/dev/null | grep -e detailed-status -e operational-status -e config-status' +echo 'curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml"' \ + '-H "Authorization: Bearer '$TOKEN'" '${NBI_URL}'/nslcm/v1/ns_instances_content/'$NSR1_ID' 2>/dev/null | ' \ + 'grep -e detailed-status -e operational-status -e config-status' diff --git a/osm_nbi/test/delete-all.sh b/osm_nbi/test/delete-all.sh index 6b04538..d8cb474 100755 --- a/osm_nbi/test/delete-all.sh +++ b/osm_nbi/test/delete-all.sh @@ -8,13 +8,13 @@ PROJECT=admin #get token -TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN +TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN echo deleting all #DELETE ALL -for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors vnfpkgm/v1/vnf_packages +for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors_content vnfpkgm/v1/vnf_packages_content do for ITEM_ID in `curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN" ${NBI_URL}/${url_item} 2>/dev/null | awk '($1=="_id:") {print $2}'` ; do diff --git a/osm_nbi/test/test.py b/osm_nbi/test/test.py new file mode 100755 index 0000000..734d667 --- /dev/null +++ b/osm_nbi/test/test.py @@ -0,0 +1,337 @@ +#! /usr/bin/python3 +# -*- coding: utf-8 -*- + +import getopt +import sys +import requests +#import base64 +#from os.path import getsize, basename +#from hashlib import md5 +import json +import logging +import yaml +#import json +import tarfile +from os import makedirs +from copy import deepcopy + +__author__ = "Alfonso Tierno, alfonso.tiernosepulveda@telefonica.com" +__date__ = "$2018-03-01$" +__version__ = "0.1" +version_date = "Mar 2018" + + +def usage(): + print("Usage: ", sys.argv[0], "[options]") + print(" --version: prints current version") + print(" -f|--file FILE: file to be sent") + print(" -h|--help: shows this help") + print(" -u|--url URL: complete server URL") + print(" -s|--chunk-size SIZE: size of chunks, by default 1000") + print(" -t|--token TOKEN: Authorizaton token, previously obtained from server") + print(" -v|--verbose print debug information, can be used several times") + return + + +r_header_json = {"Content-type": "application/json"} +headers_json = { + "Content-type": "application/json", + "Accept": "application/json", +} +r_header_yaml = {"Content-type": "application/yaml"} +headers_yaml = { + "Content-type": "application/yaml", + "Accept": "application/yaml", +} +r_header_text = {"Content-type": "text/plain"} +r_header_octect = {"Content-type": "application/octet-stream"} +headers_text = { + "Accept": "text/plain", +} +r_header_zip = {"Content-type": "application/zip"} +headers_zip = { + "Accept": "application/zip", +} +# test without authorization +test_not_authorized_list = ( + ("Invalid token", "GET", "/admin/v1/users", headers_json, None, 401, r_header_json, "json"), + ("Invalid URL", "POST", "/admin/v1/nonexist", headers_yaml, None, 405, r_header_yaml, "yaml"), + ("Invalid version", "DELETE", "/admin/v2/users", headers_yaml, None, 405, r_header_yaml, "yaml"), +) + +# test ones authorized +test_authorized_list = ( + ("Invalid vnfd id", "GET", "/vnfpkgm/v1/vnf_packages/non-existing-id", headers_json, None, 404, r_header_json, "json"), + ("Invalid nsd id", "GET", "/nsd/v1/ns_descriptors/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"), + ("Invalid nsd id", "DELETE", "/nsd/v1/ns_descriptors_content/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"), +) + +class TestException(Exception): + pass + + +class TestRest: + def __init__(self, url_base, header_base={}, verify=False): + self.url_base = url_base + self.header_base = header_base + self.s = requests.session() + self.s.headers = header_base + self.verify = verify + + def set_header(self, header): + self.s.headers.update(header) + + def test(self, name, method, url, headers, payload, expected_codes, expected_headers, expected_payload): + """ + Performs an http request and check http code response. Exit if different than allowed + :param name: name of the test + :param method: HTTP method: GET,PUT,POST,DELETE,... + :param url: complete URL or relative URL + :param headers: request headers to add to the base headers + :param payload: Can be a dict, transformed to json, a text or a file if starts with '@' + :param expected_codes: expected response codes, can be int, int tuple or int range + :param expected_headers: expected response headers, dict with key values + :param expected_payload: expected payload, 0 if empty, 'yaml', 'json', 'text', 'zip' + :return: + """ + try: + if not self.s: + self.s = requests.session() + if not url: + url = self.url_base + elif not url.startswith("http"): + url = self.url_base + url + if payload: + if isinstance(payload, str): + if payload.startswith("@"): + mode = "r" + file_name = payload[1:] + if payload.startswith("@b"): + mode = "rb" + file_name = payload[2:] + with open(file_name, mode) as f: + payload = f.read() + elif isinstance(payload, dict): + payload = json.dumps(payload) + + test = "Test {} {} {}".format(name, method, url) + logger.warning(test) + stream = False + # if expected_payload == "zip": + # stream = True + r = getattr(self.s, method.lower())(url, data=payload, headers=headers, verify=self.verify, stream=stream) + logger.debug("RX {}: {}".format(r.status_code, r.text)) + + # check response + if expected_codes: + if isinstance(expected_codes, int): + expected_codes = (expected_codes,) + if r.status_code not in expected_codes: + raise TestException( + "Got status {}. Expected {}. {}".format(r.status_code, expected_codes, r.text)) + + if expected_headers: + for header_key, header_val in expected_headers.items(): + if header_key.lower() not in r.headers: + raise TestException("Header {} not present".format(header_key)) + if header_val and header_val.lower() not in r.headers[header_key]: + raise TestException("Header {} does not contain {} but {}".format(header_key, header_val, + r.headers[header_key])) + + if expected_payload is not None: + if expected_payload == 0 and len(r.content) > 0: + raise TestException("Expected empty payload") + elif expected_payload == "json": + try: + r.json() + except Exception as e: + raise TestException("Expected json response payload, but got Exception {}".format(e)) + elif expected_payload == "yaml": + try: + yaml.safe_load(r.text) + except Exception as e: + raise TestException("Expected yaml response payload, but got Exception {}".format(e)) + elif expected_payload == "zip": + if len(r.content) == 0: + raise TestException("Expected some response payload, but got empty") + # try: + # tar = tarfile.open(None, 'r:gz', fileobj=r.raw) + # for tarinfo in tar: + # tarname = tarinfo.name + # print(tarname) + # except Exception as e: + # raise TestException("Expected zip response payload, but got Exception {}".format(e)) + elif expected_payload == "text": + if len(r.content) == 0: + raise TestException("Expected some response payload, but got empty") + #r.text + return r + except TestException as e: + logger.error("{} \nRX code{}: {}".format(e, r.status_code, r.text)) + exit(1) + except IOError as e: + logger.error("Cannot open file {}".format(e)) + exit(1) + + +if __name__ == "__main__": + global logger + test = "" + try: + logging.basicConfig(format="%(levelname)s %(message)s", level=logging.ERROR) + logger = logging.getLogger('NBI') + # load parameters and configuration + opts, args = getopt.getopt(sys.argv[1:], "hvu:p:", + ["url=", "user=", "password=", "help", "version", "verbose", "project=", "insecure"]) + url = "https://localhost:9999/osm" + user = password = project = "admin" + verbose = 0 + verify = True + + for o, a in opts: + if o == "--version": + print ("test version " + __version__ + ' ' + version_date) + sys.exit() + elif o in ("-v", "--verbose"): + verbose += 1 + elif o in ("no-verbose"): + verbose = -1 + elif o in ("-h", "--help"): + usage() + sys.exit() + elif o in ("--url"): + url = a + elif o in ("-u", "--user"): + user = a + elif o in ("-p", "--password"): + password = a + elif o in ("--project"): + project = a + elif o in ("--insecure"): + verify = False + else: + assert False, "Unhandled option" + if verbose == 0: + logger.setLevel(logging.WARNING) + elif verbose > 1: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.ERROR) + + test_rest = TestRest(url) + + # tests without authorization + for t in test_not_authorized_list: + test_rest.test(*t) + + # get token + r = test_rest.test("Obtain token", "POST", "/admin/v1/tokens", headers_json, + {"username": user, "password": password, "project_id": project}, + (200, 201), {"Content-Type": "application/json"}, "json") + response = r.json() + token = response["id"] + test_rest.set_header({"Authorization": "Bearer {}".format(token)}) + + # tests once authorized + for t in test_authorized_list: + test_rest.test(*t) + + # nsd CREATE + r = test_rest.test("Onboard NSD step 1", "POST", "/nsd/v1/ns_descriptors", headers_json, None, + 201, {"Location": "/nsd/v1/ns_descriptors/", "Content-Type": "application/json"}, "json") + location = r.headers["Location"] + nsd_id = location[location.rfind("/")+1:] + # print(location, nsd_id) + + # nsd UPLOAD test + r = test_rest.test("Onboard NSD step 2 as TEXT", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id), + r_header_text, "@./cirros_ns/cirros_nsd.yaml", 204, None, 0) + + # nsd SHOW OSM format + r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id), + headers_json, None, 200, r_header_json, "json") + + # nsd SHOW text + r = test_rest.test("Show NSD SOL005 text", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id), + headers_text, None, 200, r_header_text, "text") + + # nsd UPLOAD ZIP + makedirs("temp", exist_ok=True) + tar = tarfile.open("temp/cirros_ns.tar.gz", "w:gz") + tar.add("cirros_ns") + tar.close() + r = test_rest.test("Onboard NSD step 3 replace with ZIP", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id), + r_header_zip, "@b./temp/cirros_ns.tar.gz", 204, None, 0) + + # nsd SHOW OSM format + r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id), + headers_json, None, 200, r_header_json, "json") + + # nsd SHOW zip + r = test_rest.test("Show NSD SOL005 zip", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id), + headers_zip, None, 200, r_header_zip, "zip") + + # nsd SHOW descriptor + r = test_rest.test("Show NSD descriptor", "GET", "/nsd/v1/ns_descriptors/{}/nsd".format(nsd_id), + headers_text, None, 200, r_header_text, "text") + # nsd SHOW actifact + r = test_rest.test("Show NSD artifact", "GET", "/nsd/v1/ns_descriptors/{}/artifacts/icons/osm_2x.png".format(nsd_id), + headers_text, None, 200, r_header_octect, "text") + + # nsd DELETE + r = test_rest.test("Delete NSD SOL005 text", "DELETE", "/nsd/v1/ns_descriptors/{}".format(nsd_id), + headers_yaml, None, 204, None, 0) + + # vnfd CREATE + r = test_rest.test("Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages", headers_json, None, + 201, {"Location": "/vnfpkgm/v1/vnf_packages/", "Content-Type": "application/json"}, "json") + location = r.headers["Location"] + vnfd_id = location[location.rfind("/")+1:] + # print(location, vnfd_id) + + # vnfd UPLOAD test + r = test_rest.test("Onboard VNFD step 2 as TEXT", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id), + r_header_text, "@./cirros_vnf/cirros_vnfd.yaml", 204, None, 0) + + # vnfd SHOW OSM format + r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id), + headers_json, None, 200, r_header_json, "json") + + # vnfd SHOW text + r = test_rest.test("Show VNFD SOL005 text", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id), + headers_text, None, 200, r_header_text, "text") + + # vnfd UPLOAD ZIP + makedirs("temp", exist_ok=True) + tar = tarfile.open("temp/cirros_vnf.tar.gz", "w:gz") + tar.add("cirros_vnf") + tar.close() + r = test_rest.test("Onboard VNFD step 3 replace with ZIP", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id), + r_header_zip, "@b./temp/cirros_vnf.tar.gz", 204, None, 0) + + # vnfd SHOW OSM format + r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id), + headers_json, None, 200, r_header_json, "json") + + # vnfd SHOW zip + r = test_rest.test("Show VNFD SOL005 zip", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id), + headers_zip, None, 200, r_header_zip, "zip") + # vnfd SHOW descriptor + r = test_rest.test("Show VNFD descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(vnfd_id), + headers_text, None, 200, r_header_text, "text") + # vnfd SHOW actifact + r = test_rest.test("Show VNFD artifact", "GET", "/vnfpkgm/v1/vnf_packages/{}/artifacts/icons/cirros-64.png".format(vnfd_id), + headers_text, None, 200, r_header_octect, "text") + + # vnfd DELETE + r = test_rest.test("Delete VNFD SOL005 text", "DELETE", "/vnfpkgm/v1/vnf_packages/{}".format(vnfd_id), + headers_yaml, None, 204, None, 0) + + print("PASS") + + except Exception as e: + if test: + logger.error(test + " Exception: " + str(e)) + exit(1) + else: + logger.critical(test + " Exception: " + str(e), exc_info=True)
Username
Password