sol005 packages upload implementation 75/5875/1
authortierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 12 Mar 2018 16:08:42 +0000 (17:08 +0100)
committertierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 12 Mar 2018 16:08:42 +0000 (17:08 +0100)
Change-Id: I7bba91831d0e29e1cd874d28a566de18eb113994
Signed-off-by: tierno <alfonso.tiernosepulveda@telefonica.com>
13 files changed:
.gitignore-common
osm_nbi/dbmongo.py
osm_nbi/engine.py
osm_nbi/fslocal.py
osm_nbi/html_out.py
osm_nbi/nbi.py
osm_nbi/test/cirros_ns/cirros_nsd.yaml [new file with mode: 0644]
osm_nbi/test/cirros_ns/icons/osm_2x.png [new file with mode: 0644]
osm_nbi/test/cirros_vnf/cirros_vnfd.yaml [new file with mode: 0644]
osm_nbi/test/cirros_vnf/icons/cirros-64.png [new file with mode: 0644]
osm_nbi/test/create-ping-pong.sh
osm_nbi/test/delete-all.sh
osm_nbi/test/test.py [new file with mode: 0755]

index 77f6798..62bb57e 100644 (file)
@@ -1,24 +1,3 @@
-##
-# Copyright 2015 Telef├│nica Investigaci├│n y Desarrollo, S.A.U.
-# This file is part of openmano
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact with: nfvlabs@tid.es
-##
-
 # This is a template with common files to be igonored, after clone make a copy to .gitignore
 # cp .gitignore-common .gitignore
 
@@ -28,7 +7,7 @@
 #auto-ignore
 .gitignore
 
-#logs of openmano
+#logs
 logs 
 
 #pycharm
@@ -41,17 +20,12 @@ logs
 
 #local stuff files that end in ".local" or folders called "local"
 *.local
-vnfs/*.local
-test/*.local
-scenarios/*.local
-instance-scenarios/*.local
-database_utils/*.local
-scripts/*.local
-local
-vnfs/local
-test/local
-scenarios/local
-instance-scenarios/local
-database_utils/local
-scripts/local
+osm_nbi/local
+osm_nbi/test/local
+
+#local stuff files that end in ".temp" or folders called "temp"
+*.temp
+osm_nbi/temp
+osm_nbi/test/temp
+
 
index ffab5c7..a8ea1ca 100644 (file)
@@ -21,6 +21,7 @@ __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 #                 sleep(retry)
 #     return _retry_mongocall
 
+
 class DbMongo(DbBase):
     conn_initial_timout = 120
     conn_timout = 10
@@ -63,7 +64,7 @@ class DbMongo(DbBase):
                                                                "ncont", "neq"):
                     operator = "$" + query_k[dot_index+1:]
                     if operator == "$neq":
-                        operator = "$nq"
+                        operator = "$ne"
                     k = query_k[:dot_index]
                 else:
                     operator = "$eq"
@@ -97,7 +98,6 @@ class DbMongo(DbBase):
             raise DbException("Invalid query string filter at {}:{}. Error: {}".format(query_k, v, e),
                               http_code=HTTPStatus.BAD_REQUEST)
 
-
     def get_list(self, table, filter={}):
         try:
             l = []
@@ -121,11 +121,12 @@ class DbMongo(DbBase):
             rows = collection.find(filter)
             if rows.count() == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             elif rows.count() > 1:
                 if fail_on_more:
-                    raise DbException("Found more than one entry with filter='{}'".format(filter),
+                    raise DbException("Found more than one {} with filter='{}'".format(table[:-1], filter),
                                       HTTPStatus.CONFLICT)
             return rows[0]
         except Exception as e:  # TODO refine
@@ -147,7 +148,8 @@ class DbMongo(DbBase):
             rows = collection.delete_one(self._format_filter(filter))
             if rows.deleted_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
@@ -167,7 +169,8 @@ class DbMongo(DbBase):
             rows = collection.update_one(self._format_filter(filter), {"$set": update_dict})
             if rows.updated_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
@@ -179,7 +182,8 @@ class DbMongo(DbBase):
             rows = collection.replace_one({"_id": id}, indata)
             if rows.modified_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"replace": rows.modified_count}
         except Exception as e:  # TODO refine
index c35617b..a15420f 100644 (file)
@@ -29,6 +29,29 @@ class EngineException(Exception):
         Exception.__init__(self, message)
 
 
+def _deep_update(dict_to_change, dict_reference):
+    """
+    Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
+    :param dict_to_change:  Ends modified
+    :param dict_reference: reference
+    :return: none
+    """
+
+    for k in dict_reference:
+        if dict_reference[k] is None:   # None->Anything
+            if k in dict_to_change:
+                del dict_to_change[k]
+        elif not isinstance(dict_reference[k], dict):  #  NotDict->Anything
+            dict_to_change[k] = dict_reference[k]
+        elif k not in dict_to_change:  # Dict->Empty
+            dict_to_change[k] = deepcopy(dict_reference[k])
+            _deep_update(dict_to_change[k], dict_reference[k])
+        elif isinstance(dict_to_change[k], dict):  # Dict->Dict
+            _deep_update(dict_to_change[k], dict_reference[k])
+        else:       # Dict->NotDict
+            dict_to_change[k] = deepcopy(dict_reference[k])
+            _deep_update(dict_to_change[k], dict_reference[k])
+
 class Engine(object):
 
     def __init__(self):
@@ -189,7 +212,7 @@ class Engine(object):
         """
         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
         vnfd or nsd content
-        :param item: can be vnfds, nsds, users, projects,
+        :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds
         :param indata: Content to be inspected
         :return: the useful part of indata
         """
@@ -214,9 +237,12 @@ class Engine(object):
                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
                     raise EngineException("'nsd' must be a list only one element")
                 clean_indata = clean_indata['nsd'][0]
+        elif item == "userDefinedData":
+            if "userDefinedData" in indata:
+                clean_indata = clean_indata['userDefinedData']
         return clean_indata
 
-    def _validate_new_data(self, session, item, indata):
+    def _validate_new_data(self, session, item, indata, id=None):
         if item == "users":
             if not indata.get("username"):
                 raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY)
@@ -233,8 +259,10 @@ class Engine(object):
             # check name not exist
             if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
                 raise EngineException("name '{}' exist".format(indata["name"]), HTTPStatus.CONFLICT)
-        elif item == "vnfds" or item == "nsds":
+        elif item in ("vnfds", "nsds"):
             filter = {"id": indata["id"]}
+            if id:
+                filter["_id.neq"] = id
             # TODO add admin to filter, validate rights
             self._add_read_filter(session, item, filter)
             if self.db.get_one(item, filter, fail_on_empty=False):
@@ -242,6 +270,10 @@ class Engine(object):
                                       HTTPStatus.CONFLICT)
 
             # TODO validate with pyangbind
+        elif item == "userDefinedData":
+            # TODO validate userDefinedData is a keypair values
+            pass
+
         elif item == "nsrs":
             pass
 
@@ -254,7 +286,7 @@ class Engine(object):
         if item == "users":
             _id = indata["username"]
             salt = uuid4().hex
-            indata["_admin"]["salt"] =  salt
+            indata["_admin"]["salt"] = salt
             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
         elif item == "projects":
             _id = indata["name"]
@@ -266,36 +298,44 @@ class Engine(object):
                 storage = admin.get("storage")
             if not _id:
                 _id = str(uuid4())
-            if item == "vnfds" or item == "nsds":
+            if item in ("vnfds", "nsds"):
                 if not indata["_admin"].get("projects_read"):
                     indata["_admin"]["projects_read"] = [session["project_id"]]
                 if not indata["_admin"].get("projects_write"):
                     indata["_admin"]["projects_write"] = [session["project_id"]]
+                indata["_admin"]["onboardingState"] = "CREATED"
+                indata["_admin"]["operationalState"] = "DISABLED"
+                indata["_admin"]["usageSate"] = "NOT_IN_USE"
             if storage:
                 indata["_admin"]["storage"] = storage
         indata["_id"] = _id
 
-    def _new_item_partial(self, session, item, indata, headers):
+    def upload_content(self, session, item, _id, indata, kwargs, headers):
         """
-        Used for recieve content by chunks (with a transaction_id header and/or gzip file. It will store and extract
+        Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
         :param session: session
-        :param item:
+        :param item: can be nsds or vnfds
+        :param _id : the nsd,vnfd is already created, this is the id
         :param indata: http body request
+        :param kwargs: user query string to override parameters. NOT USED
         :param headers:  http request headers
-        :return: a dict with::
-            _id: <transaction_id>
-            storage: <path>:  where it is saving
-            desc: <dict>: descriptor: Only present when all the content is received, extracted and read the descriptor
+        :return: True package has is completely uploaded or False if partial content has been uplodaed.
+            Raise exception on error
         """
+        # Check that _id exist and it is valid
+        current_desc = self.get_item(session, item, _id)
+
         content_range_text = headers.get("Content-Range")
-        transaction_id = headers.get("Transaction-Id")
-        filename = headers.get("Content-Filename", "pkg")
-        # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
         expected_md5 = headers.get("Content-File-MD5")
         compressed = None
-        if "application/gzip" in headers.get("Content-Type") or "application/x-gzip" in headers.get("Content-Type") or \
-                "application/zip" in headers.get("Content-Type"):
+        content_type = headers.get("Content-Type")
+        if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
+                "application/zip" in content_type:
             compressed = "gzip"
+        filename = headers.get("Content-Filename")
+        if not filename:
+            filename = "package.tar.gz" if compressed else "package"
+        # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
         file_pkg = None
         error_text = ""
         try:
@@ -306,41 +346,48 @@ class Engine(object):
                 start = int(content_range[1])
                 end = int(content_range[2]) + 1
                 total = int(content_range[3])
-                if len(indata) != end-start:
-                    raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
-                        start, end-1, len(indata)), HTTPStatus.BAD_REQUEST)
             else:
                 start = 0
-                total = end = len(indata)
-            if not transaction_id:
-                # generate transaction
-                transaction_id = str(uuid4())
-                self.fs.mkdir(transaction_id)
-                # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'wb')
-                # control = {"received": 0}
-            elif not self.fs.file_exists(transaction_id):
-                raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
+
+            if start:
+                if not self.fs.file_exists(_id, 'dir'):
+                    raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
             else:
-                pass
-                # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'rw')
-                # control = yaml.load(control_file)
-                # control_file.seek(0, 0)
+                self.fs.file_delete(_id, ignore_non_exist=True)
+                self.fs.mkdir(_id)
+
             storage = self.fs.get_params()
-            storage["folder"] = transaction_id
-            storage["file"] = filename
+            storage["folder"] = _id
 
-            file_path = (transaction_id, filename)
-            if self.fs.file_exists(file_path):
+            file_path = (_id, filename)
+            if self.fs.file_exists(file_path, 'file'):
                 file_size = self.fs.file_size(file_path)
             else:
                 file_size = 0
             if file_size != start:
-                raise EngineException("invalid upload transaction sequence, expected '{}' but received '{}'".format(
-                    file_size, start), HTTPStatus.BAD_REQUEST)
+                raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
+                    file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
             file_pkg = self.fs.file_open(file_path, 'a+b')
-            file_pkg.write(indata)
-            if end != total:
-                return {"_id": transaction_id, "storage": storage}
+            if isinstance(indata, dict):
+                indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
+                file_pkg.write(indata_text.encode(encoding="utf-8"))
+            else:
+                indata_len = 0
+                while True:
+                    indata_text = indata.read(4096)
+                    indata_len += len(indata_text)
+                    if not indata_text:
+                        break
+                    file_pkg.write(indata_text)
+            if content_range_text:
+                if indata_len != end-start:
+                    raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
+                        start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
+                if end != total:
+                    # TODO update to UPLOADING
+                    return False
+
+            # PACKAGE UPLOADED
             if expected_md5:
                 file_pkg.seek(0, 0)
                 file_md5 = md5()
@@ -352,8 +399,6 @@ class Engine(object):
                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
             file_pkg.seek(0, 0)
             if compressed == "gzip":
-                # TODO unzip,
-                storage["tarfile"] = filename
                 tar = tarfile.open(mode='r', fileobj=file_pkg)
                 descriptor_file_name = None
                 for tarinfo in tar:
@@ -364,32 +409,43 @@ class Engine(object):
                     if len(tarname_path) == 1 and not tarinfo.isdir():
                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
-                        storage["file"] = tarname_path[0]
+                        storage["pkg-dir"] = tarname_path[0]
                         if len(tarname_path) == 2:
                             if descriptor_file_name:
                                 raise EngineException("Found more than one descriptor file at package descriptor tar.gz")
                             descriptor_file_name = tarname
                 if not descriptor_file_name:
                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
-                self.fs.file_extract(tar, transaction_id)
-                with self.fs.file_open((transaction_id, descriptor_file_name), "r") as descriptor_file:
+                storage["descriptor"] = descriptor_file_name
+                storage["zipfile"] = filename
+                self.fs.file_extract(tar, _id)
+                with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
                     content = descriptor_file.read()
             else:
                 content = file_pkg.read()
-                tarname = ""
+                storage["descriptor"] = descriptor_file_name = filename
 
-            if tarname.endswith(".json"):
+            if descriptor_file_name.endswith(".json"):
                 error_text = "Invalid json format "
                 indata = json.load(content)
             else:
                 error_text = "Invalid yaml format "
                 indata = yaml.load(content)
-            return {"_id": transaction_id, "storage": storage, "desc": indata}
+
+            current_desc["_admin"]["storage"] = storage
+            current_desc["_admin"]["onboardingState"] = "ONBOARDED"
+            current_desc["_admin"]["operationalState"] = "ENABLED"
+
+            self._edit_item(session, item, _id, current_desc, indata, kwargs)
+            # TODO if descriptor has changed because kwargs update content and remove cached zip
+            # TODO if zip is not present creates one
+            return True
+
         except EngineException:
             raise
         except IndexError:
             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
-                                  HTTPStatus.BAD_REQUEST)
+                                  HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
         except IOError as e:
             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
         except (ValueError, yaml.YAMLError) as e:
@@ -441,9 +497,10 @@ class Engine(object):
 
     def new_item(self, session, item, indata={}, kwargs=None, headers={}):
         """
-        Creates a new entry into database
+        Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED  entry,
+        that must be completed with a call to method upload_content
         :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param item: it can be: users, projects, nsrs, nsds, vnfds
         :param indata: data to be inserted
         :param kwargs: used to override the indata descriptor
         :param headers: http request headers
@@ -453,16 +510,20 @@ class Engine(object):
         # TODO add admin and status
 
         transaction = None
-        if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \
-            "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type"):
-            if not indata:
-                raise EngineException("Empty payload")
-            transaction = self._new_item_partial(session, item, indata, headers)
-            if "desc" not in transaction:
-                return transaction["_id"], False
-            indata = transaction["desc"]
-
-        content = self._remove_envelop(item, indata)
+        # if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \
+        #     "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type") or \
+        #     "text/plain" in headers.get("Content-Type"):
+        #     if not indata:
+        #         raise EngineException("Empty payload")
+        #     transaction = self._new_item_partial(session, item, indata, headers)
+        #     if "desc" not in transaction:
+        #         return transaction["_id"], False
+        #     indata = transaction["desc"]
+
+        item_envelop = item
+        if item in ("nsds", "vnfds"):
+            item_envelop = "userDefinedData"
+        content = self._remove_envelop(item_envelop, indata)
 
         # Override descriptor with query string kwargs
         if kwargs:
@@ -491,7 +552,7 @@ class Engine(object):
             except IndexError:
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
-        if not indata:
+        if not indata and item not in ("nsds", "vnfds"):
             raise EngineException("Empty payload")
 
         if item == "nsrs":
@@ -500,12 +561,14 @@ class Engine(object):
             content, _id = self.new_nsr(session, ns_request)
             transaction = {"_id": _id}
 
-        self._validate_new_data(session, item, content)
+        self._validate_new_data(session, item_envelop, content)
+        if item in ("nsds", "vnfds"):
+            content = {"_admin": {"userDefinedData": content}}
         self._format_new_data(session, item, content, transaction)
         _id = self.db.create(item, content)
         if item == "nsrs":
             self.msg.write("ns", "create", _id)
-        return _id, True
+        return _id
 
     def _add_read_filter(self, session, item, filter):
         if session["project_id"] == "admin":  # allows all
@@ -527,6 +590,62 @@ class Engine(object):
         elif item in ("vnfds", "nsds") and session["project_id"] != "admin":
             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 
+    def get_file(self, session, item, _id, path=None, accept_header=None):
+        """
+        Return the file content of a vnfd or nsd
+        :param session: contains the used login username and working project
+        :param item: it can be vnfds or nsds
+        :param _id: Identity of the vnfd, ndsd
+        :param path: artifact path or "$DESCRIPTOR" or None
+        :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
+        :return: opened file or raises an exception
+        """
+        accept_text = accept_zip = False
+        if accept_header:
+            if 'text/plain' in accept_header or '*/*' in accept_header:
+                accept_text = True
+            if 'application/zip' in accept_header or '*/*' in accept_header:
+                accept_zip = True
+        if not accept_text and not accept_zip:
+            raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
+                                  http_code=HTTPStatus.NOT_ACCEPTABLE)
+
+        content = self.get_item(session, item, _id)
+        if content["_admin"]["onboardingState"] != "ONBOARDED":
+            raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
+                "onboardingState is {}".format(content["_admin"]["onboardingState"]),
+                                  http_code=HTTPStatus.CONFLICT)
+        storage = content["_admin"]["storage"]
+        if path is not None and path != "$DESCRIPTOR":   # artifacts
+            if not storage.get('pkg-dir'):
+                raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
+            if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
+                folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
+                return folder_content, "text/plain"
+                # TODO manage folders in http
+            else:
+                return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
+                       "application/octet-stream"
+
+        # pkgtype   accept  ZIP  TEXT    -> result
+        # manyfiles         yes  X       -> zip
+        #                   no   yes     -> error
+        # onefile           yes  no      -> zip
+        #                   X    yes     -> text
+
+        if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
+            return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
+        elif storage.get('pkg-dir') and not accept_zip:
+            raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
+                                      "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
+        else:
+            if not storage.get('zipfile'):
+                # TODO generate zipfile if not present
+                raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in future versions"
+                                      "", http_code=HTTPStatus.NOT_ACCEPTABLE)
+            return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip"
+
+
     def get_item_list(self, session, item, filter={}):
         """
         Get a list of items
@@ -536,6 +655,9 @@ class Engine(object):
         :return: The list, it can be empty if no one match the filter.
         """
         # TODO add admin to filter, validate rights
+        # TODO transform data for SOL005 URL requests. Transform filtering
+        # TODO implement "field-type" query string SOL005
+
         self._add_read_filter(session, item, filter)
         return self.db.get_list(item, filter)
 
@@ -543,12 +665,14 @@ class Engine(object):
         """
         Get complete information on an items
         :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param item: it can be: users, projects, vnfds, nsds,
         :param _id: server id of the item
         :return: dictionary, raise exception if not found.
         """
+        database_item = item
         filter = {"_id": _id}
         # TODO add admin to filter, validate rights
+        # TODO transform data for SOL005 URL requests
         self._add_read_filter(session, item, filter)
         return self.db.get_one(item, filter)
 
@@ -611,27 +735,15 @@ class Engine(object):
         _id = self.db.create("users", indata)
         return _id
 
-    def edit_item(self, session, item, id, indata={}, kwargs=None):
-        """
-        Update an existing entry at database
-        :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
-        :param id: identity of entry to be updated
-        :param indata: data to be inserted
-        :param kwargs: used to override the indata descriptor
-        :return: dictionary, raise exception if not found.
-        """
-
-        content = self.get_item(session, item, id)
+    def _edit_item(self, session, item, id, content, indata={}, kwargs=None):
         if indata:
             indata = self._remove_envelop(item, indata)
-            # TODO update content with with a deep-update
 
         # Override descriptor with query string kwargs
         if kwargs:
             try:
                 for k, v in kwargs.items():
-                    update_content = content
+                    update_content = indata
                     kitem_old = None
                     klist = k.split(".")
                     for kitem in klist:
@@ -655,9 +767,25 @@ class Engine(object):
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 
-        self._validate_new_data(session, item, content)
+        _deep_update(content, indata)
+        self._validate_new_data(session, item, content, id)
         # self._format_new_data(session, item, content)
         self.db.replace(item, id, content)
         return id
 
+    def edit_item(self, session, item, _id, indata={}, kwargs=None):
+        """
+        Update an existing entry at database
+        :param session: contains the used login username and working project
+        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param _id: identifier to be updated
+        :param indata: data to be inserted
+        :param kwargs: used to override the indata descriptor
+        :return: dictionary, raise exception if not found.
+        """
+
+        content = self.get_item(session, item, _id)
+        return self._edit_item(session, item, _id, content, indata, kwargs)
+
+
 
index 10ddf73..b7dd839 100644 (file)
@@ -46,17 +46,23 @@ class FsLocal(FsBase):
         except Exception as e:
             raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 
-    def file_exists(self, storage):
+    def file_exists(self, storage, mode=None):
         """
         Indicates if "storage" file exist
         :param storage: can be a str or a str list
+        :param mode: can be 'file' exist as a regular file; 'dir' exists as a directory or; 'None' just exists
         :return: True, False
         """
         if isinstance(storage, str):
             f = storage
         else:
             f = "/".join(storage)
-        return os.path.exists(self.path + f)
+        if os.path.exists(self.path + f):
+            if mode == "file" and os.path.isfile(self.path + f):
+                return True
+            if mode == "dir" and os.path.isdir(self.path + f):
+                return True
+        return False
 
     def file_size(self, storage):
         """
@@ -90,11 +96,33 @@ class FsLocal(FsBase):
         :param mode: file mode
         :return: file object
         """
-        if isinstance(storage, str):
-            f = storage
-        else:
-            f = "/".join(storage)
-        return open(self.path + f, mode)
+        try:
+            if isinstance(storage, str):
+                f = storage
+            else:
+                f = "/".join(storage)
+            return open(self.path + f, mode)
+        except FileNotFoundError:
+            raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+        except IOError:
+            raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
+
+    def dir_ls(self, storage):
+        """
+        return folder content
+        :param storage: can be a str or list of str
+        :return: folder content
+        """
+        try:
+            if isinstance(storage, str):
+                f = storage
+            else:
+                f = "/".join(storage)
+            return os.listdir(self.path + f)
+        except NotADirectoryError:
+            raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+        except IOError:
+            raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
 
     def file_delete(self, storage, ignore_non_exist=False):
         """
@@ -111,4 +139,4 @@ class FsLocal(FsBase):
         if os.path.exists(f):
             rmtree(f)
         elif not ignore_non_exist:
-            raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.BAD_REQUEST)
+            raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.NOT_FOUND)
index f6f92d9..f37b5dd 100644 (file)
@@ -19,13 +19,13 @@ html_start = """
     <div>
       <a href="https://osm.etsi.org"> <img src="/osm/static/OSM-logo.png" height="42" width="100" style="vertical-align:middle"> </a>
       <a>( {} )</a>
-      <a href="/osm/vnfpkgm/v1/vnf_packages">VNFDs </a>
-      <a href="/osm/nsd/v1/ns_descriptors">NSDs </a>
-      <a href="/osm/nslcm/v1/ns_instances">NSs </a>
-      <a href="/osm/user/v1">USERs </a>
-      <a href="/osm/project/v1">PROJECTs </a>
-      <a href="/osm/token/v1">TOKENs </a>
-      <a href="/osm/token/v1?METHOD=DELETE">logout </a>
+      <a href="/osm/vnfpkgm/v1/vnf_packages_content">VNFDs </a>
+      <a href="/osm/nsd/v1/ns_descriptors_content">NSDs </a>
+      <a href="/osm/nslcm/v1/ns_instances_content">NSs </a>
+      <a href="/osm/admin/v1/users">USERs </a>
+      <a href="/osm/admin/v1/projects">PROJECTs </a>
+      <a href="/osm/admin/v1/tokens">TOKENs </a>
+      <a href="/osm/admin/v1/tokens?METHOD=DELETE">logout </a>
     </div>
   </div>
 """
@@ -61,7 +61,7 @@ html_auth2 = """
   </div>
   <div class="gerritBody" id="osm_body">
     <h1>Sign in to OSM</h1>
-    <form action="/osm/token/v1" id="login_form" method="POST">
+    <form action="/osm/admin/v1/tokens" id="login_form" method="POST">
       <table style="border: 0;">
         <tr><th>Username</th><td><input id="f_user" name="username" size="25" tabindex="1" type="text"></td></tr>
         <tr><th>Password</th><td><input id="f_pass" name="password" size="25" tabindex="2" type="password"></td></tr>
@@ -108,12 +108,15 @@ def format(data, request, response, session):
     if response.status and response.status > 202:
         body += html_body_error.format(yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False))
     elif isinstance(data, (list, tuple)):
-        if request.path_info == "/vnfpkgm/v1/vnf_packages":
+        if request.path_info == "/vnfpkgm/v1/vnf_packages_content":
             body += html_upload_body.format("VNFD", request.path_info)
-        elif request.path_info == "/nsd/v1/ns_descriptors":
+        elif request.path_info == "/nsd/v1/ns_descriptors_content":
             body += html_upload_body.format("NSD", request.path_info)
         for k in data:
-            data_id = k.pop("_id", None)
+            if isinstance(k, dict):
+                data_id = k.pop("_id", None)
+            elif isinstance(k, str):
+                data_id = k
             body += '<p> <a href="/osm/{url}/{id}">{id}</a>: {t} </p>'.format(url=request.path_info, id=data_id, t=k)
     elif isinstance(data, dict):
         if "Location" in response.headers:
index 9cdb409..99f90aa 100644 (file)
@@ -9,49 +9,63 @@ import html_out as html
 import logging
 from engine import Engine, EngineException
 from dbbase import DbException
+from fsbase import FsException
 from base64 import standard_b64decode
-from os import getenv
+#from os import getenv
 from http import HTTPStatus
-from http.client import responses as http_responses
+#from http.client import responses as http_responses
 from codecs import getreader
 from os import environ
 
 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
-__version__ = "0.1"
-version_date = "Feb 2018"
+__version__ = "0.2"
+version_date = "Mar 2018"
 
 """
-North Bound Interface  (O: OSM; S: SOL5
+North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
-        /nsd/v1
+        /nsd/v1                                                 O       O
+            /ns_descriptors_content                             O       O       
+                /<nsdInfoId>                                    O       O       O       O     
             /ns_descriptors                                     O5      O5
                 /<nsdInfoId>                                    O5                      O5      5
                     /nsd_content                                O5              O5
+                    /nsd                                        O
+                    /artifacts[/<artifactPath>]                 O
             /pnf_descriptors                                    5       5
                 /<pnfdInfoId>                                   5                       5       5
                     /pnfd_content                               5               5
-            /subcriptions                                       5       5
-                /<subcriptionId>                                5                       X
+            /subscriptions                                      5       5
+                /<subscriptionId>                               5                       X
 
         /vnfpkgm/v1
             /vnf_packages                                       O5      O5
                 /<vnfPkgId>                                     O5                      O5      5
-                    /vnfd                                       O5      O
                     /package_content                            O5               O5
                         /upload_from_uri                                X
-                    /artifacts/<artifactPatch                   X
-            /subcriptions                                       X       X
-                /<subcriptionId>                                X                       X
+                    /vnfd                                       O5
+                    /artifacts[/<artifactPath>]                 O5
+            /subscriptions                                      X       X
+                /<subscriptionId>                               X                       X
 
         /nslcm/v1
-            /ns_instances                                       O5      O5
-                /<nsInstanceId>                                 O5                      O5     
+            /ns_instances_content                               O       O
+                /<nsInstanceId>                                 O                       O     
+            /ns_instances                                       5       5
+                /<nsInstanceId>                                 5                       5     
                     TO BE COMPLETED                             
             /ns_lcm_op_occs                                     5       5
                 /<nsLcmOpOccId>                                 5                       5       5
                     TO BE COMPLETED                             5               5
-            /subcriptions                                       5       5
-                /<subcriptionId>                                5                       X
+            /subscriptions                                      5       5
+                /<subscriptionId>                               5                       X
+        /admin/v1
+            /tokens                                             O       O
+                /<id>                                           O                       O     
+            /users                                              O       O
+                /<id>                                           O                       O     
+            /projects                                           O       O
+                /<id>                                           O                       O     
 
 query string.
     <attrName>[.<attrName>...]*[.<op>]=<value>[,<value>...]&...
@@ -106,6 +120,74 @@ class Server(object):
     def __init__(self):
         self.instance += 1
         self.engine = Engine()
+        self.valid_methods = {   # contains allowed URL and methods
+            "admin": {
+                "v1": {
+                    "tokens": { "METHODS": ("GET", "POST", "DELETE"),
+                        "<ID>": { "METHODS": ("GET", "DELETE")}
+                    },
+                    "users": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "POST", "DELETE")}
+                    },
+                    "projects": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "POST", "DELETE")}
+                    },
+                }
+            },
+            "nsd": {
+                "v1": {
+                    "ns_descriptors_content": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
+                    },
+                    "ns_descriptors": { "METHODS": ("GET", "POST"),
+                        "<ID>": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH",
+                            "nsd_content": { "METHODS": ("GET", "PUT")},
+                            "nsd": {"METHODS": "GET"},  # descriptor inside package
+                            "artifacts": {"*": {"METHODS": "GET"}}
+                        }
+
+                    },
+                    "pnf_descriptors": {"TODO": ("GET", "POST"),
+                       "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
+                            "pnfd_content": {"TODO": ("GET", "PUT")}
+                        }
+                    },
+                    "subscriptions": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE"),}
+                    },
+                }
+            },
+            "vnfpkgm": {
+                "v1": {
+                    "vnf_packages_content": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
+                    },
+                    "vnf_packages": { "METHODS": ("GET", "POST"),
+                        "<ID>": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH",  # GET: vnfPkgInfo
+                            "package_content": { "METHODS": ("GET", "PUT"),         # package
+                                "upload_from_uri": {"TODO": "POST"}
+                            },
+                            "vnfd": {"METHODS": "GET"},                    # descriptor inside package
+                            "artifacts": {"*": {"METHODS": "GET"}}
+                        }
+
+                    },
+                    "subscriptions": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE"),}
+                    },
+                }
+            },
+            "nslcm": {
+                "v1": {
+                    "ns_instances_content": {"METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "DELETE")}
+                    },
+                    "ns_instances": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE")}
+                    }
+                }
+            },
+        }
 
     def _authorization(self):
         token = None
@@ -164,14 +246,15 @@ class Server(object):
                         indata = yaml.load(cherrypy.request.body)
                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
-                         "application/zip" in cherrypy.request.headers["Content-Type"]:
-                        indata = cherrypy.request.body.read()
+                         "application/zip" in cherrypy.request.headers["Content-Type"] or \
+                         "text/plain" in cherrypy.request.headers["Content-Type"]:
+                        indata = cherrypy.request.body  # .read()
                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
                         if "descriptor_file" in kwargs:
                             filecontent = kwargs.pop("descriptor_file")
                             if not filecontent.file:
                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
-                            indata = filecontent.file.read()
+                            indata = filecontent.file  # .read()
                             if filecontent.content_type.value:
                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
                     else:
@@ -186,10 +269,6 @@ class Server(object):
             if not indata:
                 indata = {}
 
-            if "METHOD" in kwargs:
-                method = kwargs.pop("METHOD")
-            else:
-                method = cherrypy.request.method
             format_yaml = False
             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
                 format_yaml = True
@@ -223,20 +302,33 @@ class Server(object):
                             except:
                                 pass
 
-            return indata, method
+            return indata
         except (ValueError, yaml.YAMLError) as exc:
             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
         except KeyError as exc:
             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 
     @staticmethod
-    def _format_out(data, session=None):
+    def _format_out(data, session=None, _format=None):
         """
         return string of dictionary data according to requested json, yaml, xml. By default json
-        :param data: response to be sent. Can be a dict or text
+        :param data: response to be sent. Can be a dict, text or file
         :param session:
+        :param _format: The format to be set as Content-Type ir data is a file
         :return: None
         """
+        if data is None:
+            cherrypy.response.status = HTTPStatus.NO_CONTENT.value
+            return
+        elif hasattr(data, "read"):  # file object
+            if _format:
+                cherrypy.response.headers["Content-Type"] = _format
+            elif "b" in data.mode:  # binariy asssumig zip
+                cherrypy.response.headers["Content-Type"] = 'application/zip'
+            else:
+                cherrypy.response.headers["Content-Type"] = 'text/plain'
+            # TODO check that cherrypy close file. If not implement pending things to close  per thread next
+            return data
         if "Accept" in cherrypy.request.headers:
             accept = cherrypy.request.headers["Accept"]
             if "application/json" in accept:
@@ -246,7 +338,7 @@ class Server(object):
             elif "text/html" in accept:
                 return html.format(data, cherrypy.request, cherrypy.response, session)
 
-            elif "application/yaml" in accept or "*/*" in accept:
+            elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
                 pass
             else:
                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
@@ -275,20 +367,17 @@ class Server(object):
             return self._format_out("Welcome to OSM!", session)
 
     @cherrypy.expose
-    def token(self, *args, **kwargs):
-        if not args:
-            raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED)
-        version = args[0]
-        if version != 'v1':
-            raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
+    def token(self, method, token_id=None, kwargs=None):
         session = None
         # self.engine.load_dbase(cherrypy.request.app.config)
+        indata = self._format_in(kwargs)
+        if not isinstance(indata, dict):
+            raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
         try:
-            indata, method = self._format_in(kwargs)
             if method == "GET":
                 session = self._authorization()
-                if len(args) >= 2:
-                    outdata = self.engine.get_token(session, args[1])
+                if token_id:
+                    outdata = self.engine.get_token(session, token_id)
                 else:
                     outdata = self.engine.get_token_list(session)
             elif method == "POST":
@@ -304,11 +393,9 @@ class Server(object):
                 # cherrypy.response.cookie["Authorization"] = outdata["id"]
                 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
             elif method == "DELETE":
-                if len(args) >= 2 and "logout" not in args:
-                    token_id = args[1]
-                elif "id" in kwargs:
+                if not token_id and "id" in kwargs:
                     token_id = kwargs["id"]
-                else:
+                elif not token_id:
                     session = self._authorization()
                     token_id = session["_id"]
                 outdata = self.engine.del_token(token_id)
@@ -330,9 +417,20 @@ class Server(object):
             return self._format_out(problem_details, session)
 
     @cherrypy.expose
+    def test2(self, args0=None, args1=None, args2=None, args3=None, *args, **kwargs):
+        return_text = (
+            "<html><pre>\n{} {} {} {} {} {} \n".format(args0, args1, args2, args3, args, kwargs))
+        return_text += "</pre></html>"
+        return return_text
+
+    @cherrypy.expose
     def test(self, *args, **kwargs):
         thread_info = None
-        if args and args[0] == "init":
+        if args and args[0] == "help":
+            return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
+                    "sleep/<time>\n</pre></html>"
+
+        elif args and args[0] == "init":
             try:
                 # self.engine.load_dbase(cherrypy.request.app.config)
                 self.engine.create_admin()
@@ -340,6 +438,17 @@ class Server(object):
             except Exception:
                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
                 return self._format_out("Database already initialized")
+        elif args and args[0] == "file":
+            return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
+                                                  "text/plain", "attachment")
+        elif args and args[0] == "file2":
+            f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
+            f = open(f_path, "r")
+            cherrypy.response.headers["Content-type"] = "text/plain"
+
+            return f
+        elif len(args) == 2 and args[0] == "db-clear":
+            return self.engine.del_item_list({"project_id": "admin"}, args[1], {})
         elif args and args[0] == "prune":
             return self.engine.prune()
         elif args and args[0] == "login":
@@ -390,83 +499,148 @@ class Server(object):
         return_text += "</pre></html>"
         return return_text
 
+    def _check_valid_url_method(self, method, *args):
+        if len(args) < 3:
+            raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED)
+
+        reference = self.valid_methods
+        for arg in args:
+            if arg is None:
+                break
+            if not isinstance(reference, dict):
+                raise NbiException("URL contains unexpected extra items '{}'".format(arg),
+                                   HTTPStatus.METHOD_NOT_ALLOWED)
+
+            if arg in reference:
+                reference = reference[arg]
+            elif "<ID>" in reference:
+                reference = reference["<ID>"]
+            elif "*" in reference:
+                reference = reference["*"]
+                break
+            else:
+                raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
+        if "TODO" in reference and method in reference["TODO"]:
+            raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
+        elif "METHODS" in reference and not method in reference["METHODS"]:
+            raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
+        return
+
+    @staticmethod
+    def _set_location_header(topic, version, item, id):
+        """
+        Insert response header Location with the URL of created item base on URL params
+        :param topic:
+        :param version:
+        :param item:
+        :param id:
+        :return: None
+        """
+        # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
+        cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(topic, version, item, id)
+        return
+
     @cherrypy.expose
-    def default(self, *args, **kwargs):
+    def default(self, topic=None, version=None, item=None, _id=None, item2=None, *args, **kwargs):
         session = None
+        outdata = None
+        _format = None
         try:
-            if not args or len(args) < 2:
-                raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED)
-            item = args[0]
-            version = args[1]
-            if item not in ("token", "user", "project", "vnfpkgm", "nsd", "nslcm"):
-                raise NbiException("URL item '{}' not supported".format(item), HTTPStatus.METHOD_NOT_ALLOWED)
+            if not topic or not version or not item:
+                raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED)
+            if topic not in ("admin", "vnfpkgm", "nsd", "nslcm"):
+                raise NbiException("URL topic '{}' not supported".format(topic), HTTPStatus.METHOD_NOT_ALLOWED)
             if version != 'v1':
                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 
+            if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
+                method = kwargs.pop("METHOD")
+            else:
+                method = cherrypy.request.method
+
+            self._check_valid_url_method(method, topic, version, item, _id, item2, *args)
+
+            if topic == "admin" and item == "tokens":
+                return self.token(method, _id, kwargs)
+
             # self.engine.load_dbase(cherrypy.request.app.config)
             session = self._authorization()
-            indata, method = self._format_in(kwargs)
-            _id = None
-
-            if item == "nsd":
-                item = "nsds"
-                if len(args) < 3 or args[2] != "ns_descriptors":
-                    raise NbiException("only ns_descriptors is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4 and args[4] != "nsd_content":
-                    raise NbiException("only nsd_content is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-            elif item == "vnfpkgm":
-                item = "vnfds"
-                if len(args) < 3 or args[2] != "vnf_packages":
-                    raise NbiException("only vnf_packages is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4 and args[4] not in ("vnfd", "package_content"):
-                    raise NbiException("only vnfd or package_content are allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-            elif item == "nslcm":
-                item = "nsrs"
-                if len(args) < 3 or args[2] != "ns_instances":
-                    raise NbiException("only ns_instances is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4:
-                    raise NbiException("This feature is not implemented", HTTPStatus.METHOD_NOT_ALLOWED)
-            else:
-                if len(args) >= 3:
-                    _id = args[2]
-                item += "s"
+            indata = self._format_in(kwargs)
+            engine_item = item
+            if item == "subscriptions":
+                engine_item = topic + "_" + item
+            if item2:
+                engine_item = item2
+
+            if topic == "nsd":
+                engine_item = "nsds"
+            elif topic == "vnfpkgm":
+                engine_item = "vnfds"
+            elif topic == "nslcm":
+                engine_item = "nsrs"
 
             if method == "GET":
-                if not _id:
-                    outdata = self.engine.get_item_list(session, item, kwargs)
-                else:  # len(args) > 1
-                    outdata = self.engine.get_item(session, item, _id)
+                if item2 in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd"):
+                    if item2 in ("vnfd", "nsd"):
+                        path = "$DESCRIPTOR"
+                    elif args:
+                        path = args
+                    elif item2 == "artifacts":
+                        path = ()
+                    else:
+                        path = None
+                    file, _format = self.engine.get_file(session, engine_item, _id, path,
+                                                            cherrypy.request.headers.get("Accept"))
+                    outdata = file
+                elif not _id:
+                    outdata = self.engine.get_item_list(session, engine_item, kwargs)
+                else:
+                    outdata = self.engine.get_item(session, engine_item, _id)
             elif method == "POST":
-                id, completed = self.engine.new_item(session, item, indata, kwargs, cherrypy.request.headers)
-                if not completed:
-                    cherrypy.response.headers["Transaction-Id"] = id
-                    cherrypy.response.status = HTTPStatus.CREATED.value
+                if item in ("ns_descriptors_content", "vnf_packages_content"):
+                    _id = cherrypy.request.headers.get("Transaction-Id")
+                    if not _id:
+                        _id = self.engine.new_item(session, engine_item, {}, None, cherrypy.request.headers)
+                    completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers)
+                    if completed:
+                        self._set_location_header(topic, version, item, _id)
+                    else:
+                        cherrypy.response.headers["Transaction-Id"] = _id
+                    outdata = {"id": _id}
+                elif item in ("ns_descriptors", "vnf_packages"):
+                    _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers)
+                    self._set_location_header(topic, version, item, _id)
+                    #TODO form NsdInfo
+                    outdata = {"id": _id}
                 else:
-                    cherrypy.response.headers["Location"] = cherrypy.request.base + "/osm/" + "/".join(args[0:3]) + "/" + id
-                outdata = {"id": id}
+                    _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers)
+                    self._set_location_header(topic, version, item, _id)
+                    outdata = {"id": _id}
+                cherrypy.response.status = HTTPStatus.CREATED.value
             elif method == "DELETE":
                 if not _id:
-                    outdata = self.engine.del_item_list(session, item, kwargs)
+                    outdata = self.engine.del_item_list(session, engine_item, kwargs)
                 else:  # len(args) > 1
-                    outdata = self.engine.del_item(session, item, _id)
+                    outdata = self.engine.del_item(session, engine_item, _id)
+                if item in ("ns_descriptors", "vnf_packages"):  # SOL005
+                    outdata = None
             elif method == "PUT":
-                if not _id:
-                    raise NbiException("Missing '/<id>' at the URL to identify item to be updated",
-                                       HTTPStatus.METHOD_NOT_ALLOWED)
-                elif not indata and not kwargs:
+                if not indata and not kwargs:
                     raise NbiException("Nothing to update. Provide payload and/or query string",
                                        HTTPStatus.BAD_REQUEST)
-                outdata = {"id": self.engine.edit_item(session, item, args[1], indata, kwargs)}
+                if item2 in ("nsd_content", "package_content"):
+                    completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers)
+                    if not completed:
+                        cherrypy.response.headers["Transaction-Id"] = id
+                    outdata = None
+                else:
+                    outdata = {"id": self.engine.edit_item(session, engine_item, args[1], indata, kwargs)}
             else:
                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
-            return self._format_out(outdata, session)
-        except (NbiException, EngineException, DbException) as e:
+            return self._format_out(outdata, session, _format)
+        except (NbiException, EngineException, DbException, FsException) as e:
+            if hasattr(outdata, "close"):  # is an open file
+                outdata.close()
             cherrypy.log("Exception {}".format(e))
             cherrypy.response.status = e.http_code.value
             problem_details = {
diff --git a/osm_nbi/test/cirros_ns/cirros_nsd.yaml b/osm_nbi/test/cirros_ns/cirros_nsd.yaml
new file mode 100644 (file)
index 0000000..5c4e214
--- /dev/null
@@ -0,0 +1,49 @@
+nsd-catalog:
+    nsd:
+    -   id: cirros_nsd
+        name: cirros_ns
+        short-name: cirros_ns
+        description: Generated by OSM pacakage generator
+        vendor: OSM
+        version: '1.0'
+
+        # Place the logo as png in icons directory and provide the name here
+        logo: osm_2x.png
+
+        # Specify the VNFDs that are part of this NSD
+        constituent-vnfd:
+            # The member-vnf-index needs to be unique, starting from 1
+            # vnfd-id-ref is the id of the VNFD
+            # Multiple constituent VNFDs can be specified
+        -   member-vnf-index: 1
+            vnfd-id-ref: cirros_vnfd
+        scaling-group-descriptor:
+        -   name: "scaling_cirros"
+            vnfd-member:
+            -   count: 1
+                member-vnf-index-ref: 1
+            min-instance-count: 0
+            max-instance-count: 10
+            scaling-policy:
+            -   scaling-type: "manual"
+                cooldown-time: 10
+                threshold-time: 10
+                name: manual_scale
+        vld:
+        # Networks for the VNFs
+            -   id: cirros_nsd_vld1
+                name: cirros_nsd_vld1
+                type: ELAN
+                mgmt-network: 'true'
+                # vim-network-name: <update>
+                # provider-network:
+                #     segmentation_id: <update>
+                vnfd-connection-point-ref:
+                # Specify the constituent VNFs
+                # member-vnf-index-ref - entry from constituent vnf
+                # vnfd-id-ref - VNFD id
+                # vnfd-connection-point-ref - connection point name in the VNFD
+                -   member-vnf-index-ref: 1
+                    vnfd-id-ref: cirros_vnfd
+                    # NOTE: Validate the entry below
+                    vnfd-connection-point-ref: eth0
diff --git a/osm_nbi/test/cirros_ns/icons/osm_2x.png b/osm_nbi/test/cirros_ns/icons/osm_2x.png
new file mode 100644 (file)
index 0000000..62012d2
Binary files /dev/null and b/osm_nbi/test/cirros_ns/icons/osm_2x.png differ
diff --git a/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml b/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml
new file mode 100644 (file)
index 0000000..94fa5f1
--- /dev/null
@@ -0,0 +1,48 @@
+vnfd-catalog:
+    vnfd:
+    -   id: cirros_vnfd
+        name: cirros_vnf
+        short-name: cirros_vnf
+        description: Simple VNF example with a cirros 
+        vendor: OSM
+        version: '1.0'
+
+        # Place the logo as png in icons directory and provide the name here
+        logo: cirros-64.png
+
+        # Management interface
+        mgmt-interface:
+            cp: eth0
+
+        # Atleast one VDU need to be specified
+        vdu:
+        -   id: cirros_vnfd-VM
+            name: cirros_vnfd-VM
+            description: cirros_vnfd-VM
+            count: 1
+
+            # Flavour of the VM to be instantiated for the VDU
+            # flavor below can fit into m1.micro
+            vm-flavor:
+                vcpu-count: 1
+                memory-mb: 256
+                storage-gb: 2
+
+            # Image/checksum or image including the full path
+            image: 'cirros034'
+            #checksum: 
+
+            interface:
+            # Specify the external interfaces
+            # There can be multiple interfaces defined
+            -   name: eth0
+                type: EXTERNAL
+                virtual-interface:
+                    type: VIRTIO
+                    bandwidth: '0'
+                    vpci: 0000:00:0a.0
+                external-connection-point-ref: eth0
+
+        connection-point:
+            -   name: eth0
+                type: VPORT
diff --git a/osm_nbi/test/cirros_vnf/icons/cirros-64.png b/osm_nbi/test/cirros_vnf/icons/cirros-64.png
new file mode 100644 (file)
index 0000000..5725d29
Binary files /dev/null and b/osm_nbi/test/cirros_vnf/icons/cirros-64.png differ
index 4d96d47..7ef1cf7 100755 (executable)
@@ -25,19 +25,23 @@ NSD3=${DESCRIPTORS}/cirros_nsd.yaml
 [ -f "$NSD3" ] || ! echo "not found cirros_nsd.yaml. Set DESCRIPTORS variable to a proper location" || exit 1
 
 #get token
-TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}'`;
+TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" \
+    --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens \
+     2>/dev/null | awk '($1=="id:"){print $2}'`;
 echo token: $TOKEN
 
 
-
-
 # VNFD
 #########
 #insert PKG
-VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'` 
+VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \
+     2>/dev/null | awk '($1=="id:"){print $2}'`
 echo ping_vnfd: $VNFD1_ID
 
-VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'`
+VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \
+     2>/dev/null | awk '($1=="id:"){print $2}'`
 echo pong_vnfd: $VNFD2_ID
 
 
@@ -45,19 +49,25 @@ echo pong_vnfd: $VNFD2_ID
 # NSD
 #########
 #insert PKG
-NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors 2>/dev/null | awk '($1=="id:"){print $2}'`
+NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"   --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors_content \
+    2>/dev/null | awk '($1=="id:"){print $2}'`
 echo ping_pong_nsd: $NSD1_ID
 
 
 # NSRS
 ##############
 #add nsr
-NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }"  ${NBI_URL}/nslcm/v1/ns_instances 2>/dev/null | awk '($1=="id:"){print $2}'` ;
+NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, \
+     ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }" \
+     ${NBI_URL}/nslcm/v1/ns_instances_content 2>/dev/null | awk '($1=="id:"){print $2}'` ;
 echo ping_pong_nsr: $NSR1_ID
 
 
-echo '
-curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer '$TOKEN'"  '${NBI_URL}'/nslcm/v1/ns_instances/'$NSR1_ID' 2>/dev/null | grep -e detailed-status -e operational-status -e config-status'
+echo 'curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml"' \
+    '-H "Authorization: Bearer '$TOKEN'"  '${NBI_URL}'/nslcm/v1/ns_instances_content/'$NSR1_ID' 2>/dev/null | ' \
+    'grep -e detailed-status -e operational-status -e config-status'
 
 
 
index 6b04538..d8cb474 100755 (executable)
@@ -8,13 +8,13 @@ PROJECT=admin
 
 
 #get token
-TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN
+TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN
 
 
 echo  deleting all
 #DELETE ALL
 
-for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors vnfpkgm/v1/vnf_packages
+for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors_content vnfpkgm/v1/vnf_packages_content
 do
     for ITEM_ID in `curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"  ${NBI_URL}/${url_item} 2>/dev/null | awk '($1=="_id:") {print $2}'` ;
     do
diff --git a/osm_nbi/test/test.py b/osm_nbi/test/test.py
new file mode 100755 (executable)
index 0000000..734d667
--- /dev/null
@@ -0,0 +1,337 @@
+#! /usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import getopt
+import sys
+import requests
+#import base64
+#from os.path import getsize, basename
+#from hashlib import md5
+import json
+import logging
+import yaml
+#import json
+import tarfile
+from os import makedirs
+from copy import deepcopy
+
+__author__ = "Alfonso Tierno, alfonso.tiernosepulveda@telefonica.com"
+__date__ = "$2018-03-01$"
+__version__ = "0.1"
+version_date = "Mar 2018"
+
+
+def usage():
+    print("Usage: ", sys.argv[0], "[options]")
+    print("      --version: prints current version")
+    print("      -f|--file FILE: file to be sent")
+    print("      -h|--help: shows this help")
+    print("      -u|--url URL: complete server URL")
+    print("      -s|--chunk-size SIZE: size of chunks, by default 1000")
+    print("      -t|--token TOKEN: Authorizaton token, previously obtained from server")
+    print("      -v|--verbose print debug information, can be used several times")
+    return
+
+
+r_header_json = {"Content-type": "application/json"}
+headers_json = {
+    "Content-type": "application/json",
+    "Accept": "application/json",
+}
+r_header_yaml = {"Content-type": "application/yaml"}
+headers_yaml = {
+    "Content-type": "application/yaml",
+    "Accept": "application/yaml",
+}
+r_header_text = {"Content-type": "text/plain"}
+r_header_octect = {"Content-type": "application/octet-stream"}
+headers_text = {
+    "Accept": "text/plain",
+}
+r_header_zip = {"Content-type": "application/zip"}
+headers_zip = {
+    "Accept": "application/zip",
+}
+# test without authorization
+test_not_authorized_list = (
+    ("Invalid token", "GET", "/admin/v1/users", headers_json, None, 401, r_header_json, "json"),
+    ("Invalid URL", "POST", "/admin/v1/nonexist", headers_yaml, None, 405, r_header_yaml, "yaml"),
+    ("Invalid version", "DELETE", "/admin/v2/users", headers_yaml, None, 405, r_header_yaml, "yaml"),
+)
+
+# test ones authorized
+test_authorized_list = (
+    ("Invalid vnfd id", "GET", "/vnfpkgm/v1/vnf_packages/non-existing-id", headers_json, None, 404, r_header_json, "json"),
+    ("Invalid nsd id", "GET", "/nsd/v1/ns_descriptors/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"),
+    ("Invalid nsd id", "DELETE", "/nsd/v1/ns_descriptors_content/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"),
+)
+
+class TestException(Exception):
+    pass
+
+
+class TestRest:
+    def __init__(self, url_base, header_base={}, verify=False):
+        self.url_base = url_base
+        self.header_base = header_base
+        self.s = requests.session()
+        self.s.headers = header_base
+        self.verify = verify
+
+    def set_header(self, header):
+        self.s.headers.update(header)
+
+    def test(self, name, method, url, headers, payload, expected_codes, expected_headers, expected_payload):
+        """
+        Performs an http request and check http code response. Exit if different than allowed
+        :param name:  name of the test
+        :param method: HTTP method: GET,PUT,POST,DELETE,...
+        :param url: complete URL or relative URL
+        :param headers: request headers to add to the base headers
+        :param payload: Can be a dict, transformed to json, a text or a file if starts with '@'
+        :param expected_codes: expected response codes, can be int, int tuple or int range
+        :param expected_headers: expected response headers, dict with key values
+        :param expected_payload: expected payload, 0 if empty, 'yaml', 'json', 'text', 'zip'
+        :return:
+        """
+        try:
+            if not self.s:
+                self.s = requests.session()
+            if not url:
+                url = self.url_base
+            elif not url.startswith("http"):
+                url = self.url_base + url
+            if payload:
+                if isinstance(payload, str):
+                    if payload.startswith("@"):
+                        mode = "r"
+                        file_name = payload[1:]
+                        if payload.startswith("@b"):
+                            mode = "rb"
+                            file_name = payload[2:]
+                        with open(file_name, mode) as f:
+                            payload = f.read()
+                elif isinstance(payload, dict):
+                    payload = json.dumps(payload)
+    
+            test = "Test {} {} {}".format(name, method, url)
+            logger.warning(test)
+            stream = False
+            # if expected_payload == "zip":
+            #     stream = True
+            r = getattr(self.s, method.lower())(url, data=payload, headers=headers, verify=self.verify, stream=stream)
+            logger.debug("RX {}: {}".format(r.status_code, r.text))
+
+            # check response
+            if expected_codes:
+                if isinstance(expected_codes, int):
+                    expected_codes = (expected_codes,)
+                if r.status_code not in expected_codes:
+                    raise TestException(
+                        "Got status {}. Expected {}. {}".format(r.status_code, expected_codes, r.text))
+
+            if expected_headers:
+                for header_key, header_val in expected_headers.items():
+                    if header_key.lower() not in r.headers:
+                        raise TestException("Header {} not present".format(header_key))
+                    if header_val and header_val.lower() not in r.headers[header_key]:
+                        raise TestException("Header {} does not contain {} but {}".format(header_key, header_val,
+                                            r.headers[header_key]))
+
+            if expected_payload is not None:
+                if expected_payload == 0 and len(r.content) > 0:
+                    raise TestException("Expected empty payload")
+                elif expected_payload == "json":
+                    try:
+                        r.json()
+                    except Exception as e:
+                        raise TestException("Expected json response payload, but got Exception {}".format(e))
+                elif expected_payload == "yaml":
+                    try:
+                        yaml.safe_load(r.text)
+                    except Exception as e:
+                        raise TestException("Expected yaml response payload, but got Exception {}".format(e))
+                elif expected_payload == "zip":
+                    if len(r.content) == 0:
+                        raise TestException("Expected some response payload, but got empty")
+                    # try:
+                    #     tar = tarfile.open(None, 'r:gz', fileobj=r.raw)
+                    #     for tarinfo in tar:
+                    #         tarname = tarinfo.name
+                    #         print(tarname)
+                    # except Exception as e:
+                    #     raise TestException("Expected zip response payload, but got Exception {}".format(e))
+                elif expected_payload == "text":
+                    if len(r.content) == 0:
+                        raise TestException("Expected some response payload, but got empty")
+                    #r.text
+            return r
+        except TestException as e:
+            logger.error("{} \nRX code{}: {}".format(e, r.status_code, r.text))
+            exit(1)
+        except IOError as e:
+            logger.error("Cannot open file {}".format(e))
+            exit(1)
+
+
+if __name__ == "__main__":
+    global logger
+    test = ""
+    try:
+        logging.basicConfig(format="%(levelname)s %(message)s", level=logging.ERROR)
+        logger = logging.getLogger('NBI')
+        # load parameters and configuration
+        opts, args = getopt.getopt(sys.argv[1:], "hvu:p:",
+                                   ["url=", "user=", "password=", "help", "version", "verbose", "project=", "insecure"])
+        url = "https://localhost:9999/osm"
+        user = password = project = "admin"
+        verbose = 0
+        verify = True
+
+        for o, a in opts:
+            if o == "--version":
+                print ("test version " + __version__ + ' ' + version_date)
+                sys.exit()
+            elif o in ("-v", "--verbose"):
+                verbose += 1
+            elif o in ("no-verbose"):
+                verbose = -1
+            elif o in ("-h", "--help"):
+                usage()
+                sys.exit()
+            elif o in ("--url"):
+                url = a
+            elif o in ("-u", "--user"):
+                user = a
+            elif o in ("-p", "--password"):
+                password = a
+            elif o in ("--project"):
+                project = a
+            elif o in ("--insecure"):
+                verify = False
+            else:
+                assert False, "Unhandled option"
+        if verbose == 0:
+            logger.setLevel(logging.WARNING)
+        elif verbose > 1:
+            logger.setLevel(logging.DEBUG)
+        else:
+            logger.setLevel(logging.ERROR)
+
+        test_rest = TestRest(url)
+
+        # tests without authorization
+        for t in test_not_authorized_list:
+            test_rest.test(*t)
+
+        # get token
+        r = test_rest.test("Obtain token", "POST", "/admin/v1/tokens", headers_json,
+                           {"username": user, "password": password, "project_id": project},
+                           (200, 201), {"Content-Type": "application/json"}, "json")
+        response = r.json()
+        token = response["id"]
+        test_rest.set_header({"Authorization": "Bearer {}".format(token)})
+
+        # tests once authorized
+        for t in test_authorized_list:
+            test_rest.test(*t)
+
+        # nsd CREATE
+        r = test_rest.test("Onboard NSD step 1", "POST", "/nsd/v1/ns_descriptors", headers_json, None,
+                           201, {"Location": "/nsd/v1/ns_descriptors/", "Content-Type": "application/json"}, "json")
+        location = r.headers["Location"]
+        nsd_id = location[location.rfind("/")+1:]
+        # print(location, nsd_id)
+
+        # nsd UPLOAD test
+        r = test_rest.test("Onboard NSD step 2 as TEXT", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           r_header_text, "@./cirros_ns/cirros_nsd.yaml", 204, None, 0)
+
+        # nsd SHOW OSM format
+        r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # nsd SHOW text
+        r = test_rest.test("Show NSD SOL005 text", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           headers_text, None, 200, r_header_text, "text")
+
+        # nsd UPLOAD ZIP
+        makedirs("temp", exist_ok=True)
+        tar = tarfile.open("temp/cirros_ns.tar.gz", "w:gz")
+        tar.add("cirros_ns")
+        tar.close()
+        r = test_rest.test("Onboard NSD step 3 replace with ZIP", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           r_header_zip, "@b./temp/cirros_ns.tar.gz", 204, None, 0)
+
+        # nsd SHOW OSM format
+        r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # nsd SHOW zip
+        r = test_rest.test("Show NSD SOL005 zip", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           headers_zip, None, 200, r_header_zip, "zip")
+
+        # nsd SHOW descriptor
+        r = test_rest.test("Show NSD descriptor", "GET", "/nsd/v1/ns_descriptors/{}/nsd".format(nsd_id),
+                           headers_text, None, 200, r_header_text, "text")
+        # nsd SHOW actifact
+        r = test_rest.test("Show NSD artifact", "GET", "/nsd/v1/ns_descriptors/{}/artifacts/icons/osm_2x.png".format(nsd_id),
+                           headers_text, None, 200, r_header_octect, "text")
+
+        # nsd DELETE
+        r = test_rest.test("Delete NSD SOL005 text", "DELETE", "/nsd/v1/ns_descriptors/{}".format(nsd_id),
+                           headers_yaml, None, 204, None, 0)
+
+        # vnfd CREATE
+        r = test_rest.test("Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages", headers_json, None,
+                           201, {"Location": "/vnfpkgm/v1/vnf_packages/", "Content-Type": "application/json"}, "json")
+        location = r.headers["Location"]
+        vnfd_id = location[location.rfind("/")+1:]
+        # print(location, vnfd_id)
+
+        # vnfd UPLOAD test
+        r = test_rest.test("Onboard VNFD step 2 as TEXT", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           r_header_text, "@./cirros_vnf/cirros_vnfd.yaml", 204, None, 0)
+
+        # vnfd SHOW OSM format
+        r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # vnfd SHOW text
+        r = test_rest.test("Show VNFD SOL005 text", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           headers_text, None, 200, r_header_text, "text")
+
+        # vnfd UPLOAD ZIP
+        makedirs("temp", exist_ok=True)
+        tar = tarfile.open("temp/cirros_vnf.tar.gz", "w:gz")
+        tar.add("cirros_vnf")
+        tar.close()
+        r = test_rest.test("Onboard VNFD step 3 replace with ZIP", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           r_header_zip, "@b./temp/cirros_vnf.tar.gz", 204, None, 0)
+
+        # vnfd SHOW OSM format
+        r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # vnfd SHOW zip
+        r = test_rest.test("Show VNFD SOL005 zip", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           headers_zip, None, 200, r_header_zip, "zip")
+        # vnfd SHOW descriptor
+        r = test_rest.test("Show VNFD descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(vnfd_id),
+                           headers_text, None, 200, r_header_text, "text")
+        # vnfd SHOW actifact
+        r = test_rest.test("Show VNFD artifact", "GET", "/vnfpkgm/v1/vnf_packages/{}/artifacts/icons/cirros-64.png".format(vnfd_id),
+                           headers_text, None, 200, r_header_octect, "text")
+
+        # vnfd DELETE
+        r = test_rest.test("Delete VNFD SOL005 text", "DELETE", "/vnfpkgm/v1/vnf_packages/{}".format(vnfd_id),
+                           headers_yaml, None, 204, None, 0)
+
+        print("PASS")
+
+    except Exception as e:
+        if test:
+            logger.error(test + " Exception: " + str(e))
+            exit(1)
+        else:
+            logger.critical(test + " Exception: " + str(e), exc_info=True)