sol005 packages upload implementation 75/5875/1
authortierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 12 Mar 2018 16:08:42 +0000 (17:08 +0100)
committertierno <alfonso.tiernosepulveda@telefonica.com>
Mon, 12 Mar 2018 16:08:42 +0000 (17:08 +0100)
Change-Id: I7bba91831d0e29e1cd874d28a566de18eb113994
Signed-off-by: tierno <alfonso.tiernosepulveda@telefonica.com>
13 files changed:
.gitignore-common
osm_nbi/dbmongo.py
osm_nbi/engine.py
osm_nbi/fslocal.py
osm_nbi/html_out.py
osm_nbi/nbi.py
osm_nbi/test/cirros_ns/cirros_nsd.yaml [new file with mode: 0644]
osm_nbi/test/cirros_ns/icons/osm_2x.png [new file with mode: 0644]
osm_nbi/test/cirros_vnf/cirros_vnfd.yaml [new file with mode: 0644]
osm_nbi/test/cirros_vnf/icons/cirros-64.png [new file with mode: 0644]
osm_nbi/test/create-ping-pong.sh
osm_nbi/test/delete-all.sh
osm_nbi/test/test.py [new file with mode: 0755]

index 77f6798..62bb57e 100644 (file)
@@ -1,24 +1,3 @@
-##
-# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
-# This file is part of openmano
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
-# For those usages not covered by the Apache License, Version 2.0 please
-# contact with: nfvlabs@tid.es
-##
-
 # This is a template with common files to be igonored, after clone make a copy to .gitignore
 # cp .gitignore-common .gitignore
 
 # This is a template with common files to be igonored, after clone make a copy to .gitignore
 # cp .gitignore-common .gitignore
 
@@ -28,7 +7,7 @@
 #auto-ignore
 .gitignore
 
 #auto-ignore
 .gitignore
 
-#logs of openmano
+#logs
 logs 
 
 #pycharm
 logs 
 
 #pycharm
@@ -41,17 +20,12 @@ logs
 
 #local stuff files that end in ".local" or folders called "local"
 *.local
 
 #local stuff files that end in ".local" or folders called "local"
 *.local
-vnfs/*.local
-test/*.local
-scenarios/*.local
-instance-scenarios/*.local
-database_utils/*.local
-scripts/*.local
-local
-vnfs/local
-test/local
-scenarios/local
-instance-scenarios/local
-database_utils/local
-scripts/local
+osm_nbi/local
+osm_nbi/test/local
+
+#local stuff files that end in ".temp" or folders called "temp"
+*.temp
+osm_nbi/temp
+osm_nbi/test/temp
+
 
 
index ffab5c7..a8ea1ca 100644 (file)
@@ -21,6 +21,7 @@ __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 #                 sleep(retry)
 #     return _retry_mongocall
 
 #                 sleep(retry)
 #     return _retry_mongocall
 
+
 class DbMongo(DbBase):
     conn_initial_timout = 120
     conn_timout = 10
 class DbMongo(DbBase):
     conn_initial_timout = 120
     conn_timout = 10
@@ -63,7 +64,7 @@ class DbMongo(DbBase):
                                                                "ncont", "neq"):
                     operator = "$" + query_k[dot_index+1:]
                     if operator == "$neq":
                                                                "ncont", "neq"):
                     operator = "$" + query_k[dot_index+1:]
                     if operator == "$neq":
-                        operator = "$nq"
+                        operator = "$ne"
                     k = query_k[:dot_index]
                 else:
                     operator = "$eq"
                     k = query_k[:dot_index]
                 else:
                     operator = "$eq"
@@ -97,7 +98,6 @@ class DbMongo(DbBase):
             raise DbException("Invalid query string filter at {}:{}. Error: {}".format(query_k, v, e),
                               http_code=HTTPStatus.BAD_REQUEST)
 
             raise DbException("Invalid query string filter at {}:{}. Error: {}".format(query_k, v, e),
                               http_code=HTTPStatus.BAD_REQUEST)
 
-
     def get_list(self, table, filter={}):
         try:
             l = []
     def get_list(self, table, filter={}):
         try:
             l = []
@@ -121,11 +121,12 @@ class DbMongo(DbBase):
             rows = collection.find(filter)
             if rows.count() == 0:
                 if fail_on_empty:
             rows = collection.find(filter)
             if rows.count() == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             elif rows.count() > 1:
                 if fail_on_more:
                 return None
             elif rows.count() > 1:
                 if fail_on_more:
-                    raise DbException("Found more than one entry with filter='{}'".format(filter),
+                    raise DbException("Found more than one {} with filter='{}'".format(table[:-1], filter),
                                       HTTPStatus.CONFLICT)
             return rows[0]
         except Exception as e:  # TODO refine
                                       HTTPStatus.CONFLICT)
             return rows[0]
         except Exception as e:  # TODO refine
@@ -147,7 +148,8 @@ class DbMongo(DbBase):
             rows = collection.delete_one(self._format_filter(filter))
             if rows.deleted_count == 0:
                 if fail_on_empty:
             rows = collection.delete_one(self._format_filter(filter))
             if rows.deleted_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
@@ -167,7 +169,8 @@ class DbMongo(DbBase):
             rows = collection.update_one(self._format_filter(filter), {"$set": update_dict})
             if rows.updated_count == 0:
                 if fail_on_empty:
             rows = collection.update_one(self._format_filter(filter), {"$set": update_dict})
             if rows.updated_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
                 return None
             return {"deleted": rows.deleted_count}
         except Exception as e:  # TODO refine
@@ -179,7 +182,8 @@ class DbMongo(DbBase):
             rows = collection.replace_one({"_id": id}, indata)
             if rows.modified_count == 0:
                 if fail_on_empty:
             rows = collection.replace_one({"_id": id}, indata)
             if rows.modified_count == 0:
                 if fail_on_empty:
-                    raise DbException("Not found entry with filter='{}'".format(filter), HTTPStatus.NOT_FOUND)
+                    raise DbException("Not found any {} with filter='{}'".format(table[:-1], filter),
+                                      HTTPStatus.NOT_FOUND)
                 return None
             return {"replace": rows.modified_count}
         except Exception as e:  # TODO refine
                 return None
             return {"replace": rows.modified_count}
         except Exception as e:  # TODO refine
index c35617b..a15420f 100644 (file)
@@ -29,6 +29,29 @@ class EngineException(Exception):
         Exception.__init__(self, message)
 
 
         Exception.__init__(self, message)
 
 
+def _deep_update(dict_to_change, dict_reference):
+    """
+    Modifies one dictionary with the information of the other following https://tools.ietf.org/html/rfc7396
+    :param dict_to_change:  Ends modified
+    :param dict_reference: reference
+    :return: none
+    """
+
+    for k in dict_reference:
+        if dict_reference[k] is None:   # None->Anything
+            if k in dict_to_change:
+                del dict_to_change[k]
+        elif not isinstance(dict_reference[k], dict):  #  NotDict->Anything
+            dict_to_change[k] = dict_reference[k]
+        elif k not in dict_to_change:  # Dict->Empty
+            dict_to_change[k] = deepcopy(dict_reference[k])
+            _deep_update(dict_to_change[k], dict_reference[k])
+        elif isinstance(dict_to_change[k], dict):  # Dict->Dict
+            _deep_update(dict_to_change[k], dict_reference[k])
+        else:       # Dict->NotDict
+            dict_to_change[k] = deepcopy(dict_reference[k])
+            _deep_update(dict_to_change[k], dict_reference[k])
+
 class Engine(object):
 
     def __init__(self):
 class Engine(object):
 
     def __init__(self):
@@ -189,7 +212,7 @@ class Engine(object):
         """
         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
         vnfd or nsd content
         """
         Obtain the useful data removing the envelop. It goes throw the vnfd or nsd catalog and returns the
         vnfd or nsd content
-        :param item: can be vnfds, nsds, users, projects,
+        :param item: can be vnfds, nsds, users, projects, userDefinedData (initial content of a vnfds, nsds
         :param indata: Content to be inspected
         :return: the useful part of indata
         """
         :param indata: Content to be inspected
         :return: the useful part of indata
         """
@@ -214,9 +237,12 @@ class Engine(object):
                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
                     raise EngineException("'nsd' must be a list only one element")
                 clean_indata = clean_indata['nsd'][0]
                 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
                     raise EngineException("'nsd' must be a list only one element")
                 clean_indata = clean_indata['nsd'][0]
+        elif item == "userDefinedData":
+            if "userDefinedData" in indata:
+                clean_indata = clean_indata['userDefinedData']
         return clean_indata
 
         return clean_indata
 
-    def _validate_new_data(self, session, item, indata):
+    def _validate_new_data(self, session, item, indata, id=None):
         if item == "users":
             if not indata.get("username"):
                 raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY)
         if item == "users":
             if not indata.get("username"):
                 raise EngineException("missing 'username'", HTTPStatus.UNPROCESSABLE_ENTITY)
@@ -233,8 +259,10 @@ class Engine(object):
             # check name not exist
             if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
                 raise EngineException("name '{}' exist".format(indata["name"]), HTTPStatus.CONFLICT)
             # check name not exist
             if self.db.get_one(item, {"name": indata.get("name")}, fail_on_empty=False, fail_on_more=False):
                 raise EngineException("name '{}' exist".format(indata["name"]), HTTPStatus.CONFLICT)
-        elif item == "vnfds" or item == "nsds":
+        elif item in ("vnfds", "nsds"):
             filter = {"id": indata["id"]}
             filter = {"id": indata["id"]}
+            if id:
+                filter["_id.neq"] = id
             # TODO add admin to filter, validate rights
             self._add_read_filter(session, item, filter)
             if self.db.get_one(item, filter, fail_on_empty=False):
             # TODO add admin to filter, validate rights
             self._add_read_filter(session, item, filter)
             if self.db.get_one(item, filter, fail_on_empty=False):
@@ -242,6 +270,10 @@ class Engine(object):
                                       HTTPStatus.CONFLICT)
 
             # TODO validate with pyangbind
                                       HTTPStatus.CONFLICT)
 
             # TODO validate with pyangbind
+        elif item == "userDefinedData":
+            # TODO validate userDefinedData is a keypair values
+            pass
+
         elif item == "nsrs":
             pass
 
         elif item == "nsrs":
             pass
 
@@ -254,7 +286,7 @@ class Engine(object):
         if item == "users":
             _id = indata["username"]
             salt = uuid4().hex
         if item == "users":
             _id = indata["username"]
             salt = uuid4().hex
-            indata["_admin"]["salt"] =  salt
+            indata["_admin"]["salt"] = salt
             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
         elif item == "projects":
             _id = indata["name"]
             indata["password"] = sha256(indata["password"].encode('utf-8') + salt.encode('utf-8')).hexdigest()
         elif item == "projects":
             _id = indata["name"]
@@ -266,36 +298,44 @@ class Engine(object):
                 storage = admin.get("storage")
             if not _id:
                 _id = str(uuid4())
                 storage = admin.get("storage")
             if not _id:
                 _id = str(uuid4())
-            if item == "vnfds" or item == "nsds":
+            if item in ("vnfds", "nsds"):
                 if not indata["_admin"].get("projects_read"):
                     indata["_admin"]["projects_read"] = [session["project_id"]]
                 if not indata["_admin"].get("projects_write"):
                     indata["_admin"]["projects_write"] = [session["project_id"]]
                 if not indata["_admin"].get("projects_read"):
                     indata["_admin"]["projects_read"] = [session["project_id"]]
                 if not indata["_admin"].get("projects_write"):
                     indata["_admin"]["projects_write"] = [session["project_id"]]
+                indata["_admin"]["onboardingState"] = "CREATED"
+                indata["_admin"]["operationalState"] = "DISABLED"
+                indata["_admin"]["usageSate"] = "NOT_IN_USE"
             if storage:
                 indata["_admin"]["storage"] = storage
         indata["_id"] = _id
 
             if storage:
                 indata["_admin"]["storage"] = storage
         indata["_id"] = _id
 
-    def _new_item_partial(self, session, item, indata, headers):
+    def upload_content(self, session, item, _id, indata, kwargs, headers):
         """
         """
-        Used for recieve content by chunks (with a transaction_id header and/or gzip file. It will store and extract
+        Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
         :param session: session
         :param session: session
-        :param item:
+        :param item: can be nsds or vnfds
+        :param _id : the nsd,vnfd is already created, this is the id
         :param indata: http body request
         :param indata: http body request
+        :param kwargs: user query string to override parameters. NOT USED
         :param headers:  http request headers
         :param headers:  http request headers
-        :return: a dict with::
-            _id: <transaction_id>
-            storage: <path>:  where it is saving
-            desc: <dict>: descriptor: Only present when all the content is received, extracted and read the descriptor
+        :return: True package has is completely uploaded or False if partial content has been uplodaed.
+            Raise exception on error
         """
         """
+        # Check that _id exist and it is valid
+        current_desc = self.get_item(session, item, _id)
+
         content_range_text = headers.get("Content-Range")
         content_range_text = headers.get("Content-Range")
-        transaction_id = headers.get("Transaction-Id")
-        filename = headers.get("Content-Filename", "pkg")
-        # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
         expected_md5 = headers.get("Content-File-MD5")
         compressed = None
         expected_md5 = headers.get("Content-File-MD5")
         compressed = None
-        if "application/gzip" in headers.get("Content-Type") or "application/x-gzip" in headers.get("Content-Type") or \
-                "application/zip" in headers.get("Content-Type"):
+        content_type = headers.get("Content-Type")
+        if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
+                "application/zip" in content_type:
             compressed = "gzip"
             compressed = "gzip"
+        filename = headers.get("Content-Filename")
+        if not filename:
+            filename = "package.tar.gz" if compressed else "package"
+        # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
         file_pkg = None
         error_text = ""
         try:
         file_pkg = None
         error_text = ""
         try:
@@ -306,41 +346,48 @@ class Engine(object):
                 start = int(content_range[1])
                 end = int(content_range[2]) + 1
                 total = int(content_range[3])
                 start = int(content_range[1])
                 end = int(content_range[2]) + 1
                 total = int(content_range[3])
-                if len(indata) != end-start:
-                    raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
-                        start, end-1, len(indata)), HTTPStatus.BAD_REQUEST)
             else:
                 start = 0
             else:
                 start = 0
-                total = end = len(indata)
-            if not transaction_id:
-                # generate transaction
-                transaction_id = str(uuid4())
-                self.fs.mkdir(transaction_id)
-                # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'wb')
-                # control = {"received": 0}
-            elif not self.fs.file_exists(transaction_id):
-                raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
+
+            if start:
+                if not self.fs.file_exists(_id, 'dir'):
+                    raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
             else:
             else:
-                pass
-                # control_file = open(self.storage["path"] + transaction_id + "/.osm.yaml", 'rw')
-                # control = yaml.load(control_file)
-                # control_file.seek(0, 0)
+                self.fs.file_delete(_id, ignore_non_exist=True)
+                self.fs.mkdir(_id)
+
             storage = self.fs.get_params()
             storage = self.fs.get_params()
-            storage["folder"] = transaction_id
-            storage["file"] = filename
+            storage["folder"] = _id
 
 
-            file_path = (transaction_id, filename)
-            if self.fs.file_exists(file_path):
+            file_path = (_id, filename)
+            if self.fs.file_exists(file_path, 'file'):
                 file_size = self.fs.file_size(file_path)
             else:
                 file_size = 0
             if file_size != start:
                 file_size = self.fs.file_size(file_path)
             else:
                 file_size = 0
             if file_size != start:
-                raise EngineException("invalid upload transaction sequence, expected '{}' but received '{}'".format(
-                    file_size, start), HTTPStatus.BAD_REQUEST)
+                raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
+                    file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
             file_pkg = self.fs.file_open(file_path, 'a+b')
             file_pkg = self.fs.file_open(file_path, 'a+b')
-            file_pkg.write(indata)
-            if end != total:
-                return {"_id": transaction_id, "storage": storage}
+            if isinstance(indata, dict):
+                indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
+                file_pkg.write(indata_text.encode(encoding="utf-8"))
+            else:
+                indata_len = 0
+                while True:
+                    indata_text = indata.read(4096)
+                    indata_len += len(indata_text)
+                    if not indata_text:
+                        break
+                    file_pkg.write(indata_text)
+            if content_range_text:
+                if indata_len != end-start:
+                    raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
+                        start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
+                if end != total:
+                    # TODO update to UPLOADING
+                    return False
+
+            # PACKAGE UPLOADED
             if expected_md5:
                 file_pkg.seek(0, 0)
                 file_md5 = md5()
             if expected_md5:
                 file_pkg.seek(0, 0)
                 file_md5 = md5()
@@ -352,8 +399,6 @@ class Engine(object):
                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
             file_pkg.seek(0, 0)
             if compressed == "gzip":
                     raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
             file_pkg.seek(0, 0)
             if compressed == "gzip":
-                # TODO unzip,
-                storage["tarfile"] = filename
                 tar = tarfile.open(mode='r', fileobj=file_pkg)
                 descriptor_file_name = None
                 for tarinfo in tar:
                 tar = tarfile.open(mode='r', fileobj=file_pkg)
                 descriptor_file_name = None
                 for tarinfo in tar:
@@ -364,32 +409,43 @@ class Engine(object):
                     if len(tarname_path) == 1 and not tarinfo.isdir():
                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
                     if len(tarname_path) == 1 and not tarinfo.isdir():
                         raise EngineException("All files must be inside a dir for package descriptor tar.gz")
                     if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
-                        storage["file"] = tarname_path[0]
+                        storage["pkg-dir"] = tarname_path[0]
                         if len(tarname_path) == 2:
                             if descriptor_file_name:
                                 raise EngineException("Found more than one descriptor file at package descriptor tar.gz")
                             descriptor_file_name = tarname
                 if not descriptor_file_name:
                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
                         if len(tarname_path) == 2:
                             if descriptor_file_name:
                                 raise EngineException("Found more than one descriptor file at package descriptor tar.gz")
                             descriptor_file_name = tarname
                 if not descriptor_file_name:
                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
-                self.fs.file_extract(tar, transaction_id)
-                with self.fs.file_open((transaction_id, descriptor_file_name), "r") as descriptor_file:
+                storage["descriptor"] = descriptor_file_name
+                storage["zipfile"] = filename
+                self.fs.file_extract(tar, _id)
+                with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
                     content = descriptor_file.read()
             else:
                 content = file_pkg.read()
                     content = descriptor_file.read()
             else:
                 content = file_pkg.read()
-                tarname = ""
+                storage["descriptor"] = descriptor_file_name = filename
 
 
-            if tarname.endswith(".json"):
+            if descriptor_file_name.endswith(".json"):
                 error_text = "Invalid json format "
                 indata = json.load(content)
             else:
                 error_text = "Invalid yaml format "
                 indata = yaml.load(content)
                 error_text = "Invalid json format "
                 indata = json.load(content)
             else:
                 error_text = "Invalid yaml format "
                 indata = yaml.load(content)
-            return {"_id": transaction_id, "storage": storage, "desc": indata}
+
+            current_desc["_admin"]["storage"] = storage
+            current_desc["_admin"]["onboardingState"] = "ONBOARDED"
+            current_desc["_admin"]["operationalState"] = "ENABLED"
+
+            self._edit_item(session, item, _id, current_desc, indata, kwargs)
+            # TODO if descriptor has changed because kwargs update content and remove cached zip
+            # TODO if zip is not present creates one
+            return True
+
         except EngineException:
             raise
         except IndexError:
             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
         except EngineException:
             raise
         except IndexError:
             raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
-                                  HTTPStatus.BAD_REQUEST)
+                                  HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
         except IOError as e:
             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
         except (ValueError, yaml.YAMLError) as e:
         except IOError as e:
             raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
         except (ValueError, yaml.YAMLError) as e:
@@ -441,9 +497,10 @@ class Engine(object):
 
     def new_item(self, session, item, indata={}, kwargs=None, headers={}):
         """
 
     def new_item(self, session, item, indata={}, kwargs=None, headers={}):
         """
-        Creates a new entry into database
+        Creates a new entry into database. For nsds and vnfds it creates an almost empty DISABLED  entry,
+        that must be completed with a call to method upload_content
         :param session: contains the used login username and working project
         :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param item: it can be: users, projects, nsrs, nsds, vnfds
         :param indata: data to be inserted
         :param kwargs: used to override the indata descriptor
         :param headers: http request headers
         :param indata: data to be inserted
         :param kwargs: used to override the indata descriptor
         :param headers: http request headers
@@ -453,16 +510,20 @@ class Engine(object):
         # TODO add admin and status
 
         transaction = None
         # TODO add admin and status
 
         transaction = None
-        if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \
-            "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type"):
-            if not indata:
-                raise EngineException("Empty payload")
-            transaction = self._new_item_partial(session, item, indata, headers)
-            if "desc" not in transaction:
-                return transaction["_id"], False
-            indata = transaction["desc"]
-
-        content = self._remove_envelop(item, indata)
+        # if headers.get("Content-Range") or "application/gzip" in headers.get("Content-Type") or \
+        #     "application/x-gzip" in headers.get("Content-Type") or "application/zip" in headers.get("Content-Type") or \
+        #     "text/plain" in headers.get("Content-Type"):
+        #     if not indata:
+        #         raise EngineException("Empty payload")
+        #     transaction = self._new_item_partial(session, item, indata, headers)
+        #     if "desc" not in transaction:
+        #         return transaction["_id"], False
+        #     indata = transaction["desc"]
+
+        item_envelop = item
+        if item in ("nsds", "vnfds"):
+            item_envelop = "userDefinedData"
+        content = self._remove_envelop(item_envelop, indata)
 
         # Override descriptor with query string kwargs
         if kwargs:
 
         # Override descriptor with query string kwargs
         if kwargs:
@@ -491,7 +552,7 @@ class Engine(object):
             except IndexError:
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
             except IndexError:
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
-        if not indata:
+        if not indata and item not in ("nsds", "vnfds"):
             raise EngineException("Empty payload")
 
         if item == "nsrs":
             raise EngineException("Empty payload")
 
         if item == "nsrs":
@@ -500,12 +561,14 @@ class Engine(object):
             content, _id = self.new_nsr(session, ns_request)
             transaction = {"_id": _id}
 
             content, _id = self.new_nsr(session, ns_request)
             transaction = {"_id": _id}
 
-        self._validate_new_data(session, item, content)
+        self._validate_new_data(session, item_envelop, content)
+        if item in ("nsds", "vnfds"):
+            content = {"_admin": {"userDefinedData": content}}
         self._format_new_data(session, item, content, transaction)
         _id = self.db.create(item, content)
         if item == "nsrs":
             self.msg.write("ns", "create", _id)
         self._format_new_data(session, item, content, transaction)
         _id = self.db.create(item, content)
         if item == "nsrs":
             self.msg.write("ns", "create", _id)
-        return _id, True
+        return _id
 
     def _add_read_filter(self, session, item, filter):
         if session["project_id"] == "admin":  # allows all
 
     def _add_read_filter(self, session, item, filter):
         if session["project_id"] == "admin":  # allows all
@@ -527,6 +590,62 @@ class Engine(object):
         elif item in ("vnfds", "nsds") and session["project_id"] != "admin":
             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 
         elif item in ("vnfds", "nsds") and session["project_id"] != "admin":
             filter["_admin.projects_write.cont"] = ["ANY", session["project_id"]]
 
+    def get_file(self, session, item, _id, path=None, accept_header=None):
+        """
+        Return the file content of a vnfd or nsd
+        :param session: contains the used login username and working project
+        :param item: it can be vnfds or nsds
+        :param _id: Identity of the vnfd, ndsd
+        :param path: artifact path or "$DESCRIPTOR" or None
+        :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
+        :return: opened file or raises an exception
+        """
+        accept_text = accept_zip = False
+        if accept_header:
+            if 'text/plain' in accept_header or '*/*' in accept_header:
+                accept_text = True
+            if 'application/zip' in accept_header or '*/*' in accept_header:
+                accept_zip = True
+        if not accept_text and not accept_zip:
+            raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
+                                  http_code=HTTPStatus.NOT_ACCEPTABLE)
+
+        content = self.get_item(session, item, _id)
+        if content["_admin"]["onboardingState"] != "ONBOARDED":
+            raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
+                "onboardingState is {}".format(content["_admin"]["onboardingState"]),
+                                  http_code=HTTPStatus.CONFLICT)
+        storage = content["_admin"]["storage"]
+        if path is not None and path != "$DESCRIPTOR":   # artifacts
+            if not storage.get('pkg-dir'):
+                raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
+            if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
+                folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
+                return folder_content, "text/plain"
+                # TODO manage folders in http
+            else:
+                return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"), \
+                       "application/octet-stream"
+
+        # pkgtype   accept  ZIP  TEXT    -> result
+        # manyfiles         yes  X       -> zip
+        #                   no   yes     -> error
+        # onefile           yes  no      -> zip
+        #                   X    yes     -> text
+
+        if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
+            return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
+        elif storage.get('pkg-dir') and not accept_zip:
+            raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
+                                      "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
+        else:
+            if not storage.get('zipfile'):
+                # TODO generate zipfile if not present
+                raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in future versions"
+                                      "", http_code=HTTPStatus.NOT_ACCEPTABLE)
+            return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), "application/zip"
+
+
     def get_item_list(self, session, item, filter={}):
         """
         Get a list of items
     def get_item_list(self, session, item, filter={}):
         """
         Get a list of items
@@ -536,6 +655,9 @@ class Engine(object):
         :return: The list, it can be empty if no one match the filter.
         """
         # TODO add admin to filter, validate rights
         :return: The list, it can be empty if no one match the filter.
         """
         # TODO add admin to filter, validate rights
+        # TODO transform data for SOL005 URL requests. Transform filtering
+        # TODO implement "field-type" query string SOL005
+
         self._add_read_filter(session, item, filter)
         return self.db.get_list(item, filter)
 
         self._add_read_filter(session, item, filter)
         return self.db.get_list(item, filter)
 
@@ -543,12 +665,14 @@ class Engine(object):
         """
         Get complete information on an items
         :param session: contains the used login username and working project
         """
         Get complete information on an items
         :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param item: it can be: users, projects, vnfds, nsds,
         :param _id: server id of the item
         :return: dictionary, raise exception if not found.
         """
         :param _id: server id of the item
         :return: dictionary, raise exception if not found.
         """
+        database_item = item
         filter = {"_id": _id}
         # TODO add admin to filter, validate rights
         filter = {"_id": _id}
         # TODO add admin to filter, validate rights
+        # TODO transform data for SOL005 URL requests
         self._add_read_filter(session, item, filter)
         return self.db.get_one(item, filter)
 
         self._add_read_filter(session, item, filter)
         return self.db.get_one(item, filter)
 
@@ -611,27 +735,15 @@ class Engine(object):
         _id = self.db.create("users", indata)
         return _id
 
         _id = self.db.create("users", indata)
         return _id
 
-    def edit_item(self, session, item, id, indata={}, kwargs=None):
-        """
-        Update an existing entry at database
-        :param session: contains the used login username and working project
-        :param item: it can be: users, projects, vnfds, nsds, ...
-        :param id: identity of entry to be updated
-        :param indata: data to be inserted
-        :param kwargs: used to override the indata descriptor
-        :return: dictionary, raise exception if not found.
-        """
-
-        content = self.get_item(session, item, id)
+    def _edit_item(self, session, item, id, content, indata={}, kwargs=None):
         if indata:
             indata = self._remove_envelop(item, indata)
         if indata:
             indata = self._remove_envelop(item, indata)
-            # TODO update content with with a deep-update
 
         # Override descriptor with query string kwargs
         if kwargs:
             try:
                 for k, v in kwargs.items():
 
         # Override descriptor with query string kwargs
         if kwargs:
             try:
                 for k, v in kwargs.items():
-                    update_content = content
+                    update_content = indata
                     kitem_old = None
                     klist = k.split(".")
                     for kitem in klist:
                     kitem_old = None
                     klist = k.split(".")
                     for kitem in klist:
@@ -655,9 +767,25 @@ class Engine(object):
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 
                 raise EngineException(
                     "Invalid query string '{}'. Index '{}' out of  range".format(k, kitem_old))
 
-        self._validate_new_data(session, item, content)
+        _deep_update(content, indata)
+        self._validate_new_data(session, item, content, id)
         # self._format_new_data(session, item, content)
         self.db.replace(item, id, content)
         return id
 
         # self._format_new_data(session, item, content)
         self.db.replace(item, id, content)
         return id
 
+    def edit_item(self, session, item, _id, indata={}, kwargs=None):
+        """
+        Update an existing entry at database
+        :param session: contains the used login username and working project
+        :param item: it can be: users, projects, vnfds, nsds, ...
+        :param _id: identifier to be updated
+        :param indata: data to be inserted
+        :param kwargs: used to override the indata descriptor
+        :return: dictionary, raise exception if not found.
+        """
+
+        content = self.get_item(session, item, _id)
+        return self._edit_item(session, item, _id, content, indata, kwargs)
+
+
 
 
index 10ddf73..b7dd839 100644 (file)
@@ -46,17 +46,23 @@ class FsLocal(FsBase):
         except Exception as e:
             raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 
         except Exception as e:
             raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 
-    def file_exists(self, storage):
+    def file_exists(self, storage, mode=None):
         """
         Indicates if "storage" file exist
         :param storage: can be a str or a str list
         """
         Indicates if "storage" file exist
         :param storage: can be a str or a str list
+        :param mode: can be 'file' exist as a regular file; 'dir' exists as a directory or; 'None' just exists
         :return: True, False
         """
         if isinstance(storage, str):
             f = storage
         else:
             f = "/".join(storage)
         :return: True, False
         """
         if isinstance(storage, str):
             f = storage
         else:
             f = "/".join(storage)
-        return os.path.exists(self.path + f)
+        if os.path.exists(self.path + f):
+            if mode == "file" and os.path.isfile(self.path + f):
+                return True
+            if mode == "dir" and os.path.isdir(self.path + f):
+                return True
+        return False
 
     def file_size(self, storage):
         """
 
     def file_size(self, storage):
         """
@@ -90,11 +96,33 @@ class FsLocal(FsBase):
         :param mode: file mode
         :return: file object
         """
         :param mode: file mode
         :return: file object
         """
-        if isinstance(storage, str):
-            f = storage
-        else:
-            f = "/".join(storage)
-        return open(self.path + f, mode)
+        try:
+            if isinstance(storage, str):
+                f = storage
+            else:
+                f = "/".join(storage)
+            return open(self.path + f, mode)
+        except FileNotFoundError:
+            raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+        except IOError:
+            raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
+
+    def dir_ls(self, storage):
+        """
+        return folder content
+        :param storage: can be a str or list of str
+        :return: folder content
+        """
+        try:
+            if isinstance(storage, str):
+                f = storage
+            else:
+                f = "/".join(storage)
+            return os.listdir(self.path + f)
+        except NotADirectoryError:
+            raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+        except IOError:
+            raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
 
     def file_delete(self, storage, ignore_non_exist=False):
         """
 
     def file_delete(self, storage, ignore_non_exist=False):
         """
@@ -111,4 +139,4 @@ class FsLocal(FsBase):
         if os.path.exists(f):
             rmtree(f)
         elif not ignore_non_exist:
         if os.path.exists(f):
             rmtree(f)
         elif not ignore_non_exist:
-            raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.BAD_REQUEST)
+            raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.NOT_FOUND)
index f6f92d9..f37b5dd 100644 (file)
@@ -19,13 +19,13 @@ html_start = """
     <div>
       <a href="https://osm.etsi.org"> <img src="/osm/static/OSM-logo.png" height="42" width="100" style="vertical-align:middle"> </a>
       <a>( {} )</a>
     <div>
       <a href="https://osm.etsi.org"> <img src="/osm/static/OSM-logo.png" height="42" width="100" style="vertical-align:middle"> </a>
       <a>( {} )</a>
-      <a href="/osm/vnfpkgm/v1/vnf_packages">VNFDs </a>
-      <a href="/osm/nsd/v1/ns_descriptors">NSDs </a>
-      <a href="/osm/nslcm/v1/ns_instances">NSs </a>
-      <a href="/osm/user/v1">USERs </a>
-      <a href="/osm/project/v1">PROJECTs </a>
-      <a href="/osm/token/v1">TOKENs </a>
-      <a href="/osm/token/v1?METHOD=DELETE">logout </a>
+      <a href="/osm/vnfpkgm/v1/vnf_packages_content">VNFDs </a>
+      <a href="/osm/nsd/v1/ns_descriptors_content">NSDs </a>
+      <a href="/osm/nslcm/v1/ns_instances_content">NSs </a>
+      <a href="/osm/admin/v1/users">USERs </a>
+      <a href="/osm/admin/v1/projects">PROJECTs </a>
+      <a href="/osm/admin/v1/tokens">TOKENs </a>
+      <a href="/osm/admin/v1/tokens?METHOD=DELETE">logout </a>
     </div>
   </div>
 """
     </div>
   </div>
 """
@@ -61,7 +61,7 @@ html_auth2 = """
   </div>
   <div class="gerritBody" id="osm_body">
     <h1>Sign in to OSM</h1>
   </div>
   <div class="gerritBody" id="osm_body">
     <h1>Sign in to OSM</h1>
-    <form action="/osm/token/v1" id="login_form" method="POST">
+    <form action="/osm/admin/v1/tokens" id="login_form" method="POST">
       <table style="border: 0;">
         <tr><th>Username</th><td><input id="f_user" name="username" size="25" tabindex="1" type="text"></td></tr>
         <tr><th>Password</th><td><input id="f_pass" name="password" size="25" tabindex="2" type="password"></td></tr>
       <table style="border: 0;">
         <tr><th>Username</th><td><input id="f_user" name="username" size="25" tabindex="1" type="text"></td></tr>
         <tr><th>Password</th><td><input id="f_pass" name="password" size="25" tabindex="2" type="password"></td></tr>
@@ -108,12 +108,15 @@ def format(data, request, response, session):
     if response.status and response.status > 202:
         body += html_body_error.format(yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False))
     elif isinstance(data, (list, tuple)):
     if response.status and response.status > 202:
         body += html_body_error.format(yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False))
     elif isinstance(data, (list, tuple)):
-        if request.path_info == "/vnfpkgm/v1/vnf_packages":
+        if request.path_info == "/vnfpkgm/v1/vnf_packages_content":
             body += html_upload_body.format("VNFD", request.path_info)
             body += html_upload_body.format("VNFD", request.path_info)
-        elif request.path_info == "/nsd/v1/ns_descriptors":
+        elif request.path_info == "/nsd/v1/ns_descriptors_content":
             body += html_upload_body.format("NSD", request.path_info)
         for k in data:
             body += html_upload_body.format("NSD", request.path_info)
         for k in data:
-            data_id = k.pop("_id", None)
+            if isinstance(k, dict):
+                data_id = k.pop("_id", None)
+            elif isinstance(k, str):
+                data_id = k
             body += '<p> <a href="/osm/{url}/{id}">{id}</a>: {t} </p>'.format(url=request.path_info, id=data_id, t=k)
     elif isinstance(data, dict):
         if "Location" in response.headers:
             body += '<p> <a href="/osm/{url}/{id}">{id}</a>: {t} </p>'.format(url=request.path_info, id=data_id, t=k)
     elif isinstance(data, dict):
         if "Location" in response.headers:
index 9cdb409..99f90aa 100644 (file)
@@ -9,49 +9,63 @@ import html_out as html
 import logging
 from engine import Engine, EngineException
 from dbbase import DbException
 import logging
 from engine import Engine, EngineException
 from dbbase import DbException
+from fsbase import FsException
 from base64 import standard_b64decode
 from base64 import standard_b64decode
-from os import getenv
+#from os import getenv
 from http import HTTPStatus
 from http import HTTPStatus
-from http.client import responses as http_responses
+#from http.client import responses as http_responses
 from codecs import getreader
 from os import environ
 
 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 from codecs import getreader
 from os import environ
 
 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
-__version__ = "0.1"
-version_date = "Feb 2018"
+__version__ = "0.2"
+version_date = "Mar 2018"
 
 """
 
 """
-North Bound Interface  (O: OSM; S: SOL5
+North Bound Interface  (O: OSM specific; 5,X: SOL005 not implemented yet; O5: SOL005 implemented)
 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
 URL: /osm                                                       GET     POST    PUT     DELETE  PATCH
-        /nsd/v1
+        /nsd/v1                                                 O       O
+            /ns_descriptors_content                             O       O       
+                /<nsdInfoId>                                    O       O       O       O     
             /ns_descriptors                                     O5      O5
                 /<nsdInfoId>                                    O5                      O5      5
                     /nsd_content                                O5              O5
             /ns_descriptors                                     O5      O5
                 /<nsdInfoId>                                    O5                      O5      5
                     /nsd_content                                O5              O5
+                    /nsd                                        O
+                    /artifacts[/<artifactPath>]                 O
             /pnf_descriptors                                    5       5
                 /<pnfdInfoId>                                   5                       5       5
                     /pnfd_content                               5               5
             /pnf_descriptors                                    5       5
                 /<pnfdInfoId>                                   5                       5       5
                     /pnfd_content                               5               5
-            /subcriptions                                       5       5
-                /<subcriptionId>                                5                       X
+            /subscriptions                                      5       5
+                /<subscriptionId>                               5                       X
 
         /vnfpkgm/v1
             /vnf_packages                                       O5      O5
                 /<vnfPkgId>                                     O5                      O5      5
 
         /vnfpkgm/v1
             /vnf_packages                                       O5      O5
                 /<vnfPkgId>                                     O5                      O5      5
-                    /vnfd                                       O5      O
                     /package_content                            O5               O5
                         /upload_from_uri                                X
                     /package_content                            O5               O5
                         /upload_from_uri                                X
-                    /artifacts/<artifactPatch                   X
-            /subcriptions                                       X       X
-                /<subcriptionId>                                X                       X
+                    /vnfd                                       O5
+                    /artifacts[/<artifactPath>]                 O5
+            /subscriptions                                      X       X
+                /<subscriptionId>                               X                       X
 
         /nslcm/v1
 
         /nslcm/v1
-            /ns_instances                                       O5      O5
-                /<nsInstanceId>                                 O5                      O5     
+            /ns_instances_content                               O       O
+                /<nsInstanceId>                                 O                       O     
+            /ns_instances                                       5       5
+                /<nsInstanceId>                                 5                       5     
                     TO BE COMPLETED                             
             /ns_lcm_op_occs                                     5       5
                 /<nsLcmOpOccId>                                 5                       5       5
                     TO BE COMPLETED                             5               5
                     TO BE COMPLETED                             
             /ns_lcm_op_occs                                     5       5
                 /<nsLcmOpOccId>                                 5                       5       5
                     TO BE COMPLETED                             5               5
-            /subcriptions                                       5       5
-                /<subcriptionId>                                5                       X
+            /subscriptions                                      5       5
+                /<subscriptionId>                               5                       X
+        /admin/v1
+            /tokens                                             O       O
+                /<id>                                           O                       O     
+            /users                                              O       O
+                /<id>                                           O                       O     
+            /projects                                           O       O
+                /<id>                                           O                       O     
 
 query string.
     <attrName>[.<attrName>...]*[.<op>]=<value>[,<value>...]&...
 
 query string.
     <attrName>[.<attrName>...]*[.<op>]=<value>[,<value>...]&...
@@ -106,6 +120,74 @@ class Server(object):
     def __init__(self):
         self.instance += 1
         self.engine = Engine()
     def __init__(self):
         self.instance += 1
         self.engine = Engine()
+        self.valid_methods = {   # contains allowed URL and methods
+            "admin": {
+                "v1": {
+                    "tokens": { "METHODS": ("GET", "POST", "DELETE"),
+                        "<ID>": { "METHODS": ("GET", "DELETE")}
+                    },
+                    "users": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "POST", "DELETE")}
+                    },
+                    "projects": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "POST", "DELETE")}
+                    },
+                }
+            },
+            "nsd": {
+                "v1": {
+                    "ns_descriptors_content": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
+                    },
+                    "ns_descriptors": { "METHODS": ("GET", "POST"),
+                        "<ID>": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH",
+                            "nsd_content": { "METHODS": ("GET", "PUT")},
+                            "nsd": {"METHODS": "GET"},  # descriptor inside package
+                            "artifacts": {"*": {"METHODS": "GET"}}
+                        }
+
+                    },
+                    "pnf_descriptors": {"TODO": ("GET", "POST"),
+                       "<ID>": {"TODO": ("GET", "DELETE", "PATCH"),
+                            "pnfd_content": {"TODO": ("GET", "PUT")}
+                        }
+                    },
+                    "subscriptions": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE"),}
+                    },
+                }
+            },
+            "vnfpkgm": {
+                "v1": {
+                    "vnf_packages_content": { "METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "PUT", "DELETE")}
+                    },
+                    "vnf_packages": { "METHODS": ("GET", "POST"),
+                        "<ID>": { "METHODS": ("GET", "DELETE"), "TODO": "PATCH",  # GET: vnfPkgInfo
+                            "package_content": { "METHODS": ("GET", "PUT"),         # package
+                                "upload_from_uri": {"TODO": "POST"}
+                            },
+                            "vnfd": {"METHODS": "GET"},                    # descriptor inside package
+                            "artifacts": {"*": {"METHODS": "GET"}}
+                        }
+
+                    },
+                    "subscriptions": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE"),}
+                    },
+                }
+            },
+            "nslcm": {
+                "v1": {
+                    "ns_instances_content": {"METHODS": ("GET", "POST"),
+                        "<ID>": {"METHODS": ("GET", "DELETE")}
+                    },
+                    "ns_instances": {"TODO": ("GET", "POST"),
+                        "<ID>": {"TODO": ("GET", "DELETE")}
+                    }
+                }
+            },
+        }
 
     def _authorization(self):
         token = None
 
     def _authorization(self):
         token = None
@@ -164,14 +246,15 @@ class Server(object):
                         indata = yaml.load(cherrypy.request.body)
                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
                         indata = yaml.load(cherrypy.request.body)
                     elif "application/binary" in cherrypy.request.headers["Content-Type"] or \
                          "application/gzip" in cherrypy.request.headers["Content-Type"] or \
-                         "application/zip" in cherrypy.request.headers["Content-Type"]:
-                        indata = cherrypy.request.body.read()
+                         "application/zip" in cherrypy.request.headers["Content-Type"] or \
+                         "text/plain" in cherrypy.request.headers["Content-Type"]:
+                        indata = cherrypy.request.body  # .read()
                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
                         if "descriptor_file" in kwargs:
                             filecontent = kwargs.pop("descriptor_file")
                             if not filecontent.file:
                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
                     elif "multipart/form-data" in cherrypy.request.headers["Content-Type"]:
                         if "descriptor_file" in kwargs:
                             filecontent = kwargs.pop("descriptor_file")
                             if not filecontent.file:
                                 raise NbiException("empty file or content", HTTPStatus.BAD_REQUEST)
-                            indata = filecontent.file.read()
+                            indata = filecontent.file  # .read()
                             if filecontent.content_type.value:
                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
                     else:
                             if filecontent.content_type.value:
                                 cherrypy.request.headers["Content-Type"] = filecontent.content_type.value
                     else:
@@ -186,10 +269,6 @@ class Server(object):
             if not indata:
                 indata = {}
 
             if not indata:
                 indata = {}
 
-            if "METHOD" in kwargs:
-                method = kwargs.pop("METHOD")
-            else:
-                method = cherrypy.request.method
             format_yaml = False
             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
                 format_yaml = True
             format_yaml = False
             if cherrypy.request.headers.get("Query-String-Format") == "yaml":
                 format_yaml = True
@@ -223,20 +302,33 @@ class Server(object):
                             except:
                                 pass
 
                             except:
                                 pass
 
-            return indata, method
+            return indata
         except (ValueError, yaml.YAMLError) as exc:
             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
         except KeyError as exc:
             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 
     @staticmethod
         except (ValueError, yaml.YAMLError) as exc:
             raise NbiException(error_text + str(exc), HTTPStatus.BAD_REQUEST)
         except KeyError as exc:
             raise NbiException("Query string error: " + str(exc), HTTPStatus.BAD_REQUEST)
 
     @staticmethod
-    def _format_out(data, session=None):
+    def _format_out(data, session=None, _format=None):
         """
         return string of dictionary data according to requested json, yaml, xml. By default json
         """
         return string of dictionary data according to requested json, yaml, xml. By default json
-        :param data: response to be sent. Can be a dict or text
+        :param data: response to be sent. Can be a dict, text or file
         :param session:
         :param session:
+        :param _format: The format to be set as Content-Type ir data is a file
         :return: None
         """
         :return: None
         """
+        if data is None:
+            cherrypy.response.status = HTTPStatus.NO_CONTENT.value
+            return
+        elif hasattr(data, "read"):  # file object
+            if _format:
+                cherrypy.response.headers["Content-Type"] = _format
+            elif "b" in data.mode:  # binariy asssumig zip
+                cherrypy.response.headers["Content-Type"] = 'application/zip'
+            else:
+                cherrypy.response.headers["Content-Type"] = 'text/plain'
+            # TODO check that cherrypy close file. If not implement pending things to close  per thread next
+            return data
         if "Accept" in cherrypy.request.headers:
             accept = cherrypy.request.headers["Accept"]
             if "application/json" in accept:
         if "Accept" in cherrypy.request.headers:
             accept = cherrypy.request.headers["Accept"]
             if "application/json" in accept:
@@ -246,7 +338,7 @@ class Server(object):
             elif "text/html" in accept:
                 return html.format(data, cherrypy.request, cherrypy.response, session)
 
             elif "text/html" in accept:
                 return html.format(data, cherrypy.request, cherrypy.response, session)
 
-            elif "application/yaml" in accept or "*/*" in accept:
+            elif "application/yaml" in accept or "*/*" in accept or "text/plain" in accept:
                 pass
             else:
                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
                 pass
             else:
                 raise cherrypy.HTTPError(HTTPStatus.NOT_ACCEPTABLE.value,
@@ -275,20 +367,17 @@ class Server(object):
             return self._format_out("Welcome to OSM!", session)
 
     @cherrypy.expose
             return self._format_out("Welcome to OSM!", session)
 
     @cherrypy.expose
-    def token(self, *args, **kwargs):
-        if not args:
-            raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED)
-        version = args[0]
-        if version != 'v1':
-            raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
+    def token(self, method, token_id=None, kwargs=None):
         session = None
         # self.engine.load_dbase(cherrypy.request.app.config)
         session = None
         # self.engine.load_dbase(cherrypy.request.app.config)
+        indata = self._format_in(kwargs)
+        if not isinstance(indata, dict):
+            raise NbiException("Expected application/yaml or application/json Content-Type", HTTPStatus.BAD_REQUEST)
         try:
         try:
-            indata, method = self._format_in(kwargs)
             if method == "GET":
                 session = self._authorization()
             if method == "GET":
                 session = self._authorization()
-                if len(args) >= 2:
-                    outdata = self.engine.get_token(session, args[1])
+                if token_id:
+                    outdata = self.engine.get_token(session, token_id)
                 else:
                     outdata = self.engine.get_token_list(session)
             elif method == "POST":
                 else:
                     outdata = self.engine.get_token_list(session)
             elif method == "POST":
@@ -304,11 +393,9 @@ class Server(object):
                 # cherrypy.response.cookie["Authorization"] = outdata["id"]
                 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
             elif method == "DELETE":
                 # cherrypy.response.cookie["Authorization"] = outdata["id"]
                 # cherrypy.response.cookie["Authorization"]['expires'] = 3600
             elif method == "DELETE":
-                if len(args) >= 2 and "logout" not in args:
-                    token_id = args[1]
-                elif "id" in kwargs:
+                if not token_id and "id" in kwargs:
                     token_id = kwargs["id"]
                     token_id = kwargs["id"]
-                else:
+                elif not token_id:
                     session = self._authorization()
                     token_id = session["_id"]
                 outdata = self.engine.del_token(token_id)
                     session = self._authorization()
                     token_id = session["_id"]
                 outdata = self.engine.del_token(token_id)
@@ -329,10 +416,21 @@ class Server(object):
             }
             return self._format_out(problem_details, session)
 
             }
             return self._format_out(problem_details, session)
 
+    @cherrypy.expose
+    def test2(self, args0=None, args1=None, args2=None, args3=None, *args, **kwargs):
+        return_text = (
+            "<html><pre>\n{} {} {} {} {} {} \n".format(args0, args1, args2, args3, args, kwargs))
+        return_text += "</pre></html>"
+        return return_text
+
     @cherrypy.expose
     def test(self, *args, **kwargs):
         thread_info = None
     @cherrypy.expose
     def test(self, *args, **kwargs):
         thread_info = None
-        if args and args[0] == "init":
+        if args and args[0] == "help":
+            return "<html><pre>\ninit\nfile/<name>  download file\ndb-clear/table\nprune\nlogin\nlogin2\n"\
+                    "sleep/<time>\n</pre></html>"
+
+        elif args and args[0] == "init":
             try:
                 # self.engine.load_dbase(cherrypy.request.app.config)
                 self.engine.create_admin()
             try:
                 # self.engine.load_dbase(cherrypy.request.app.config)
                 self.engine.create_admin()
@@ -340,6 +438,17 @@ class Server(object):
             except Exception:
                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
                 return self._format_out("Database already initialized")
             except Exception:
                 cherrypy.response.status = HTTPStatus.FORBIDDEN.value
                 return self._format_out("Database already initialized")
+        elif args and args[0] == "file":
+            return cherrypy.lib.static.serve_file(cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1],
+                                                  "text/plain", "attachment")
+        elif args and args[0] == "file2":
+            f_path = cherrypy.tree.apps['/osm'].config["storage"]["path"] + "/" + args[1]
+            f = open(f_path, "r")
+            cherrypy.response.headers["Content-type"] = "text/plain"
+
+            return f
+        elif len(args) == 2 and args[0] == "db-clear":
+            return self.engine.del_item_list({"project_id": "admin"}, args[1], {})
         elif args and args[0] == "prune":
             return self.engine.prune()
         elif args and args[0] == "login":
         elif args and args[0] == "prune":
             return self.engine.prune()
         elif args and args[0] == "login":
@@ -390,83 +499,148 @@ class Server(object):
         return_text += "</pre></html>"
         return return_text
 
         return_text += "</pre></html>"
         return return_text
 
+    def _check_valid_url_method(self, method, *args):
+        if len(args) < 3:
+            raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED)
+
+        reference = self.valid_methods
+        for arg in args:
+            if arg is None:
+                break
+            if not isinstance(reference, dict):
+                raise NbiException("URL contains unexpected extra items '{}'".format(arg),
+                                   HTTPStatus.METHOD_NOT_ALLOWED)
+
+            if arg in reference:
+                reference = reference[arg]
+            elif "<ID>" in reference:
+                reference = reference["<ID>"]
+            elif "*" in reference:
+                reference = reference["*"]
+                break
+            else:
+                raise NbiException("Unexpected URL item {}".format(arg), HTTPStatus.METHOD_NOT_ALLOWED)
+        if "TODO" in reference and method in reference["TODO"]:
+            raise NbiException("Method {} not supported yet for this URL".format(method), HTTPStatus.NOT_IMPLEMENTED)
+        elif "METHODS" in reference and not method in reference["METHODS"]:
+            raise NbiException("Method {} not supported for this URL".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
+        return
+
+    @staticmethod
+    def _set_location_header(topic, version, item, id):
+        """
+        Insert response header Location with the URL of created item base on URL params
+        :param topic:
+        :param version:
+        :param item:
+        :param id:
+        :return: None
+        """
+        # Use cherrypy.request.base for absoluted path and make use of request.header HOST just in case behind aNAT
+        cherrypy.response.headers["Location"] = "/osm/{}/{}/{}/{}".format(topic, version, item, id)
+        return
+
     @cherrypy.expose
     @cherrypy.expose
-    def default(self, *args, **kwargs):
+    def default(self, topic=None, version=None, item=None, _id=None, item2=None, *args, **kwargs):
         session = None
         session = None
+        outdata = None
+        _format = None
         try:
         try:
-            if not args or len(args) < 2:
-                raise NbiException("URL must contain at least 'item/version'", HTTPStatus.METHOD_NOT_ALLOWED)
-            item = args[0]
-            version = args[1]
-            if item not in ("token", "user", "project", "vnfpkgm", "nsd", "nslcm"):
-                raise NbiException("URL item '{}' not supported".format(item), HTTPStatus.METHOD_NOT_ALLOWED)
+            if not topic or not version or not item:
+                raise NbiException("URL must contain at least 'topic/version/item'", HTTPStatus.METHOD_NOT_ALLOWED)
+            if topic not in ("admin", "vnfpkgm", "nsd", "nslcm"):
+                raise NbiException("URL topic '{}' not supported".format(topic), HTTPStatus.METHOD_NOT_ALLOWED)
             if version != 'v1':
                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 
             if version != 'v1':
                 raise NbiException("URL version '{}' not supported".format(version), HTTPStatus.METHOD_NOT_ALLOWED)
 
+            if kwargs and "METHOD" in kwargs and kwargs["METHOD"] in ("PUT", "POST", "DELETE", "GET", "PATCH"):
+                method = kwargs.pop("METHOD")
+            else:
+                method = cherrypy.request.method
+
+            self._check_valid_url_method(method, topic, version, item, _id, item2, *args)
+
+            if topic == "admin" and item == "tokens":
+                return self.token(method, _id, kwargs)
+
             # self.engine.load_dbase(cherrypy.request.app.config)
             session = self._authorization()
             # self.engine.load_dbase(cherrypy.request.app.config)
             session = self._authorization()
-            indata, method = self._format_in(kwargs)
-            _id = None
-
-            if item == "nsd":
-                item = "nsds"
-                if len(args) < 3 or args[2] != "ns_descriptors":
-                    raise NbiException("only ns_descriptors is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4 and args[4] != "nsd_content":
-                    raise NbiException("only nsd_content is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-            elif item == "vnfpkgm":
-                item = "vnfds"
-                if len(args) < 3 or args[2] != "vnf_packages":
-                    raise NbiException("only vnf_packages is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4 and args[4] not in ("vnfd", "package_content"):
-                    raise NbiException("only vnfd or package_content are allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-            elif item == "nslcm":
-                item = "nsrs"
-                if len(args) < 3 or args[2] != "ns_instances":
-                    raise NbiException("only ns_instances is allowed", HTTPStatus.METHOD_NOT_ALLOWED)
-                if len(args) > 3:
-                    _id = args[3]
-                if len(args) > 4:
-                    raise NbiException("This feature is not implemented", HTTPStatus.METHOD_NOT_ALLOWED)
-            else:
-                if len(args) >= 3:
-                    _id = args[2]
-                item += "s"
+            indata = self._format_in(kwargs)
+            engine_item = item
+            if item == "subscriptions":
+                engine_item = topic + "_" + item
+            if item2:
+                engine_item = item2
+
+            if topic == "nsd":
+                engine_item = "nsds"
+            elif topic == "vnfpkgm":
+                engine_item = "vnfds"
+            elif topic == "nslcm":
+                engine_item = "nsrs"
 
             if method == "GET":
 
             if method == "GET":
-                if not _id:
-                    outdata = self.engine.get_item_list(session, item, kwargs)
-                else:  # len(args) > 1
-                    outdata = self.engine.get_item(session, item, _id)
+                if item2 in ("nsd_content", "package_content", "artifacts", "vnfd", "nsd"):
+                    if item2 in ("vnfd", "nsd"):
+                        path = "$DESCRIPTOR"
+                    elif args:
+                        path = args
+                    elif item2 == "artifacts":
+                        path = ()
+                    else:
+                        path = None
+                    file, _format = self.engine.get_file(session, engine_item, _id, path,
+                                                            cherrypy.request.headers.get("Accept"))
+                    outdata = file
+                elif not _id:
+                    outdata = self.engine.get_item_list(session, engine_item, kwargs)
+                else:
+                    outdata = self.engine.get_item(session, engine_item, _id)
             elif method == "POST":
             elif method == "POST":
-                id, completed = self.engine.new_item(session, item, indata, kwargs, cherrypy.request.headers)
-                if not completed:
-                    cherrypy.response.headers["Transaction-Id"] = id
-                    cherrypy.response.status = HTTPStatus.CREATED.value
+                if item in ("ns_descriptors_content", "vnf_packages_content"):
+                    _id = cherrypy.request.headers.get("Transaction-Id")
+                    if not _id:
+                        _id = self.engine.new_item(session, engine_item, {}, None, cherrypy.request.headers)
+                    completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers)
+                    if completed:
+                        self._set_location_header(topic, version, item, _id)
+                    else:
+                        cherrypy.response.headers["Transaction-Id"] = _id
+                    outdata = {"id": _id}
+                elif item in ("ns_descriptors", "vnf_packages"):
+                    _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers)
+                    self._set_location_header(topic, version, item, _id)
+                    #TODO form NsdInfo
+                    outdata = {"id": _id}
                 else:
                 else:
-                    cherrypy.response.headers["Location"] = cherrypy.request.base + "/osm/" + "/".join(args[0:3]) + "/" + id
-                outdata = {"id": id}
+                    _id = self.engine.new_item(session, engine_item, indata, kwargs, cherrypy.request.headers)
+                    self._set_location_header(topic, version, item, _id)
+                    outdata = {"id": _id}
+                cherrypy.response.status = HTTPStatus.CREATED.value
             elif method == "DELETE":
                 if not _id:
             elif method == "DELETE":
                 if not _id:
-                    outdata = self.engine.del_item_list(session, item, kwargs)
+                    outdata = self.engine.del_item_list(session, engine_item, kwargs)
                 else:  # len(args) > 1
                 else:  # len(args) > 1
-                    outdata = self.engine.del_item(session, item, _id)
+                    outdata = self.engine.del_item(session, engine_item, _id)
+                if item in ("ns_descriptors", "vnf_packages"):  # SOL005
+                    outdata = None
             elif method == "PUT":
             elif method == "PUT":
-                if not _id:
-                    raise NbiException("Missing '/<id>' at the URL to identify item to be updated",
-                                       HTTPStatus.METHOD_NOT_ALLOWED)
-                elif not indata and not kwargs:
+                if not indata and not kwargs:
                     raise NbiException("Nothing to update. Provide payload and/or query string",
                                        HTTPStatus.BAD_REQUEST)
                     raise NbiException("Nothing to update. Provide payload and/or query string",
                                        HTTPStatus.BAD_REQUEST)
-                outdata = {"id": self.engine.edit_item(session, item, args[1], indata, kwargs)}
+                if item2 in ("nsd_content", "package_content"):
+                    completed = self.engine.upload_content(session, engine_item, _id, indata, kwargs, cherrypy.request.headers)
+                    if not completed:
+                        cherrypy.response.headers["Transaction-Id"] = id
+                    outdata = None
+                else:
+                    outdata = {"id": self.engine.edit_item(session, engine_item, args[1], indata, kwargs)}
             else:
                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
             else:
                 raise NbiException("Method {} not allowed".format(method), HTTPStatus.METHOD_NOT_ALLOWED)
-            return self._format_out(outdata, session)
-        except (NbiException, EngineException, DbException) as e:
+            return self._format_out(outdata, session, _format)
+        except (NbiException, EngineException, DbException, FsException) as e:
+            if hasattr(outdata, "close"):  # is an open file
+                outdata.close()
             cherrypy.log("Exception {}".format(e))
             cherrypy.response.status = e.http_code.value
             problem_details = {
             cherrypy.log("Exception {}".format(e))
             cherrypy.response.status = e.http_code.value
             problem_details = {
diff --git a/osm_nbi/test/cirros_ns/cirros_nsd.yaml b/osm_nbi/test/cirros_ns/cirros_nsd.yaml
new file mode 100644 (file)
index 0000000..5c4e214
--- /dev/null
@@ -0,0 +1,49 @@
+nsd-catalog:
+    nsd:
+    -   id: cirros_nsd
+        name: cirros_ns
+        short-name: cirros_ns
+        description: Generated by OSM pacakage generator
+        vendor: OSM
+        version: '1.0'
+
+        # Place the logo as png in icons directory and provide the name here
+        logo: osm_2x.png
+
+        # Specify the VNFDs that are part of this NSD
+        constituent-vnfd:
+            # The member-vnf-index needs to be unique, starting from 1
+            # vnfd-id-ref is the id of the VNFD
+            # Multiple constituent VNFDs can be specified
+        -   member-vnf-index: 1
+            vnfd-id-ref: cirros_vnfd
+        scaling-group-descriptor:
+        -   name: "scaling_cirros"
+            vnfd-member:
+            -   count: 1
+                member-vnf-index-ref: 1
+            min-instance-count: 0
+            max-instance-count: 10
+            scaling-policy:
+            -   scaling-type: "manual"
+                cooldown-time: 10
+                threshold-time: 10
+                name: manual_scale
+        vld:
+        # Networks for the VNFs
+            -   id: cirros_nsd_vld1
+                name: cirros_nsd_vld1
+                type: ELAN
+                mgmt-network: 'true'
+                # vim-network-name: <update>
+                # provider-network:
+                #     segmentation_id: <update>
+                vnfd-connection-point-ref:
+                # Specify the constituent VNFs
+                # member-vnf-index-ref - entry from constituent vnf
+                # vnfd-id-ref - VNFD id
+                # vnfd-connection-point-ref - connection point name in the VNFD
+                -   member-vnf-index-ref: 1
+                    vnfd-id-ref: cirros_vnfd
+                    # NOTE: Validate the entry below
+                    vnfd-connection-point-ref: eth0
diff --git a/osm_nbi/test/cirros_ns/icons/osm_2x.png b/osm_nbi/test/cirros_ns/icons/osm_2x.png
new file mode 100644 (file)
index 0000000..62012d2
Binary files /dev/null and b/osm_nbi/test/cirros_ns/icons/osm_2x.png differ
diff --git a/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml b/osm_nbi/test/cirros_vnf/cirros_vnfd.yaml
new file mode 100644 (file)
index 0000000..94fa5f1
--- /dev/null
@@ -0,0 +1,48 @@
+vnfd-catalog:
+    vnfd:
+    -   id: cirros_vnfd
+        name: cirros_vnf
+        short-name: cirros_vnf
+        description: Simple VNF example with a cirros 
+        vendor: OSM
+        version: '1.0'
+
+        # Place the logo as png in icons directory and provide the name here
+        logo: cirros-64.png
+
+        # Management interface
+        mgmt-interface:
+            cp: eth0
+
+        # Atleast one VDU need to be specified
+        vdu:
+        -   id: cirros_vnfd-VM
+            name: cirros_vnfd-VM
+            description: cirros_vnfd-VM
+            count: 1
+
+            # Flavour of the VM to be instantiated for the VDU
+            # flavor below can fit into m1.micro
+            vm-flavor:
+                vcpu-count: 1
+                memory-mb: 256
+                storage-gb: 2
+
+            # Image/checksum or image including the full path
+            image: 'cirros034'
+            #checksum: 
+
+            interface:
+            # Specify the external interfaces
+            # There can be multiple interfaces defined
+            -   name: eth0
+                type: EXTERNAL
+                virtual-interface:
+                    type: VIRTIO
+                    bandwidth: '0'
+                    vpci: 0000:00:0a.0
+                external-connection-point-ref: eth0
+
+        connection-point:
+            -   name: eth0
+                type: VPORT
diff --git a/osm_nbi/test/cirros_vnf/icons/cirros-64.png b/osm_nbi/test/cirros_vnf/icons/cirros-64.png
new file mode 100644 (file)
index 0000000..5725d29
Binary files /dev/null and b/osm_nbi/test/cirros_vnf/icons/cirros-64.png differ
index 4d96d47..7ef1cf7 100755 (executable)
@@ -25,19 +25,23 @@ NSD3=${DESCRIPTORS}/cirros_nsd.yaml
 [ -f "$NSD3" ] || ! echo "not found cirros_nsd.yaml. Set DESCRIPTORS variable to a proper location" || exit 1
 
 #get token
 [ -f "$NSD3" ] || ! echo "not found cirros_nsd.yaml. Set DESCRIPTORS variable to a proper location" || exit 1
 
 #get token
-TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}'`;
+TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml" \
+    --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens \
+     2>/dev/null | awk '($1=="id:"){print $2}'`;
 echo token: $TOKEN
 
 
 echo token: $TOKEN
 
 
-
-
 # VNFD
 #########
 #insert PKG
 # VNFD
 #########
 #insert PKG
-VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'` 
+VNFD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data-binary "@$VNFD1" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \
+     2>/dev/null | awk '($1=="id:"){print $2}'`
 echo ping_vnfd: $VNFD1_ID
 
 echo ping_vnfd: $VNFD1_ID
 
-VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages 2>/dev/null | awk '($1=="id:"){print $2}'`
+VNFD2_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data-binary "@$VNFD2" ${NBI_URL}/vnfpkgm/v1/vnf_packages_content \
+     2>/dev/null | awk '($1=="id:"){print $2}'`
 echo pong_vnfd: $VNFD2_ID
 
 
 echo pong_vnfd: $VNFD2_ID
 
 
@@ -45,19 +49,25 @@ echo pong_vnfd: $VNFD2_ID
 # NSD
 #########
 #insert PKG
 # NSD
 #########
 #insert PKG
-NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors 2>/dev/null | awk '($1=="id:"){print $2}'`
+NSD1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/gzip" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"   --data-binary "@$NSD1" ${NBI_URL}/nsd/v1/ns_descriptors_content \
+    2>/dev/null | awk '($1=="id:"){print $2}'`
 echo ping_pong_nsd: $NSD1_ID
 
 
 # NSRS
 ##############
 #add nsr
 echo ping_pong_nsd: $NSD1_ID
 
 
 # NSRS
 ##############
 #add nsr
-NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"   --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }"  ${NBI_URL}/nslcm/v1/ns_instances 2>/dev/null | awk '($1=="id:"){print $2}'` ;
+NSR1_ID=`curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" \
+    -H "Authorization: Bearer $TOKEN"  --data "{ nsDescription: default description, nsName: NSNAME, nsdId: $NSD1_ID, \
+     ssh-authorized-key: [ {key-pair-ref: gerardo}, {key-pair-ref: alfonso}], vimAccountId: $VIM }" \
+     ${NBI_URL}/nslcm/v1/ns_instances_content 2>/dev/null | awk '($1=="id:"){print $2}'` ;
 echo ping_pong_nsr: $NSR1_ID
 
 
 echo ping_pong_nsr: $NSR1_ID
 
 
-echo '
-curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer '$TOKEN'"  '${NBI_URL}'/nslcm/v1/ns_instances/'$NSR1_ID' 2>/dev/null | grep -e detailed-status -e operational-status -e config-status'
+echo 'curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml"' \
+    '-H "Authorization: Bearer '$TOKEN'"  '${NBI_URL}'/nslcm/v1/ns_instances_content/'$NSR1_ID' 2>/dev/null | ' \
+    'grep -e detailed-status -e operational-status -e config-status'
 
 
 
 
 
 
index 6b04538..d8cb474 100755 (executable)
@@ -8,13 +8,13 @@ PROJECT=admin
 
 
 #get token
 
 
 #get token
-TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/token/v1 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN
+TOKEN=`curl --insecure -H "Content-Type: application/yaml" -H "Accept: application/yaml"  --data "{username: $USERNAME, password: $PASSWORD, project_id: $PROJECT}" ${NBI_URL}/admin/v1/tokens 2>/dev/null | awk '($1=="id:"){print $2}' ` ; echo $TOKEN
 
 
 echo  deleting all
 #DELETE ALL
 
 
 
 echo  deleting all
 #DELETE ALL
 
-for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors vnfpkgm/v1/vnf_packages
+for url_item in nslcm/v1/ns_instances nsd/v1/ns_descriptors_content vnfpkgm/v1/vnf_packages_content
 do
     for ITEM_ID in `curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"  ${NBI_URL}/${url_item} 2>/dev/null | awk '($1=="_id:") {print $2}'` ;
     do
 do
     for ITEM_ID in `curl --insecure -w "%{http_code}\n" -H "Content-Type: application/yaml" -H "Accept: application/yaml" -H "Authorization: Bearer $TOKEN"  ${NBI_URL}/${url_item} 2>/dev/null | awk '($1=="_id:") {print $2}'` ;
     do
diff --git a/osm_nbi/test/test.py b/osm_nbi/test/test.py
new file mode 100755 (executable)
index 0000000..734d667
--- /dev/null
@@ -0,0 +1,337 @@
+#! /usr/bin/python3
+# -*- coding: utf-8 -*-
+
+import getopt
+import sys
+import requests
+#import base64
+#from os.path import getsize, basename
+#from hashlib import md5
+import json
+import logging
+import yaml
+#import json
+import tarfile
+from os import makedirs
+from copy import deepcopy
+
+__author__ = "Alfonso Tierno, alfonso.tiernosepulveda@telefonica.com"
+__date__ = "$2018-03-01$"
+__version__ = "0.1"
+version_date = "Mar 2018"
+
+
+def usage():
+    print("Usage: ", sys.argv[0], "[options]")
+    print("      --version: prints current version")
+    print("      -f|--file FILE: file to be sent")
+    print("      -h|--help: shows this help")
+    print("      -u|--url URL: complete server URL")
+    print("      -s|--chunk-size SIZE: size of chunks, by default 1000")
+    print("      -t|--token TOKEN: Authorizaton token, previously obtained from server")
+    print("      -v|--verbose print debug information, can be used several times")
+    return
+
+
+r_header_json = {"Content-type": "application/json"}
+headers_json = {
+    "Content-type": "application/json",
+    "Accept": "application/json",
+}
+r_header_yaml = {"Content-type": "application/yaml"}
+headers_yaml = {
+    "Content-type": "application/yaml",
+    "Accept": "application/yaml",
+}
+r_header_text = {"Content-type": "text/plain"}
+r_header_octect = {"Content-type": "application/octet-stream"}
+headers_text = {
+    "Accept": "text/plain",
+}
+r_header_zip = {"Content-type": "application/zip"}
+headers_zip = {
+    "Accept": "application/zip",
+}
+# test without authorization
+test_not_authorized_list = (
+    ("Invalid token", "GET", "/admin/v1/users", headers_json, None, 401, r_header_json, "json"),
+    ("Invalid URL", "POST", "/admin/v1/nonexist", headers_yaml, None, 405, r_header_yaml, "yaml"),
+    ("Invalid version", "DELETE", "/admin/v2/users", headers_yaml, None, 405, r_header_yaml, "yaml"),
+)
+
+# test ones authorized
+test_authorized_list = (
+    ("Invalid vnfd id", "GET", "/vnfpkgm/v1/vnf_packages/non-existing-id", headers_json, None, 404, r_header_json, "json"),
+    ("Invalid nsd id", "GET", "/nsd/v1/ns_descriptors/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"),
+    ("Invalid nsd id", "DELETE", "/nsd/v1/ns_descriptors_content/non-existing-id", headers_yaml, None, 404, r_header_yaml, "yaml"),
+)
+
+class TestException(Exception):
+    pass
+
+
+class TestRest:
+    def __init__(self, url_base, header_base={}, verify=False):
+        self.url_base = url_base
+        self.header_base = header_base
+        self.s = requests.session()
+        self.s.headers = header_base
+        self.verify = verify
+
+    def set_header(self, header):
+        self.s.headers.update(header)
+
+    def test(self, name, method, url, headers, payload, expected_codes, expected_headers, expected_payload):
+        """
+        Performs an http request and check http code response. Exit if different than allowed
+        :param name:  name of the test
+        :param method: HTTP method: GET,PUT,POST,DELETE,...
+        :param url: complete URL or relative URL
+        :param headers: request headers to add to the base headers
+        :param payload: Can be a dict, transformed to json, a text or a file if starts with '@'
+        :param expected_codes: expected response codes, can be int, int tuple or int range
+        :param expected_headers: expected response headers, dict with key values
+        :param expected_payload: expected payload, 0 if empty, 'yaml', 'json', 'text', 'zip'
+        :return:
+        """
+        try:
+            if not self.s:
+                self.s = requests.session()
+            if not url:
+                url = self.url_base
+            elif not url.startswith("http"):
+                url = self.url_base + url
+            if payload:
+                if isinstance(payload, str):
+                    if payload.startswith("@"):
+                        mode = "r"
+                        file_name = payload[1:]
+                        if payload.startswith("@b"):
+                            mode = "rb"
+                            file_name = payload[2:]
+                        with open(file_name, mode) as f:
+                            payload = f.read()
+                elif isinstance(payload, dict):
+                    payload = json.dumps(payload)
+    
+            test = "Test {} {} {}".format(name, method, url)
+            logger.warning(test)
+            stream = False
+            # if expected_payload == "zip":
+            #     stream = True
+            r = getattr(self.s, method.lower())(url, data=payload, headers=headers, verify=self.verify, stream=stream)
+            logger.debug("RX {}: {}".format(r.status_code, r.text))
+
+            # check response
+            if expected_codes:
+                if isinstance(expected_codes, int):
+                    expected_codes = (expected_codes,)
+                if r.status_code not in expected_codes:
+                    raise TestException(
+                        "Got status {}. Expected {}. {}".format(r.status_code, expected_codes, r.text))
+
+            if expected_headers:
+                for header_key, header_val in expected_headers.items():
+                    if header_key.lower() not in r.headers:
+                        raise TestException("Header {} not present".format(header_key))
+                    if header_val and header_val.lower() not in r.headers[header_key]:
+                        raise TestException("Header {} does not contain {} but {}".format(header_key, header_val,
+                                            r.headers[header_key]))
+
+            if expected_payload is not None:
+                if expected_payload == 0 and len(r.content) > 0:
+                    raise TestException("Expected empty payload")
+                elif expected_payload == "json":
+                    try:
+                        r.json()
+                    except Exception as e:
+                        raise TestException("Expected json response payload, but got Exception {}".format(e))
+                elif expected_payload == "yaml":
+                    try:
+                        yaml.safe_load(r.text)
+                    except Exception as e:
+                        raise TestException("Expected yaml response payload, but got Exception {}".format(e))
+                elif expected_payload == "zip":
+                    if len(r.content) == 0:
+                        raise TestException("Expected some response payload, but got empty")
+                    # try:
+                    #     tar = tarfile.open(None, 'r:gz', fileobj=r.raw)
+                    #     for tarinfo in tar:
+                    #         tarname = tarinfo.name
+                    #         print(tarname)
+                    # except Exception as e:
+                    #     raise TestException("Expected zip response payload, but got Exception {}".format(e))
+                elif expected_payload == "text":
+                    if len(r.content) == 0:
+                        raise TestException("Expected some response payload, but got empty")
+                    #r.text
+            return r
+        except TestException as e:
+            logger.error("{} \nRX code{}: {}".format(e, r.status_code, r.text))
+            exit(1)
+        except IOError as e:
+            logger.error("Cannot open file {}".format(e))
+            exit(1)
+
+
+if __name__ == "__main__":
+    global logger
+    test = ""
+    try:
+        logging.basicConfig(format="%(levelname)s %(message)s", level=logging.ERROR)
+        logger = logging.getLogger('NBI')
+        # load parameters and configuration
+        opts, args = getopt.getopt(sys.argv[1:], "hvu:p:",
+                                   ["url=", "user=", "password=", "help", "version", "verbose", "project=", "insecure"])
+        url = "https://localhost:9999/osm"
+        user = password = project = "admin"
+        verbose = 0
+        verify = True
+
+        for o, a in opts:
+            if o == "--version":
+                print ("test version " + __version__ + ' ' + version_date)
+                sys.exit()
+            elif o in ("-v", "--verbose"):
+                verbose += 1
+            elif o in ("no-verbose"):
+                verbose = -1
+            elif o in ("-h", "--help"):
+                usage()
+                sys.exit()
+            elif o in ("--url"):
+                url = a
+            elif o in ("-u", "--user"):
+                user = a
+            elif o in ("-p", "--password"):
+                password = a
+            elif o in ("--project"):
+                project = a
+            elif o in ("--insecure"):
+                verify = False
+            else:
+                assert False, "Unhandled option"
+        if verbose == 0:
+            logger.setLevel(logging.WARNING)
+        elif verbose > 1:
+            logger.setLevel(logging.DEBUG)
+        else:
+            logger.setLevel(logging.ERROR)
+
+        test_rest = TestRest(url)
+
+        # tests without authorization
+        for t in test_not_authorized_list:
+            test_rest.test(*t)
+
+        # get token
+        r = test_rest.test("Obtain token", "POST", "/admin/v1/tokens", headers_json,
+                           {"username": user, "password": password, "project_id": project},
+                           (200, 201), {"Content-Type": "application/json"}, "json")
+        response = r.json()
+        token = response["id"]
+        test_rest.set_header({"Authorization": "Bearer {}".format(token)})
+
+        # tests once authorized
+        for t in test_authorized_list:
+            test_rest.test(*t)
+
+        # nsd CREATE
+        r = test_rest.test("Onboard NSD step 1", "POST", "/nsd/v1/ns_descriptors", headers_json, None,
+                           201, {"Location": "/nsd/v1/ns_descriptors/", "Content-Type": "application/json"}, "json")
+        location = r.headers["Location"]
+        nsd_id = location[location.rfind("/")+1:]
+        # print(location, nsd_id)
+
+        # nsd UPLOAD test
+        r = test_rest.test("Onboard NSD step 2 as TEXT", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           r_header_text, "@./cirros_ns/cirros_nsd.yaml", 204, None, 0)
+
+        # nsd SHOW OSM format
+        r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # nsd SHOW text
+        r = test_rest.test("Show NSD SOL005 text", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           headers_text, None, 200, r_header_text, "text")
+
+        # nsd UPLOAD ZIP
+        makedirs("temp", exist_ok=True)
+        tar = tarfile.open("temp/cirros_ns.tar.gz", "w:gz")
+        tar.add("cirros_ns")
+        tar.close()
+        r = test_rest.test("Onboard NSD step 3 replace with ZIP", "PUT", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           r_header_zip, "@b./temp/cirros_ns.tar.gz", 204, None, 0)
+
+        # nsd SHOW OSM format
+        r = test_rest.test("Show NSD OSM format", "GET", "/nsd/v1/ns_descriptors_content/{}".format(nsd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # nsd SHOW zip
+        r = test_rest.test("Show NSD SOL005 zip", "GET", "/nsd/v1/ns_descriptors/{}/nsd_content".format(nsd_id),
+                           headers_zip, None, 200, r_header_zip, "zip")
+
+        # nsd SHOW descriptor
+        r = test_rest.test("Show NSD descriptor", "GET", "/nsd/v1/ns_descriptors/{}/nsd".format(nsd_id),
+                           headers_text, None, 200, r_header_text, "text")
+        # nsd SHOW actifact
+        r = test_rest.test("Show NSD artifact", "GET", "/nsd/v1/ns_descriptors/{}/artifacts/icons/osm_2x.png".format(nsd_id),
+                           headers_text, None, 200, r_header_octect, "text")
+
+        # nsd DELETE
+        r = test_rest.test("Delete NSD SOL005 text", "DELETE", "/nsd/v1/ns_descriptors/{}".format(nsd_id),
+                           headers_yaml, None, 204, None, 0)
+
+        # vnfd CREATE
+        r = test_rest.test("Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages", headers_json, None,
+                           201, {"Location": "/vnfpkgm/v1/vnf_packages/", "Content-Type": "application/json"}, "json")
+        location = r.headers["Location"]
+        vnfd_id = location[location.rfind("/")+1:]
+        # print(location, vnfd_id)
+
+        # vnfd UPLOAD test
+        r = test_rest.test("Onboard VNFD step 2 as TEXT", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           r_header_text, "@./cirros_vnf/cirros_vnfd.yaml", 204, None, 0)
+
+        # vnfd SHOW OSM format
+        r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # vnfd SHOW text
+        r = test_rest.test("Show VNFD SOL005 text", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           headers_text, None, 200, r_header_text, "text")
+
+        # vnfd UPLOAD ZIP
+        makedirs("temp", exist_ok=True)
+        tar = tarfile.open("temp/cirros_vnf.tar.gz", "w:gz")
+        tar.add("cirros_vnf")
+        tar.close()
+        r = test_rest.test("Onboard VNFD step 3 replace with ZIP", "PUT", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           r_header_zip, "@b./temp/cirros_vnf.tar.gz", 204, None, 0)
+
+        # vnfd SHOW OSM format
+        r = test_rest.test("Show VNFD OSM format", "GET", "/vnfpkgm/v1/vnf_packages_content/{}".format(vnfd_id),
+                           headers_json, None, 200, r_header_json, "json")
+
+        # vnfd SHOW zip
+        r = test_rest.test("Show VNFD SOL005 zip", "GET", "/vnfpkgm/v1/vnf_packages/{}/package_content".format(vnfd_id),
+                           headers_zip, None, 200, r_header_zip, "zip")
+        # vnfd SHOW descriptor
+        r = test_rest.test("Show VNFD descriptor", "GET", "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(vnfd_id),
+                           headers_text, None, 200, r_header_text, "text")
+        # vnfd SHOW actifact
+        r = test_rest.test("Show VNFD artifact", "GET", "/vnfpkgm/v1/vnf_packages/{}/artifacts/icons/cirros-64.png".format(vnfd_id),
+                           headers_text, None, 200, r_header_octect, "text")
+
+        # vnfd DELETE
+        r = test_rest.test("Delete VNFD SOL005 text", "DELETE", "/vnfpkgm/v1/vnf_packages/{}".format(vnfd_id),
+                           headers_yaml, None, 204, None, 0)
+
+        print("PASS")
+
+    except Exception as e:
+        if test:
+            logger.error(test + " Exception: " + str(e))
+            exit(1)
+        else:
+            logger.critical(test + " Exception: " + str(e), exc_info=True)