Merge "Bug 637 NSD-level IP Profiles not working in mgmt-network fix single for loop...
[osm/NBI.git] / osm_nbi / descriptor_topics.py
index 9e367c9..a1c3c15 100644 (file)
@@ -1,5 +1,18 @@
 # -*- coding: utf-8 -*-
 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import tarfile
 import yaml
 import json
@@ -11,6 +24,7 @@ from validation import ValidationError, pdu_new_schema, pdu_edit_schema
 from base_topic import BaseTopic, EngineException, get_iterable
 from osm_im.vnfd import vnfd as vnfd_im
 from osm_im.nsd import nsd as nsd_im
+from osm_im.nst import nst as nst_im
 from pyangbind.lib.serialise import pybindJSONDecoder
 import pyangbind.lib.pybindJSON as pybindJSON
 
@@ -23,24 +37,39 @@ class DescriptorTopic(BaseTopic):
         BaseTopic.__init__(self, db, fs, msg)
 
     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
-        # check that this id is not present
-        _filter = {"id": final_content["id"]}
-        if _id:
-            _filter["_id.neq"] = _id
+        # 1. validate again with pyangbind
+        # 1.1. remove internal keys
+        internal_keys = {}
+        for k in ("_id", "_admin"):
+            if k in final_content:
+                internal_keys[k] = final_content.pop(k)
+        storage_params = internal_keys["_admin"].get("storage")
+        serialized = self._validate_input_new(final_content, storage_params, force)
+        # 1.2. modify final_content with a serialized version
+        final_content.clear()
+        final_content.update(serialized)
+        # 1.3. restore internal keys
+        for k, v in internal_keys.items():
+            final_content[k] = v
 
-        _filter.update(self._get_project_filter(session, write=False, show_all=False))
-        if self.db.get_one(self.topic, _filter, fail_on_empty=False):
-            raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
-                                                                                           final_content["id"]),
-                                  HTTPStatus.CONFLICT)
-        # TODO validate with pyangbind. Load and dumps to convert data types
+        if force:
+            return
+        # 2. check that this id is not present
+        if "id" in edit_content:
+            _filter = self._get_project_filter(session, write=False, show_all=False)
+            _filter["id"] = final_content["id"]
+            _filter["_id.neq"] = _id
+            if self.db.get_one(self.topic, _filter, fail_on_empty=False):
+                raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
+                                                                                               final_content["id"]),
+                                      HTTPStatus.CONFLICT)
 
     @staticmethod
     def format_on_new(content, project_id=None, make_public=False):
         BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
         content["_admin"]["onboardingState"] = "CREATED"
         content["_admin"]["operationalState"] = "DISABLED"
-        content["_admin"]["usageSate"] = "NOT_IN_USE"
+        content["_admin"]["usageState"] = "NOT_IN_USE"
 
     def delete(self, session, _id, force=False, dry_run=False):
         """
@@ -57,6 +86,7 @@ class DescriptorTopic(BaseTopic):
             return
         v = self.db.del_one(self.topic, {"_id": _id})
         self.fs.file_delete(_id, ignore_non_exist=True)
+        self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
         self._send_msg("delete", {"_id": _id})
         return v
 
@@ -128,7 +158,7 @@ class DescriptorTopic(BaseTopic):
         :param kwargs: user query string to override parameters. NOT USED
         :param headers:  http request headers
         :param force: to be more tolerant with validation
-        :return: True package has is completely uploaded or False if partial content has been uplodaed.
+        :return: True if package is completely uploaded or False if partial content has been uploded
             Raise exception on error
         """
         # Check that _id exists and it is valid
@@ -157,18 +187,19 @@ class DescriptorTopic(BaseTopic):
                 total = int(content_range[3])
             else:
                 start = 0
+            temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 
             if start:
-                if not self.fs.file_exists(_id, 'dir'):
+                if not self.fs.file_exists(temp_folder, 'dir'):
                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
             else:
-                self.fs.file_delete(_id, ignore_non_exist=True)
-                self.fs.mkdir(_id)
+                self.fs.file_delete(temp_folder, ignore_non_exist=True)
+                self.fs.mkdir(temp_folder)
 
             storage = self.fs.get_params()
             storage["folder"] = _id
 
-            file_path = (_id, filename)
+            file_path = (temp_folder, filename)
             if self.fs.file_exists(file_path, 'file'):
                 file_size = self.fs.file_size(file_path)
             else:
@@ -228,8 +259,8 @@ class DescriptorTopic(BaseTopic):
                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
                 storage["descriptor"] = descriptor_file_name
                 storage["zipfile"] = filename
-                self.fs.file_extract(tar, _id)
-                with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
+                self.fs.file_extract(tar, temp_folder)
+                with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
                     content = descriptor_file.read()
             else:
                 content = file_pkg.read()
@@ -252,11 +283,12 @@ class DescriptorTopic(BaseTopic):
             if kwargs:
                 self._update_input_with_kwargs(indata, kwargs)
             # it will call overrides method at VnfdTopic or NsdTopic
-            indata = self._validate_input_new(indata, force=force)
+            # indata = self._validate_input_edit(indata, force=force)
 
             deep_update_rfc7396(current_desc, indata)
             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
             self.db.replace(self.topic, _id, current_desc)
+            self.fs.dir_rename(temp_folder, _id)
 
             indata["_id"] = _id
             self._send_msg("created", indata)
@@ -351,6 +383,11 @@ class DescriptorTopic(BaseTopic):
                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
                                                  path_helper=True, skip_unknown=force)
                 out = pybindJSON.dumps(mynsd, mode="ietf")
+            elif item == "nsts":
+                mynst = nst_im()
+                pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
+                                                 path_helper=True, skip_unknown=force)
+                out = pybindJSON.dumps(mynst, mode="ietf")
             else:
                 raise EngineException("Not possible to validate '{}' item".format(item),
                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -389,6 +426,23 @@ class VnfdTopic(DescriptorTopic):
             clean_indata = clean_indata['vnfd:vnfd'][0]
         return clean_indata
 
+    def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
+        super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
+
+        # set type of vnfd
+        contains_pdu = False
+        contains_vdu = False
+        for vdu in get_iterable(final_content.get("vdu")):
+            if vdu.get("pdu-type"):
+                contains_pdu = True
+            else:
+                contains_vdu = True
+        if contains_pdu:
+            final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
+        elif contains_vdu:
+            final_content["_admin"]["type"] = "vnfd"
+        # if neither vud nor pdu do not fill type
+
     def check_conflict_on_del(self, session, _id, force=False):
         """
         Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
@@ -417,21 +471,21 @@ class VnfdTopic(DescriptorTopic):
         if self.db.get_list("nsds", _filter):
             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 
-    def _validate_input_new(self, indata, force=False):
-        # TODO validate with pyangbind, serialize
+    def _validate_input_new(self, indata, storage_params, force=False):
         indata = self.pyangbind_validation("vnfds", indata, force)
         # Cross references validation in the descriptor
-        if not indata.get("mgmt-interface"):
-            raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
-                                  http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
-        if indata["mgmt-interface"].get("cp"):
-            for cp in get_iterable(indata.get("connection-point")):
-                if cp["name"] == indata["mgmt-interface"]["cp"]:
-                    break
-            else:
-                raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
-                                      .format(indata["mgmt-interface"]["cp"]),
+        if indata.get("vdu"):
+            if not indata.get("mgmt-interface"):
+                raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
                                       http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+            if indata["mgmt-interface"].get("cp"):
+                for cp in get_iterable(indata.get("connection-point")):
+                    if cp["name"] == indata["mgmt-interface"]["cp"]:
+                        break
+                else:
+                    raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
+                                          .format(indata["mgmt-interface"]["cp"]),
+                                          http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 
         for vdu in get_iterable(indata.get("vdu")):
             for interface in get_iterable(vdu.get("interface")):
@@ -456,7 +510,34 @@ class VnfdTopic(DescriptorTopic):
                                               .format(vdu["id"], interface["name"],
                                                       interface["internal-connection-point-ref"]),
                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+            # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
+            if vdu.get("vdu-configuration"):
+                if vdu["vdu-configuration"].get("juju"):
+                    if not self._validate_package_folders(storage_params, 'charms'):
+                        raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
+                                              "package".format(indata["id"], vdu["id"]))
+            # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
+            if vdu.get("cloud-init-file"):
+                if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
+                    raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
+                                          "package".format(indata["id"], vdu["id"]))
+        # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
+        if indata.get("vnf-configuration"):
+            if indata["vnf-configuration"].get("juju"):
+                if not self._validate_package_folders(storage_params, 'charms'):
+                    raise EngineException("Charm defined in vnf[id={}] but not present in "
+                                          "package".format(indata["id"]))
+        vld_names = []  # For detection of duplicated VLD names
         for ivld in get_iterable(indata.get("internal-vld")):
+            # BEGIN Detection of duplicated VLD names
+            ivld_name = ivld["name"]
+            if ivld_name in vld_names:
+                raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
+                                      .format(ivld["name"], indata["id"], ivld["id"]),
+                                      http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+            else:
+                vld_names.append(ivld_name)
+            # END Detection of duplicated VLD names
             for icp in get_iterable(ivld.get("internal-connection-point")):
                 icp_mark = False
                 for vdu in get_iterable(indata.get("vdu")):
@@ -554,9 +635,25 @@ class VnfdTopic(DescriptorTopic):
         return indata
 
     def _validate_input_edit(self, indata, force=False):
-        # TODO validate with pyangbind, serialize
+        # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
         return indata
 
+    def _validate_package_folders(self, storage_params, folder, file=None):
+        if not storage_params or not storage_params.get("pkg-dir"):
+            return False
+        else:
+            if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
+                f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
+            else:
+                f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
+            if file:
+                return self.fs.file_exists("{}/{}".format(f, file), 'file')
+            else:
+                if self.fs.file_exists(f, 'dir'):
+                    if self.fs.dir_ls(f):
+                        return True
+            return False
+
 
 class NsdTopic(DescriptorTopic):
     topic = "nsds"
@@ -585,37 +682,88 @@ class NsdTopic(DescriptorTopic):
             clean_indata = clean_indata['nsd:nsd'][0]
         return clean_indata
 
-    def _validate_input_new(self, indata, force=False):
-
-        # TODO validate with pyangbind, serialize
+    def _validate_input_new(self, indata, storage_params, force=False):
         indata = self.pyangbind_validation("nsds", indata, force)
+        # Cross references validation in the descriptor
+        # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
+        for vld in get_iterable(indata.get("vld")):
+            if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
+                raise EngineException("Error at vld[id='{}']:ip-profile-ref"
+                                      " You cannot set an ip-profile when mgmt-network is True"
+                                      .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+            for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
+                for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
+                    if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
+                        if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
+                            raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
+                                                  "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
+                                                  " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
+                                                                 constituent_vnfd["member-vnf-index"],
+                                                                 constituent_vnfd["vnfd-id-ref"]),
+                                                  http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+                        break
+                else:
+                    raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
+                                          "does not match any constituent-vnfd:member-vnf-index"
+                                          .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
+                                          http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
         return indata
 
     def _validate_input_edit(self, indata, force=False):
-        # TODO validate with pyangbind, serialize
+        # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
         return indata
 
-    def _check_descriptor_dependencies(self, session, descriptor):
+    def _check_descriptor_dependencies(self, session, descriptor, force=False):
         """
-        Check that the dependent descriptors exist on a new descriptor or edition
+        Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
+        connection points are ok
         :param session: client session information
         :param descriptor: descriptor to be inserted or edit
+        :param force: if true skip dependencies checking
         :return: None or raises exception
         """
-        if not descriptor.get("constituent-vnfd"):
+        if force:
             return
-        for vnf in descriptor["constituent-vnfd"]:
-            vnfd_id = vnf["vnfd-id-ref"]
-            filter_q = self._get_project_filter(session, write=False, show_all=True)
-            filter_q["id"] = vnfd_id
-            if not self.db.get_list("vnfds", filter_q):
-                raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
-                                      "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
+        member_vnfd_index = {}
+        if descriptor.get("constituent-vnfd") and not force:
+            for vnf in descriptor["constituent-vnfd"]:
+                vnfd_id = vnf["vnfd-id-ref"]
+                filter_q = self._get_project_filter(session, write=False, show_all=True)
+                filter_q["id"] = vnfd_id
+                vnf_list = self.db.get_list("vnfds", filter_q)
+                if not vnf_list:
+                    raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
+                                          "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
+                # elif len(vnf_list) > 1:
+                #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
+                #                           http_code=HTTPStatus.CONFLICT)
+                member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
+
+        # Cross references validation in the descriptor and vnfd connection point validation
+        for vld in get_iterable(descriptor.get("vld")):
+            for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
+                # look if this vnfd contains this connection point
+                vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
+                if not vnfd:
+                    raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
+                                          "does not match any constituent-vnfd:member-vnf-index"
+                                          .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
+                                          http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+                for vnfd_cp in get_iterable(vnfd.get("connection-point")):
+                    if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
+                        break
+                else:
+                    raise EngineException(
+                        "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
+                        "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
+                        .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
+                                referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
+                        http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 
     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 
-        self._check_descriptor_dependencies(session, final_content)
+        self._check_descriptor_dependencies(session, final_content, force)
 
     def check_conflict_on_del(self, session, _id, force=False):
         """
@@ -647,30 +795,24 @@ class NstTopic(DescriptorTopic):
             return {}
         clean_indata = indata
 
-        if clean_indata.get('nst:nst'):
-            clean_indata = clean_indata['nst:nst']
-        elif clean_indata.get('nst'):
-            clean_indata = clean_indata['nst']
         if clean_indata.get('nst'):
             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
                 raise EngineException("'nst' must be a list only one element")
             clean_indata = clean_indata['nst'][0]
+        elif clean_indata.get('nst:nst'):
+            if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
+                raise EngineException("'nst:nst' must be a list only one element")
+            clean_indata = clean_indata['nst:nst'][0]
         return clean_indata
 
-    def _validate_input_new(self, indata, force=False):
-        # transform netslice-subnet:nsd-ref to string
-        if indata.get("netslice-subnet"):
-            for nsd_ref in indata["netslice-subnet"]:
-                if "nsd-ref" in nsd_ref:
-                    nsd_ref["nsd-ref"] = str(nsd_ref["nsd-ref"])
-
-        # TODO validate with pyangbind, serialize
-        return indata
-
     def _validate_input_edit(self, indata, force=False):
         # TODO validate with pyangbind, serialize
         return indata
 
+    def _validate_input_new(self, indata, storage_params, force=False):
+        indata = self.pyangbind_validation("nsts", indata, force)
+        return indata.copy()
+
     def _check_descriptor_dependencies(self, session, descriptor):
         """
         Check that the dependent descriptors exist on a new descriptor or edition
@@ -698,17 +840,25 @@ class NstTopic(DescriptorTopic):
         Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
         that NST can be public and be used by other projects.
         :param session:
-        :param _id: nsd internal id
+        :param _id: nst internal id
         :param force: Avoid this checking
         :return: None or raises EngineException with the conflict
         """
         # TODO: Check this method
         if force:
             return
+        # Get Network Slice Template from Database
         _filter = self._get_project_filter(session, write=False, show_all=False)
-        _filter["nst"] = _id
-        if self.db.get_list("nsis", _filter):
-            raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
+        _filter["_id"] = _id
+        nst = self.db.get_one("nsts", _filter)
+        
+        # Search NSIs using NST via nst-ref
+        _filter = self._get_project_filter(session, write=False, show_all=False)
+        _filter["nst-ref"] = nst["id"]
+        nsis_list = self.db.get_list("nsis", _filter)
+        for nsi_item in nsis_list:
+            if nsi_item["_admin"].get("nsiState") != "TERMINATED":
+                raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 
 
 class PduTopic(BaseTopic):
@@ -722,10 +872,10 @@ class PduTopic(BaseTopic):
 
     @staticmethod
     def format_on_new(content, project_id=None, make_public=False):
-        BaseTopic.format_on_new(content, project_id=None, make_public=make_public)
+        BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
         content["_admin"]["onboardingState"] = "CREATED"
-        content["_admin"]["operationalState"] = "DISABLED"
-        content["_admin"]["usageSate"] = "NOT_IN_USE"
+        content["_admin"]["operationalState"] = "ENABLED"
+        content["_admin"]["usageState"] = "NOT_IN_USE"
 
     def check_conflict_on_del(self, session, _id, force=False):
         if force: