Merge "RBAC permission storage in MongoDB"
[osm/NBI.git] / osm_nbi / descriptor_topics.py
index 6285e38..94b74a9 100644 (file)
@@ -1,5 +1,18 @@
 # -*- coding: utf-8 -*-
 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import tarfile
 import yaml
 import json
@@ -11,6 +24,7 @@ from validation import ValidationError, pdu_new_schema, pdu_edit_schema
 from base_topic import BaseTopic, EngineException, get_iterable
 from osm_im.vnfd import vnfd as vnfd_im
 from osm_im.nsd import nsd as nsd_im
+from osm_im.nst import nst as nst_im
 from pyangbind.lib.serialise import pybindJSONDecoder
 import pyangbind.lib.pybindJSON as pybindJSON
 
@@ -29,7 +43,8 @@ class DescriptorTopic(BaseTopic):
         for k in ("_id", "_admin"):
             if k in final_content:
                 internal_keys[k] = final_content.pop(k)
-        serialized = self._validate_input_new(final_content, force)
+        storage_params = internal_keys["_admin"].get("storage")
+        serialized = self._validate_input_new(final_content, storage_params, force)
         # 1.2. modify final_content with a serialized version
         final_content.clear()
         final_content.update(serialized)
@@ -37,6 +52,8 @@ class DescriptorTopic(BaseTopic):
         for k, v in internal_keys.items():
             final_content[k] = v
 
+        if force:
+            return
         # 2. check that this id is not present
         if "id" in edit_content:
             _filter = self._get_project_filter(session, write=False, show_all=False)
@@ -69,6 +86,7 @@ class DescriptorTopic(BaseTopic):
             return
         v = self.db.del_one(self.topic, {"_id": _id})
         self.fs.file_delete(_id, ignore_non_exist=True)
+        self.fs.file_delete(_id + "_", ignore_non_exist=True)  # remove temp folder
         self._send_msg("delete", {"_id": _id})
         return v
 
@@ -140,7 +158,7 @@ class DescriptorTopic(BaseTopic):
         :param kwargs: user query string to override parameters. NOT USED
         :param headers:  http request headers
         :param force: to be more tolerant with validation
-        :return: True package has is completely uploaded or False if partial content has been uplodaed.
+        :return: True if package is completely uploaded or False if partial content has been uploded
             Raise exception on error
         """
         # Check that _id exists and it is valid
@@ -169,18 +187,19 @@ class DescriptorTopic(BaseTopic):
                 total = int(content_range[3])
             else:
                 start = 0
+            temp_folder = _id + "_"  # all the content is upload here and if ok, it is rename from id_ to is folder
 
             if start:
-                if not self.fs.file_exists(_id, 'dir'):
+                if not self.fs.file_exists(temp_folder, 'dir'):
                     raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
             else:
-                self.fs.file_delete(_id, ignore_non_exist=True)
-                self.fs.mkdir(_id)
+                self.fs.file_delete(temp_folder, ignore_non_exist=True)
+                self.fs.mkdir(temp_folder)
 
             storage = self.fs.get_params()
             storage["folder"] = _id
 
-            file_path = (_id, filename)
+            file_path = (temp_folder, filename)
             if self.fs.file_exists(file_path, 'file'):
                 file_size = self.fs.file_size(file_path)
             else:
@@ -240,8 +259,8 @@ class DescriptorTopic(BaseTopic):
                     raise EngineException("Not found any descriptor file at package descriptor tar.gz")
                 storage["descriptor"] = descriptor_file_name
                 storage["zipfile"] = filename
-                self.fs.file_extract(tar, _id)
-                with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
+                self.fs.file_extract(tar, temp_folder)
+                with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
                     content = descriptor_file.read()
             else:
                 content = file_pkg.read()
@@ -264,11 +283,12 @@ class DescriptorTopic(BaseTopic):
             if kwargs:
                 self._update_input_with_kwargs(indata, kwargs)
             # it will call overrides method at VnfdTopic or NsdTopic
-            indata = self._validate_input_new(indata, force=force)
+            # indata = self._validate_input_edit(indata, force=force)
 
             deep_update_rfc7396(current_desc, indata)
             self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
             self.db.replace(self.topic, _id, current_desc)
+            self.fs.dir_rename(temp_folder, _id)
 
             indata["_id"] = _id
             self._send_msg("created", indata)
@@ -363,6 +383,11 @@ class DescriptorTopic(BaseTopic):
                 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
                                                  path_helper=True, skip_unknown=force)
                 out = pybindJSON.dumps(mynsd, mode="ietf")
+            elif item == "nsts":
+                mynst = nst_im()
+                pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
+                                                 path_helper=True, skip_unknown=force)
+                out = pybindJSON.dumps(mynst, mode="ietf")
             else:
                 raise EngineException("Not possible to validate '{}' item".format(item),
                                       http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -446,7 +471,7 @@ class VnfdTopic(DescriptorTopic):
         if self.db.get_list("nsds", _filter):
             raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
 
-    def _validate_input_new(self, indata, force=False):
+    def _validate_input_new(self, indata, storage_params, force=False):
         indata = self.pyangbind_validation("vnfds", indata, force)
         # Cross references validation in the descriptor
         if indata.get("vdu"):
@@ -485,6 +510,23 @@ class VnfdTopic(DescriptorTopic):
                                               .format(vdu["id"], interface["name"],
                                                       interface["internal-connection-point-ref"]),
                                               http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+            # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
+            if vdu.get("vdu-configuration"):
+                if vdu["vdu-configuration"].get("juju"):
+                    if not self._validate_package_folders(storage_params, 'charms'):
+                        raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
+                                              "package".format(indata["id"], vdu["id"]))
+            # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
+            if vdu.get("cloud-init-file"):
+                if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
+                    raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
+                                          "package".format(indata["id"], vdu["id"]))
+        # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
+        if indata.get("vnf-configuration"):
+            if indata["vnf-configuration"].get("juju"):
+                if not self._validate_package_folders(storage_params, 'charms'):
+                    raise EngineException("Charm defined in vnf[id={}] but not present in "
+                                          "package".format(indata["id"]))
         for ivld in get_iterable(indata.get("internal-vld")):
             for icp in get_iterable(ivld.get("internal-connection-point")):
                 icp_mark = False
@@ -580,13 +622,28 @@ class VnfdTopic(DescriptorTopic):
                                           "vnf-configuration:config-primitive:name"
                                           .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
                                           http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
-        # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
         return indata
 
     def _validate_input_edit(self, indata, force=False):
         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
         return indata
 
+    def _validate_package_folders(self, storage_params, folder, file=None):
+        if not storage_params or not storage_params.get("pkg-dir"):
+            return False
+        else:
+            if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
+                f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
+            else:
+                f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
+            if file:
+                return self.fs.file_exists("{}/{}".format(f, file), 'file')
+            else:
+                if self.fs.file_exists(f, 'dir'):
+                    if self.fs.dir_ls(f):
+                        return True
+            return False
+
 
 class NsdTopic(DescriptorTopic):
     topic = "nsds"
@@ -615,37 +672,84 @@ class NsdTopic(DescriptorTopic):
             clean_indata = clean_indata['nsd:nsd'][0]
         return clean_indata
 
-    def _validate_input_new(self, indata, force=False):
+    def _validate_input_new(self, indata, storage_params, force=False):
         indata = self.pyangbind_validation("nsds", indata, force)
+        # Cross references validation in the descriptor
         # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
+        for vld in get_iterable(indata.get("vld")):
+            for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
+                for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
+                    if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
+                        if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
+                            raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
+                                                  "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
+                                                  " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
+                                                                 constituent_vnfd["member-vnf-index"],
+                                                                 constituent_vnfd["vnfd-id-ref"]),
+                                                  http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+                        break
+                else:
+                    raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
+                                          "does not match any constituent-vnfd:member-vnf-index"
+                                          .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
+                                          http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
         return indata
 
     def _validate_input_edit(self, indata, force=False):
         # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
         return indata
 
-    def _check_descriptor_dependencies(self, session, descriptor):
+    def _check_descriptor_dependencies(self, session, descriptor, force=False):
         """
-        Check that the dependent descriptors exist on a new descriptor or edition
+        Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
+        connection points are ok
         :param session: client session information
         :param descriptor: descriptor to be inserted or edit
+        :param force: if true skip dependencies checking
         :return: None or raises exception
         """
-        if not descriptor.get("constituent-vnfd"):
+        if force:
             return
-        for vnf in descriptor["constituent-vnfd"]:
-            vnfd_id = vnf["vnfd-id-ref"]
-            filter_q = self._get_project_filter(session, write=False, show_all=True)
-            filter_q["id"] = vnfd_id
-            if not self.db.get_list("vnfds", filter_q):
-                raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
-                                      "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
+        member_vnfd_index = {}
+        if descriptor.get("constituent-vnfd") and not force:
+            for vnf in descriptor["constituent-vnfd"]:
+                vnfd_id = vnf["vnfd-id-ref"]
+                filter_q = self._get_project_filter(session, write=False, show_all=True)
+                filter_q["id"] = vnfd_id
+                vnf_list = self.db.get_list("vnfds", filter_q)
+                if not vnf_list:
+                    raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
+                                          "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
+                # elif len(vnf_list) > 1:
+                #     raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
+                #                           http_code=HTTPStatus.CONFLICT)
+                member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
+
+        # Cross references validation in the descriptor and vnfd connection point validation
+        for vld in get_iterable(descriptor.get("vld")):
+            for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
+                # look if this vnfd contains this connection point
+                vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
+                if not vnfd:
+                    raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
+                                          "does not match any constituent-vnfd:member-vnf-index"
+                                          .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
+                                          http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+                for vnfd_cp in get_iterable(vnfd.get("connection-point")):
+                    if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
+                        break
+                else:
+                    raise EngineException(
+                        "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
+                        "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
+                        .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
+                                referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
+                        http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
 
     def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
         super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
 
-        if not force:
-            self._check_descriptor_dependencies(session, final_content)
+        self._check_descriptor_dependencies(session, final_content, force)
 
     def check_conflict_on_del(self, session, _id, force=False):
         """
@@ -677,20 +781,24 @@ class NstTopic(DescriptorTopic):
             return {}
         clean_indata = indata
 
-        if clean_indata.get('nst:nst'):
-            clean_indata = clean_indata['nst:nst']
-        elif clean_indata.get('nst'):
-            clean_indata = clean_indata['nst']
         if clean_indata.get('nst'):
             if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
                 raise EngineException("'nst' must be a list only one element")
             clean_indata = clean_indata['nst'][0]
+        elif clean_indata.get('nst:nst'):
+            if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
+                raise EngineException("'nst:nst' must be a list only one element")
+            clean_indata = clean_indata['nst:nst'][0]
         return clean_indata
 
     def _validate_input_edit(self, indata, force=False):
         # TODO validate with pyangbind, serialize
         return indata
 
+    def _validate_input_new(self, indata, storage_params, force=False):
+        indata = self.pyangbind_validation("nsts", indata, force)
+        return indata.copy()
+
     def _check_descriptor_dependencies(self, session, descriptor):
         """
         Check that the dependent descriptors exist on a new descriptor or edition
@@ -728,13 +836,15 @@ class NstTopic(DescriptorTopic):
         # Get Network Slice Template from Database
         _filter = self._get_project_filter(session, write=False, show_all=False)
         _filter["_id"] = _id
-        nst = self.db.get_one("nst", _filter)
+        nst = self.db.get_one("nsts", _filter)
         
         # Search NSIs using NST via nst-ref
         _filter = self._get_project_filter(session, write=False, show_all=False)
         _filter["nst-ref"] = nst["id"]
-        if self.db.get_list("nsis", _filter):
-            raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
+        nsis_list = self.db.get_list("nsis", _filter)
+        for nsi_item in nsis_list:
+            if nsi_item["_admin"].get("nsiState") != "TERMINATED":
+                raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
 
 
 class PduTopic(BaseTopic):