Feature 10908: NBI Validation of packages upload 66/11866/12
authoraticig <gulsum.atici@canonical.com>
Thu, 7 Apr 2022 08:57:18 +0000 (11:57 +0300)
committergarciadeblas <gerardo.garciadeblas@telefonica.com>
Wed, 18 May 2022 13:45:07 +0000 (15:45 +0200)
Descriptor validation methods are added for package update operations.

Change-Id: I38aa0319c698fcc7ff87f95712d2cef83c3aff8b
Signed-off-by: aticig <gulsum.atici@canonical.com>
osm_nbi/descriptor_topics.py
osm_nbi/tests/test_descriptor_topics.py
requirements-dev.txt
requirements-test.in
requirements-test.txt
requirements.in
requirements.txt

index 4661964..590380a 100644 (file)
@@ -19,8 +19,10 @@ import json
 import copy
 import os
 import shutil
+import functools
 
 # import logging
+from deepdiff import DeepDiff
 from hashlib import md5
 from osm_common.dbbase import DbException, deep_update_rfc7396
 from http import HTTPStatus
@@ -1184,6 +1186,158 @@ class VnfdTopic(DescriptorTopic):
 
         return super().sol005_projection(data)
 
+    @staticmethod
+    def find_software_version(vnfd: dict) -> str:
+        """Find the sotware version in the VNFD descriptors
+
+        Args:
+            vnfd (dict): Descriptor as a dictionary
+
+        Returns:
+            software-version (str)
+        """
+        default_sw_version = "1.0"
+        if vnfd.get("vnfd"):
+            vnfd = vnfd["vnfd"]
+        if vnfd.get("software-version"):
+            return vnfd["software-version"]
+        else:
+            return default_sw_version
+
+    @staticmethod
+    def extract_policies(vnfd: dict) -> dict:
+        """Removes the policies from the VNFD descriptors
+
+        Args:
+            vnfd (dict):   Descriptor as a dictionary
+
+        Returns:
+            vnfd (dict): VNFD which does not include policies
+        """
+        # TODO: Extract the policy related parts from the VNFD
+        return vnfd
+
+    @staticmethod
+    def extract_day12_primitives(vnfd: dict) -> dict:
+        """Removes the day12 primitives from the VNFD descriptors
+
+        Args:
+            vnfd (dict):   Descriptor as a dictionary
+
+        Returns:
+            vnfd (dict)
+        """
+        for df_id, df in enumerate(vnfd.get("df", {})):
+            if (
+                df.get("lcm-operations-configuration", {})
+                .get("operate-vnf-op-config", {})
+                .get("day1-2")
+            ):
+                day12 = df["lcm-operations-configuration"]["operate-vnf-op-config"].get(
+                    "day1-2"
+                )
+                for config_id, config in enumerate(day12):
+                    for key in [
+                        "initial-config-primitive",
+                        "config-primitive",
+                        "terminate-config-primitive",
+                    ]:
+                        config.pop(key, None)
+                        day12[config_id] = config
+                df["lcm-operations-configuration"]["operate-vnf-op-config"][
+                    "day1-2"
+                ] = day12
+            vnfd["df"][df_id] = df
+        return vnfd
+
+    def remove_modifiable_items(self, vnfd: dict) -> dict:
+        """Removes the modifiable parts from the VNFD descriptors
+
+        It calls different extract functions according to different update types
+        to clear all the modifiable items from VNFD
+
+        Args:
+            vnfd (dict): Descriptor as a dictionary
+
+        Returns:
+            vnfd (dict): Descriptor which does not include modifiable contents
+        """
+        if vnfd.get("vnfd"):
+            vnfd = vnfd["vnfd"]
+        vnfd.pop("_admin", None)
+        # If the other extractions need to be done from VNFD,
+        # the new extract methods could be appended to below list.
+        for extract_function in [self.extract_day12_primitives, self.extract_policies]:
+            vnfd_temp = extract_function(vnfd)
+            vnfd = vnfd_temp
+        return vnfd
+
+    def _validate_descriptor_changes(
+        self,
+        descriptor_file_name: str,
+        old_descriptor_directory: str,
+        new_descriptor_directory: str,
+    ):
+        """Compares the old and new VNFD descriptors and validates the new descriptor.
+
+        Args:
+            old_descriptor_directory (str):   Directory of descriptor which is in-use
+            new_descriptor_directory (str):   Directory of directory which is proposed to update (new revision)
+
+        Returns:
+            None
+
+        Raises:
+            EngineException:    In case of error when there are unallowed changes
+        """
+        try:
+            with self.fs.file_open(
+                (old_descriptor_directory.rstrip("/"), descriptor_file_name), "r"
+            ) as old_descriptor_file:
+                with self.fs.file_open(
+                    (new_descriptor_directory, descriptor_file_name), "r"
+                ) as new_descriptor_file:
+                    old_content = yaml.load(
+                        old_descriptor_file.read(), Loader=yaml.SafeLoader
+                    )
+                    new_content = yaml.load(
+                        new_descriptor_file.read(), Loader=yaml.SafeLoader
+                    )
+                    if old_content and new_content:
+                        if self.find_software_version(
+                            old_content
+                        ) != self.find_software_version(new_content):
+                            return
+                        disallowed_change = DeepDiff(
+                            self.remove_modifiable_items(old_content),
+                            self.remove_modifiable_items(new_content),
+                        )
+                        if disallowed_change:
+                            changed_nodes = functools.reduce(
+                                lambda a, b: a + " , " + b,
+                                [
+                                    node.lstrip("root")
+                                    for node in disallowed_change.get(
+                                        "values_changed"
+                                    ).keys()
+                                ],
+                            )
+                            raise EngineException(
+                                f"Error in validating new descriptor: {changed_nodes} cannot be modified, "
+                                "there are disallowed changes in the vnf descriptor.",
+                                http_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+                            )
+        except (
+            DbException,
+            AttributeError,
+            IndexError,
+            KeyError,
+            ValueError,
+        ) as e:
+            raise type(e)(
+                "VNF Descriptor could not be processed with error: {}.".format(e)
+            )
+
 
 class NsdTopic(DescriptorTopic):
     topic = "nsds"
@@ -1458,6 +1612,112 @@ class NsdTopic(DescriptorTopic):
         super().delete_extra(session, _id, db_content, not_send_msg)
         self.db.del_list(self.topic+"_revisions", { "_id": { "$regex": _id}})
 
+    @staticmethod
+    def extract_day12_primitives(nsd: dict) -> dict:
+        """Removes the day12 primitives from the NSD descriptors
+
+        Args:
+            nsd (dict):    Descriptor as a dictionary
+
+        Returns:
+            nsd (dict):    Cleared NSD
+        """
+        if nsd.get("ns-configuration"):
+            for key in [
+                "config-primitive",
+                "initial-config-primitive",
+                "terminate-config-primitive",
+            ]:
+                nsd["ns-configuration"].pop(key, None)
+        return nsd
+
+    def remove_modifiable_items(self, nsd: dict) -> dict:
+        """Removes the modifiable parts from the VNFD descriptors
+
+        It calls different extract functions according to different update types
+        to clear all the modifiable items from NSD
+
+        Args:
+            nsd (dict):  Descriptor as a dictionary
+
+        Returns:
+            nsd (dict):  Descriptor which does not include modifiable contents
+        """
+        while isinstance(nsd, dict) and nsd.get("nsd"):
+            nsd = nsd["nsd"]
+        if isinstance(nsd, list):
+            nsd = nsd[0]
+        nsd.pop("_admin", None)
+        # If the more extractions need to be done from NSD,
+        # the new extract methods could be appended to below list.
+        for extract_function in [self.extract_day12_primitives]:
+            nsd_temp = extract_function(nsd)
+            nsd = nsd_temp
+        return nsd
+
+    def _validate_descriptor_changes(
+        self,
+        descriptor_file_name: str,
+        old_descriptor_directory: str,
+        new_descriptor_directory: str,
+    ):
+        """Compares the old and new NSD descriptors and validates the new descriptor
+
+        Args:
+            old_descriptor_directory:   Directory of descriptor which is in-use
+            new_descriptor_directory:   Directory of directory which is proposed to update (new revision)
+
+        Returns:
+            None
+
+        Raises:
+            EngineException:    In case of error if the changes are not allowed
+        """
+
+        try:
+            with self.fs.file_open(
+                (old_descriptor_directory, descriptor_file_name), "r"
+            ) as old_descriptor_file:
+                with self.fs.file_open(
+                    (new_descriptor_directory.rstrip("/"), descriptor_file_name), "r"
+                ) as new_descriptor_file:
+                    old_content = yaml.load(
+                        old_descriptor_file.read(), Loader=yaml.SafeLoader
+                    )
+                    new_content = yaml.load(
+                        new_descriptor_file.read(), Loader=yaml.SafeLoader
+                    )
+                    if old_content and new_content:
+                        disallowed_change = DeepDiff(
+                            self.remove_modifiable_items(old_content),
+                            self.remove_modifiable_items(new_content),
+                        )
+                        if disallowed_change:
+                            changed_nodes = functools.reduce(
+                                lambda a, b: a + ", " + b,
+                                [
+                                    node.lstrip("root")
+                                    for node in disallowed_change.get(
+                                        "values_changed"
+                                    ).keys()
+                                ],
+                            )
+                            raise EngineException(
+                                f"Error in validating new descriptor: {changed_nodes} cannot be modified, "
+                                "there are disallowed changes in the ns descriptor. ",
+                                http_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+                            )
+        except (
+            DbException,
+            AttributeError,
+            IndexError,
+            KeyError,
+            ValueError,
+        ) as e:
+            raise type(e)(
+                "NS Descriptor could not be processed with error: {}.".format(e)
+            )
+
     def sol005_projection(self, data):
         data["nsdOnboardingState"] = data["_admin"]["onboardingState"]
         data["nsdOperationalState"] = data["_admin"]["operationalState"]
index 280c93f..6b9b2c3 100755 (executable)
@@ -17,6 +17,7 @@
 __author__ = "Pedro de la Cruz Ramos, pedro.delacruzramos@altran.com"
 __date__ = "2019-11-20"
 
+from contextlib import contextmanager
 import unittest
 from unittest import TestCase
 from unittest.mock import Mock, patch
@@ -92,6 +93,18 @@ class Test_VnfdTopic(TestCase):
         self.topic = VnfdTopic(self.db, self.fs, self.msg, self.auth)
         self.topic.check_quota = Mock(return_value=None)  # skip quota
 
+    @contextmanager
+    def assertNotRaises(self, exception_type):
+        try:
+            yield None
+        except exception_type:
+            raise self.failureException("{} raised".format(exception_type.__name__))
+
+    def create_desc_temp(self, template):
+        old_desc = deepcopy(template)
+        new_desc = deepcopy(template)
+        return old_desc, new_desc
+
     @patch("osm_nbi.descriptor_topics.shutil")
     @patch("osm_nbi.descriptor_topics.os.rename")
     def test_new_vnfd(self, mock_rename, mock_shutil):
@@ -294,7 +307,6 @@ class Test_VnfdTopic(TestCase):
                 self.topic.upload_content(
                     fake_session, did, test_vnfd, {}, {"Content-Type": []}
                 )
-            print(str(e.exception))
             self.assertEqual(
                 e.exception.http_code, HTTPStatus.BAD_REQUEST, "Wrong HTTP status code"
             )
@@ -802,6 +814,52 @@ class Test_VnfdTopic(TestCase):
             self.fs.file_delete.assert_not_called()
         return
 
+    @patch("osm_nbi.descriptor_topics.yaml")
+    def test_validate_descriptor_changes(self, mock_yaml):
+        descriptor_name = "test_descriptor"
+        self.fs.file_open.side_effect = lambda path, mode: open(
+            "/tmp/" + str(uuid4()), "a+b"
+        )
+        with self.subTest(i=1, t="VNFD has changes in day1-2 config primitive"):
+            old_vnfd, new_vnfd = self.create_desc_temp(db_vnfd_content)
+            new_vnfd["df"][0]["lcm-operations-configuration"]["operate-vnf-op-config"][
+                "day1-2"
+            ][0]["config-primitive"][0]["name"] = "new_action"
+            mock_yaml.load.side_effect = [old_vnfd, new_vnfd]
+            with self.assertNotRaises(EngineException):
+                self.topic._validate_descriptor_changes(
+                    descriptor_name, "/tmp/", "/tmp:1/"
+                )
+        with self.subTest(i=2, t="VNFD sw version changed"):
+            # old vnfd uses the default software version: 1.0
+            new_vnfd["software-version"] = "1.3"
+            new_vnfd["sw-image-desc"][0]["name"] = "new-image"
+            mock_yaml.load.side_effect = [old_vnfd, new_vnfd]
+            with self.assertNotRaises(EngineException):
+                self.topic._validate_descriptor_changes(
+                    descriptor_name, "/tmp/", "/tmp:1/"
+                )
+        with self.subTest(
+            i=3, t="VNFD sw version is not changed and mgmt-cp has changed"
+        ):
+            old_vnfd, new_vnfd = self.create_desc_temp(db_vnfd_content)
+            new_vnfd["mgmt-cp"] = "new-mgmt-cp"
+            mock_yaml.load.side_effect = [old_vnfd, new_vnfd]
+            with self.assertRaises(EngineException) as e:
+                self.topic._validate_descriptor_changes(
+                    descriptor_name, "/tmp/", "/tmp:1/"
+                )
+            self.assertEqual(
+                e.exception.http_code,
+                HTTPStatus.UNPROCESSABLE_ENTITY,
+                "Wrong HTTP status code",
+            )
+            self.assertIn(
+                norm("there are disallowed changes in the vnf descriptor"),
+                norm(str(e.exception)),
+                "Wrong exception text",
+            )
+
     def test_validate_mgmt_interface_connection_point_on_valid_descriptor(self):
         indata = deepcopy(db_vnfd_content)
         self.topic.validate_mgmt_interface_connection_point(indata)
@@ -1247,6 +1305,18 @@ class Test_NsdTopic(TestCase):
         self.topic = NsdTopic(self.db, self.fs, self.msg, self.auth)
         self.topic.check_quota = Mock(return_value=None)  # skip quota
 
+    @contextmanager
+    def assertNotRaises(self, exception_type):
+        try:
+            yield None
+        except exception_type:
+            raise self.failureException("{} raised".format(exception_type.__name__))
+
+    def create_desc_temp(self, template):
+        old_desc = deepcopy(template)
+        new_desc = deepcopy(template)
+        return old_desc, new_desc
+
     @patch("osm_nbi.descriptor_topics.shutil")
     @patch("osm_nbi.descriptor_topics.os.rename")
     def test_new_nsd(self, mock_rename, mock_shutil):
@@ -1732,6 +1802,46 @@ class Test_NsdTopic(TestCase):
             self.fs.file_delete.assert_not_called()
         return
 
+    @patch("osm_nbi.descriptor_topics.yaml")
+    def test_validate_descriptor_changes(self, mock_yaml):
+        descriptor_name = "test_ns_descriptor"
+        self.fs.file_open.side_effect = lambda path, mode: open(
+            "/tmp/" + str(uuid4()), "a+b"
+        )
+        with self.subTest(
+            i=1, t="NSD has changes in ns-configuration:config-primitive"
+        ):
+            old_nsd, new_nsd = self.create_desc_temp(db_nsd_content)
+            old_nsd.update(
+                {"ns-configuration": {"config-primitive": [{"name": "add-user"}]}}
+            )
+            new_nsd.update(
+                {"ns-configuration": {"config-primitive": [{"name": "del-user"}]}}
+            )
+            mock_yaml.load.side_effect = [old_nsd, new_nsd]
+            with self.assertNotRaises(EngineException):
+                self.topic._validate_descriptor_changes(
+                    descriptor_name, "/tmp", "/tmp:1"
+                )
+        with self.subTest(i=2, t="NSD name has changed"):
+            old_nsd, new_nsd = self.create_desc_temp(db_nsd_content)
+            new_nsd["name"] = "nscharm-ns2"
+            mock_yaml.load.side_effect = [old_nsd, new_nsd]
+            with self.assertRaises(EngineException) as e:
+                self.topic._validate_descriptor_changes(
+                    descriptor_name, "/tmp", "/tmp:1"
+                )
+            self.assertEqual(
+                e.exception.http_code,
+                HTTPStatus.UNPROCESSABLE_ENTITY,
+                "Wrong HTTP status code",
+            )
+            self.assertIn(
+                norm("there are disallowed changes in the ns descriptor"),
+                norm(str(e.exception)),
+                "Wrong exception text",
+            )
+
     def test_validate_vld_mgmt_network_with_virtual_link_protocol_data_on_valid_descriptor(
         self,
     ):
index a5862bc..9a0dc90 100644 (file)
@@ -16,7 +16,7 @@
 
 aiokafka==0.7.2
     # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
-bitarray==1.8.1
+bitarray==2.3.5
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
     #   pyangbind
@@ -53,7 +53,7 @@ pyyaml==5.4.1
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
     #   -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master
-regex==2021.3.17
+regex==2021.11.10
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master
     #   pyangbind
index 4564164..99848ac 100644 (file)
@@ -15,6 +15,7 @@ aiohttp>=2.3.10,<=3.6.2
 aioresponses
 asynctest
 coverage
+deepdiff
 nose2
 requests==2.25.1
 pyang
index b71b0f2..143d7c8 100644 (file)
@@ -32,29 +32,33 @@ chardet==3.0.4
     # via
     #   aiohttp
     #   requests
-coverage==6.2
+coverage==6.3.2
     # via
     #   -r requirements-test.in
     #   nose2
-idna==3.3
+deepdiff==5.7.0
+    # via -r requirements-test.in
+idna==2.10
     # via
     #   requests
     #   yarl
-lxml==4.7.1
+lxml==4.8.0
     # via pyang
 multidict==4.7.6
     # via
     #   aiohttp
     #   yarl
-nose2==0.10.0
+nose2==0.11.0
     # via -r requirements-test.in
-pyang==2.5.2
+ordered-set==4.0.2
+    # via deepdiff
+pyang==2.5.3
     # via -r requirements-test.in
-requests==2.27.1
+requests==2.25.1
     # via -r requirements-test.in
 six==1.16.0
     # via nose2
-urllib3==1.26.8
+urllib3==1.26.9
     # via requests
 yarl==1.7.2
     # via aiohttp
index a961d05..5476674 100644 (file)
@@ -12,6 +12,7 @@
 
 aiohttp>=2.3.10,<=3.6.2
 CherryPy>=18.1.2
+deepdiff
 jsonschema>=3.2.0
 python-keystoneclient
 pyyaml==5.4.1
index b8a375f..ad4d82c 100644 (file)
@@ -26,22 +26,24 @@ certifi==2021.10.8
     # via requests
 chardet==3.0.4
     # via aiohttp
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 cheroot==8.6.0
     # via cherrypy
 cherrypy==18.6.1
     # via -r requirements.in
-debtcollector==2.4.0
+debtcollector==2.5.0
     # via
     #   oslo.config
     #   oslo.utils
     #   python-keystoneclient
+deepdiff==5.7.0
+    # via -r requirements.in
 idna==3.3
     # via
     #   requests
     #   yarl
-importlib-resources==5.4.0
+importlib-resources==5.6.0
     # via
     #   jaraco.text
     #   jsonschema
@@ -53,16 +55,18 @@ jaraco.classes==3.2.1
     # via jaraco.collections
 jaraco.collections==3.5.1
     # via cherrypy
+jaraco.context==4.1.1
+    # via jaraco.text
 jaraco.functools==3.5.0
     # via
     #   cheroot
     #   jaraco.text
     #   tempora
-jaraco.text==3.6.0
+jaraco.text==3.7.0
     # via jaraco.collections
 jsonschema==4.4.0
     # via -r requirements.in
-keystoneauth1==4.4.0
+keystoneauth1==4.5.0
     # via python-keystoneclient
 more-itertools==8.12.0
     # via
@@ -82,26 +86,27 @@ netaddr==0.8.0
     #   oslo.utils
 netifaces==0.11.0
     # via oslo.utils
+ordered-set==4.0.2
+    # via deepdiff
 os-service-types==1.7.0
     # via keystoneauth1
-oslo.config==8.7.1
+oslo.config==8.8.0
     # via python-keystoneclient
 oslo.i18n==5.1.0
     # via
     #   oslo.config
     #   oslo.utils
     #   python-keystoneclient
-oslo.serialization==4.2.0
+oslo.serialization==4.3.0
     # via python-keystoneclient
-oslo.utils==4.12.0
+oslo.utils==4.12.2
     # via
     #   oslo.serialization
     #   python-keystoneclient
 packaging==21.3
     # via oslo.utils
-pbr==5.8.0
+pbr==5.8.1
     # via
-    #   debtcollector
     #   keystoneauth1
     #   os-service-types
     #   oslo.i18n
@@ -111,7 +116,7 @@ pbr==5.8.0
     #   stevedore
 portend==3.1.0
     # via cherrypy
-pyparsing==3.0.6
+pyparsing==3.0.7
     # via
     #   oslo.utils
     #   packaging
@@ -119,7 +124,7 @@ pyrsistent==0.18.1
     # via jsonschema
 python-keystoneclient==4.4.0
     # via -r requirements.in
-pytz==2021.3
+pytz==2022.1
     # via
     #   oslo.serialization
     #   oslo.utils
@@ -147,19 +152,19 @@ stevedore==3.5.0
     #   keystoneauth1
     #   oslo.config
     #   python-keystoneclient
-tacacs-plus==2.6
+tacacs_plus==2.6
     # via -r requirements.in
-tempora==5.0.0
+tempora==5.0.1
     # via portend
-urllib3==1.26.8
+urllib3==1.26.9
     # via requests
-wrapt==1.13.3
+wrapt==1.14.0
     # via debtcollector
 yarl==1.7.2
     # via aiohttp
 zc.lockfile==2.0
     # via cherrypy
-zipp==3.7.0
+zipp==3.8.0
     # via importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file: