From: aticig Date: Thu, 7 Apr 2022 08:57:18 +0000 (+0300) Subject: Feature 10908: NBI Validation of packages upload X-Git-Tag: v12.0.0rc1~19 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FNBI.git;a=commitdiff_plain;h=9cfa8163385df6fca38094a4750f6cab22dbf250 Feature 10908: NBI Validation of packages upload Descriptor validation methods are added for package update operations. Change-Id: I38aa0319c698fcc7ff87f95712d2cef83c3aff8b Signed-off-by: aticig --- diff --git a/osm_nbi/descriptor_topics.py b/osm_nbi/descriptor_topics.py index 4661964..590380a 100644 --- a/osm_nbi/descriptor_topics.py +++ b/osm_nbi/descriptor_topics.py @@ -19,8 +19,10 @@ import json import copy import os import shutil +import functools # import logging +from deepdiff import DeepDiff from hashlib import md5 from osm_common.dbbase import DbException, deep_update_rfc7396 from http import HTTPStatus @@ -1184,6 +1186,158 @@ class VnfdTopic(DescriptorTopic): return super().sol005_projection(data) + @staticmethod + def find_software_version(vnfd: dict) -> str: + """Find the sotware version in the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + software-version (str) + """ + default_sw_version = "1.0" + if vnfd.get("vnfd"): + vnfd = vnfd["vnfd"] + if vnfd.get("software-version"): + return vnfd["software-version"] + else: + return default_sw_version + + @staticmethod + def extract_policies(vnfd: dict) -> dict: + """Removes the policies from the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict): VNFD which does not include policies + """ + # TODO: Extract the policy related parts from the VNFD + return vnfd + + @staticmethod + def extract_day12_primitives(vnfd: dict) -> dict: + """Removes the day12 primitives from the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict) + """ + for df_id, df in enumerate(vnfd.get("df", {})): + if ( + df.get("lcm-operations-configuration", {}) + .get("operate-vnf-op-config", {}) + .get("day1-2") + ): + day12 = df["lcm-operations-configuration"]["operate-vnf-op-config"].get( + "day1-2" + ) + for config_id, config in enumerate(day12): + for key in [ + "initial-config-primitive", + "config-primitive", + "terminate-config-primitive", + ]: + config.pop(key, None) + day12[config_id] = config + df["lcm-operations-configuration"]["operate-vnf-op-config"][ + "day1-2" + ] = day12 + vnfd["df"][df_id] = df + return vnfd + + def remove_modifiable_items(self, vnfd: dict) -> dict: + """Removes the modifiable parts from the VNFD descriptors + + It calls different extract functions according to different update types + to clear all the modifiable items from VNFD + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict): Descriptor which does not include modifiable contents + """ + if vnfd.get("vnfd"): + vnfd = vnfd["vnfd"] + vnfd.pop("_admin", None) + # If the other extractions need to be done from VNFD, + # the new extract methods could be appended to below list. + for extract_function in [self.extract_day12_primitives, self.extract_policies]: + vnfd_temp = extract_function(vnfd) + vnfd = vnfd_temp + return vnfd + + def _validate_descriptor_changes( + self, + descriptor_file_name: str, + old_descriptor_directory: str, + new_descriptor_directory: str, + ): + """Compares the old and new VNFD descriptors and validates the new descriptor. + + Args: + old_descriptor_directory (str): Directory of descriptor which is in-use + new_descriptor_directory (str): Directory of directory which is proposed to update (new revision) + + Returns: + None + + Raises: + EngineException: In case of error when there are unallowed changes + """ + try: + with self.fs.file_open( + (old_descriptor_directory.rstrip("/"), descriptor_file_name), "r" + ) as old_descriptor_file: + with self.fs.file_open( + (new_descriptor_directory, descriptor_file_name), "r" + ) as new_descriptor_file: + old_content = yaml.load( + old_descriptor_file.read(), Loader=yaml.SafeLoader + ) + new_content = yaml.load( + new_descriptor_file.read(), Loader=yaml.SafeLoader + ) + if old_content and new_content: + if self.find_software_version( + old_content + ) != self.find_software_version(new_content): + return + disallowed_change = DeepDiff( + self.remove_modifiable_items(old_content), + self.remove_modifiable_items(new_content), + ) + if disallowed_change: + changed_nodes = functools.reduce( + lambda a, b: a + " , " + b, + [ + node.lstrip("root") + for node in disallowed_change.get( + "values_changed" + ).keys() + ], + ) + raise EngineException( + f"Error in validating new descriptor: {changed_nodes} cannot be modified, " + "there are disallowed changes in the vnf descriptor.", + http_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except ( + DbException, + AttributeError, + IndexError, + KeyError, + ValueError, + ) as e: + raise type(e)( + "VNF Descriptor could not be processed with error: {}.".format(e) + ) + class NsdTopic(DescriptorTopic): topic = "nsds" @@ -1458,6 +1612,112 @@ class NsdTopic(DescriptorTopic): super().delete_extra(session, _id, db_content, not_send_msg) self.db.del_list(self.topic+"_revisions", { "_id": { "$regex": _id}}) + @staticmethod + def extract_day12_primitives(nsd: dict) -> dict: + """Removes the day12 primitives from the NSD descriptors + + Args: + nsd (dict): Descriptor as a dictionary + + Returns: + nsd (dict): Cleared NSD + """ + if nsd.get("ns-configuration"): + for key in [ + "config-primitive", + "initial-config-primitive", + "terminate-config-primitive", + ]: + nsd["ns-configuration"].pop(key, None) + return nsd + + def remove_modifiable_items(self, nsd: dict) -> dict: + """Removes the modifiable parts from the VNFD descriptors + + It calls different extract functions according to different update types + to clear all the modifiable items from NSD + + Args: + nsd (dict): Descriptor as a dictionary + + Returns: + nsd (dict): Descriptor which does not include modifiable contents + """ + while isinstance(nsd, dict) and nsd.get("nsd"): + nsd = nsd["nsd"] + if isinstance(nsd, list): + nsd = nsd[0] + nsd.pop("_admin", None) + # If the more extractions need to be done from NSD, + # the new extract methods could be appended to below list. + for extract_function in [self.extract_day12_primitives]: + nsd_temp = extract_function(nsd) + nsd = nsd_temp + return nsd + + def _validate_descriptor_changes( + self, + descriptor_file_name: str, + old_descriptor_directory: str, + new_descriptor_directory: str, + ): + """Compares the old and new NSD descriptors and validates the new descriptor + + Args: + old_descriptor_directory: Directory of descriptor which is in-use + new_descriptor_directory: Directory of directory which is proposed to update (new revision) + + Returns: + None + + Raises: + EngineException: In case of error if the changes are not allowed + """ + + try: + with self.fs.file_open( + (old_descriptor_directory, descriptor_file_name), "r" + ) as old_descriptor_file: + with self.fs.file_open( + (new_descriptor_directory.rstrip("/"), descriptor_file_name), "r" + ) as new_descriptor_file: + old_content = yaml.load( + old_descriptor_file.read(), Loader=yaml.SafeLoader + ) + new_content = yaml.load( + new_descriptor_file.read(), Loader=yaml.SafeLoader + ) + if old_content and new_content: + disallowed_change = DeepDiff( + self.remove_modifiable_items(old_content), + self.remove_modifiable_items(new_content), + ) + if disallowed_change: + changed_nodes = functools.reduce( + lambda a, b: a + ", " + b, + [ + node.lstrip("root") + for node in disallowed_change.get( + "values_changed" + ).keys() + ], + ) + raise EngineException( + f"Error in validating new descriptor: {changed_nodes} cannot be modified, " + "there are disallowed changes in the ns descriptor. ", + http_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except ( + DbException, + AttributeError, + IndexError, + KeyError, + ValueError, + ) as e: + raise type(e)( + "NS Descriptor could not be processed with error: {}.".format(e) + ) + def sol005_projection(self, data): data["nsdOnboardingState"] = data["_admin"]["onboardingState"] data["nsdOperationalState"] = data["_admin"]["operationalState"] diff --git a/osm_nbi/tests/test_descriptor_topics.py b/osm_nbi/tests/test_descriptor_topics.py index 280c93f..6b9b2c3 100755 --- a/osm_nbi/tests/test_descriptor_topics.py +++ b/osm_nbi/tests/test_descriptor_topics.py @@ -17,6 +17,7 @@ __author__ = "Pedro de la Cruz Ramos, pedro.delacruzramos@altran.com" __date__ = "2019-11-20" +from contextlib import contextmanager import unittest from unittest import TestCase from unittest.mock import Mock, patch @@ -92,6 +93,18 @@ class Test_VnfdTopic(TestCase): self.topic = VnfdTopic(self.db, self.fs, self.msg, self.auth) self.topic.check_quota = Mock(return_value=None) # skip quota + @contextmanager + def assertNotRaises(self, exception_type): + try: + yield None + except exception_type: + raise self.failureException("{} raised".format(exception_type.__name__)) + + def create_desc_temp(self, template): + old_desc = deepcopy(template) + new_desc = deepcopy(template) + return old_desc, new_desc + @patch("osm_nbi.descriptor_topics.shutil") @patch("osm_nbi.descriptor_topics.os.rename") def test_new_vnfd(self, mock_rename, mock_shutil): @@ -294,7 +307,6 @@ class Test_VnfdTopic(TestCase): self.topic.upload_content( fake_session, did, test_vnfd, {}, {"Content-Type": []} ) - print(str(e.exception)) self.assertEqual( e.exception.http_code, HTTPStatus.BAD_REQUEST, "Wrong HTTP status code" ) @@ -802,6 +814,52 @@ class Test_VnfdTopic(TestCase): self.fs.file_delete.assert_not_called() return + @patch("osm_nbi.descriptor_topics.yaml") + def test_validate_descriptor_changes(self, mock_yaml): + descriptor_name = "test_descriptor" + self.fs.file_open.side_effect = lambda path, mode: open( + "/tmp/" + str(uuid4()), "a+b" + ) + with self.subTest(i=1, t="VNFD has changes in day1-2 config primitive"): + old_vnfd, new_vnfd = self.create_desc_temp(db_vnfd_content) + new_vnfd["df"][0]["lcm-operations-configuration"]["operate-vnf-op-config"][ + "day1-2" + ][0]["config-primitive"][0]["name"] = "new_action" + mock_yaml.load.side_effect = [old_vnfd, new_vnfd] + with self.assertNotRaises(EngineException): + self.topic._validate_descriptor_changes( + descriptor_name, "/tmp/", "/tmp:1/" + ) + with self.subTest(i=2, t="VNFD sw version changed"): + # old vnfd uses the default software version: 1.0 + new_vnfd["software-version"] = "1.3" + new_vnfd["sw-image-desc"][0]["name"] = "new-image" + mock_yaml.load.side_effect = [old_vnfd, new_vnfd] + with self.assertNotRaises(EngineException): + self.topic._validate_descriptor_changes( + descriptor_name, "/tmp/", "/tmp:1/" + ) + with self.subTest( + i=3, t="VNFD sw version is not changed and mgmt-cp has changed" + ): + old_vnfd, new_vnfd = self.create_desc_temp(db_vnfd_content) + new_vnfd["mgmt-cp"] = "new-mgmt-cp" + mock_yaml.load.side_effect = [old_vnfd, new_vnfd] + with self.assertRaises(EngineException) as e: + self.topic._validate_descriptor_changes( + descriptor_name, "/tmp/", "/tmp:1/" + ) + self.assertEqual( + e.exception.http_code, + HTTPStatus.UNPROCESSABLE_ENTITY, + "Wrong HTTP status code", + ) + self.assertIn( + norm("there are disallowed changes in the vnf descriptor"), + norm(str(e.exception)), + "Wrong exception text", + ) + def test_validate_mgmt_interface_connection_point_on_valid_descriptor(self): indata = deepcopy(db_vnfd_content) self.topic.validate_mgmt_interface_connection_point(indata) @@ -1247,6 +1305,18 @@ class Test_NsdTopic(TestCase): self.topic = NsdTopic(self.db, self.fs, self.msg, self.auth) self.topic.check_quota = Mock(return_value=None) # skip quota + @contextmanager + def assertNotRaises(self, exception_type): + try: + yield None + except exception_type: + raise self.failureException("{} raised".format(exception_type.__name__)) + + def create_desc_temp(self, template): + old_desc = deepcopy(template) + new_desc = deepcopy(template) + return old_desc, new_desc + @patch("osm_nbi.descriptor_topics.shutil") @patch("osm_nbi.descriptor_topics.os.rename") def test_new_nsd(self, mock_rename, mock_shutil): @@ -1732,6 +1802,46 @@ class Test_NsdTopic(TestCase): self.fs.file_delete.assert_not_called() return + @patch("osm_nbi.descriptor_topics.yaml") + def test_validate_descriptor_changes(self, mock_yaml): + descriptor_name = "test_ns_descriptor" + self.fs.file_open.side_effect = lambda path, mode: open( + "/tmp/" + str(uuid4()), "a+b" + ) + with self.subTest( + i=1, t="NSD has changes in ns-configuration:config-primitive" + ): + old_nsd, new_nsd = self.create_desc_temp(db_nsd_content) + old_nsd.update( + {"ns-configuration": {"config-primitive": [{"name": "add-user"}]}} + ) + new_nsd.update( + {"ns-configuration": {"config-primitive": [{"name": "del-user"}]}} + ) + mock_yaml.load.side_effect = [old_nsd, new_nsd] + with self.assertNotRaises(EngineException): + self.topic._validate_descriptor_changes( + descriptor_name, "/tmp", "/tmp:1" + ) + with self.subTest(i=2, t="NSD name has changed"): + old_nsd, new_nsd = self.create_desc_temp(db_nsd_content) + new_nsd["name"] = "nscharm-ns2" + mock_yaml.load.side_effect = [old_nsd, new_nsd] + with self.assertRaises(EngineException) as e: + self.topic._validate_descriptor_changes( + descriptor_name, "/tmp", "/tmp:1" + ) + self.assertEqual( + e.exception.http_code, + HTTPStatus.UNPROCESSABLE_ENTITY, + "Wrong HTTP status code", + ) + self.assertIn( + norm("there are disallowed changes in the ns descriptor"), + norm(str(e.exception)), + "Wrong exception text", + ) + def test_validate_vld_mgmt_network_with_virtual_link_protocol_data_on_valid_descriptor( self, ): diff --git a/requirements-dev.txt b/requirements-dev.txt index a5862bc..9a0dc90 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -16,7 +16,7 @@ aiokafka==0.7.2 # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master -bitarray==1.8.1 +bitarray==2.3.5 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # pyangbind @@ -53,7 +53,7 @@ pyyaml==5.4.1 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=master -regex==2021.3.17 +regex==2021.11.10 # via # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=master # pyangbind diff --git a/requirements-test.in b/requirements-test.in index 4564164..99848ac 100644 --- a/requirements-test.in +++ b/requirements-test.in @@ -15,6 +15,7 @@ aiohttp>=2.3.10,<=3.6.2 aioresponses asynctest coverage +deepdiff nose2 requests==2.25.1 pyang diff --git a/requirements-test.txt b/requirements-test.txt index b71b0f2..143d7c8 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -32,29 +32,33 @@ chardet==3.0.4 # via # aiohttp # requests -coverage==6.2 +coverage==6.3.2 # via # -r requirements-test.in # nose2 -idna==3.3 +deepdiff==5.7.0 + # via -r requirements-test.in +idna==2.10 # via # requests # yarl -lxml==4.7.1 +lxml==4.8.0 # via pyang multidict==4.7.6 # via # aiohttp # yarl -nose2==0.10.0 +nose2==0.11.0 # via -r requirements-test.in -pyang==2.5.2 +ordered-set==4.0.2 + # via deepdiff +pyang==2.5.3 # via -r requirements-test.in -requests==2.27.1 +requests==2.25.1 # via -r requirements-test.in six==1.16.0 # via nose2 -urllib3==1.26.8 +urllib3==1.26.9 # via requests yarl==1.7.2 # via aiohttp diff --git a/requirements.in b/requirements.in index a961d05..5476674 100644 --- a/requirements.in +++ b/requirements.in @@ -12,6 +12,7 @@ aiohttp>=2.3.10,<=3.6.2 CherryPy>=18.1.2 +deepdiff jsonschema>=3.2.0 python-keystoneclient pyyaml==5.4.1 diff --git a/requirements.txt b/requirements.txt index b8a375f..ad4d82c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,22 +26,24 @@ certifi==2021.10.8 # via requests chardet==3.0.4 # via aiohttp -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests cheroot==8.6.0 # via cherrypy cherrypy==18.6.1 # via -r requirements.in -debtcollector==2.4.0 +debtcollector==2.5.0 # via # oslo.config # oslo.utils # python-keystoneclient +deepdiff==5.7.0 + # via -r requirements.in idna==3.3 # via # requests # yarl -importlib-resources==5.4.0 +importlib-resources==5.6.0 # via # jaraco.text # jsonschema @@ -53,16 +55,18 @@ jaraco.classes==3.2.1 # via jaraco.collections jaraco.collections==3.5.1 # via cherrypy +jaraco.context==4.1.1 + # via jaraco.text jaraco.functools==3.5.0 # via # cheroot # jaraco.text # tempora -jaraco.text==3.6.0 +jaraco.text==3.7.0 # via jaraco.collections jsonschema==4.4.0 # via -r requirements.in -keystoneauth1==4.4.0 +keystoneauth1==4.5.0 # via python-keystoneclient more-itertools==8.12.0 # via @@ -82,26 +86,27 @@ netaddr==0.8.0 # oslo.utils netifaces==0.11.0 # via oslo.utils +ordered-set==4.0.2 + # via deepdiff os-service-types==1.7.0 # via keystoneauth1 -oslo.config==8.7.1 +oslo.config==8.8.0 # via python-keystoneclient oslo.i18n==5.1.0 # via # oslo.config # oslo.utils # python-keystoneclient -oslo.serialization==4.2.0 +oslo.serialization==4.3.0 # via python-keystoneclient -oslo.utils==4.12.0 +oslo.utils==4.12.2 # via # oslo.serialization # python-keystoneclient packaging==21.3 # via oslo.utils -pbr==5.8.0 +pbr==5.8.1 # via - # debtcollector # keystoneauth1 # os-service-types # oslo.i18n @@ -111,7 +116,7 @@ pbr==5.8.0 # stevedore portend==3.1.0 # via cherrypy -pyparsing==3.0.6 +pyparsing==3.0.7 # via # oslo.utils # packaging @@ -119,7 +124,7 @@ pyrsistent==0.18.1 # via jsonschema python-keystoneclient==4.4.0 # via -r requirements.in -pytz==2021.3 +pytz==2022.1 # via # oslo.serialization # oslo.utils @@ -147,19 +152,19 @@ stevedore==3.5.0 # keystoneauth1 # oslo.config # python-keystoneclient -tacacs-plus==2.6 +tacacs_plus==2.6 # via -r requirements.in -tempora==5.0.0 +tempora==5.0.1 # via portend -urllib3==1.26.8 +urllib3==1.26.9 # via requests -wrapt==1.13.3 +wrapt==1.14.0 # via debtcollector yarl==1.7.2 # via aiohttp zc.lockfile==2.0 # via cherrypy -zipp==3.7.0 +zipp==3.8.0 # via importlib-resources # The following packages are considered to be unsafe in a requirements file: