From: aticig Date: Thu, 19 May 2022 10:03:17 +0000 (+0300) Subject: Fix Bug 2012 use existing volumes as instantiation parameters X-Git-Tag: v12.0.0rc1~5 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=commitdiff_plain;h=cf14bb1f6b681cc5a5d1771e2034c306ea8b60f5 Fix Bug 2012 use existing volumes as instantiation parameters RO processes the existing vim-volume-id which is provided through the instantation parameters, then they are consumed while creating the new VDUs. Change-Id: Ic3bd71ac7679ab0ccbe9ee0847e4a8d3059c7c58 Signed-off-by: aticig --- diff --git a/NG-RO/osm_ng_ro/ns.py b/NG-RO/osm_ng_ro/ns.py index a823dbe3..096d4dfd 100644 --- a/NG-RO/osm_ng_ro/ns.py +++ b/NG-RO/osm_ng_ro/ns.py @@ -17,6 +17,7 @@ ## from http import HTTPStatus +from itertools import product import logging from random import choice as random_choice from threading import Lock @@ -776,18 +777,35 @@ class Ns(object): Returns: Dict[str, Any]: [description] """ + db = kwargs.get("db") + target_vdur = {} + flavor_data = { "disk": int(target_flavor["storage-gb"]), "ram": int(target_flavor["memory-mb"]), "vcpus": int(target_flavor["vcpu-count"]), } - target_vdur = {} for vnf in indata.get("vnf", []): for vdur in vnf.get("vdur", []): - if vdur.get("ns-flavor-id") == target_flavor["id"]: + if vdur.get("ns-flavor-id") == target_flavor.get("id"): target_vdur = vdur + if db and isinstance(indata.get("vnf"), list): + vnfd_id = indata.get("vnf")[0].get("vnfd-id") + vnfd = db.get_one("vnfds", {"_id": vnfd_id}) + # check if there is persistent root disk + for vdu in vnfd.get("vdu", ()): + if vdu["name"] == target_vdur.get("vdu-name"): + for vsd in vnfd.get("virtual-storage-desc", ()): + if vsd.get("id") == vdu.get("virtual-storage-desc", [[]])[0]: + root_disk = vsd + if ( + root_disk.get("type-of-storage") + == "persistent-storage:persistent-storage" + ): + flavor_data["disk"] = 0 + for storage in target_vdur.get("virtual-storages", []): if ( storage.get("type-of-storage") @@ -922,6 +940,115 @@ class Ns(object): return extra_dict + @staticmethod + def find_persistent_root_volumes( + vnfd: dict, + target_vdu: str, + vdu_instantiation_volumes_list: list, + disk_list: list, + ) -> (list, dict): + """Find the persistent root volumes and add them to the disk_list + by parsing the instantiation parameters + + Args: + vnfd: VNFD + target_vdu: processed VDU + vdu_instantiation_volumes_list: instantiation parameters for the each VDU as a list + disk_list: to be filled up + + Returns: + disk_list: filled VDU list which is used for VDU creation + + """ + persistent_root_disk = {} + + for vdu, vsd in product( + vnfd.get("vdu", ()), vnfd.get("virtual-storage-desc", ()) + ): + if ( + vdu["name"] == target_vdu["vdu-name"] + and vsd.get("id") == vdu.get("virtual-storage-desc", [[]])[0] + ): + root_disk = vsd + if ( + root_disk.get("type-of-storage") + == "persistent-storage:persistent-storage" + ): + for vdu_volume in vdu_instantiation_volumes_list: + + if ( + vdu_volume["vim-volume-id"] + and root_disk["id"] == vdu_volume["name"] + ): + + persistent_root_disk[vsd["id"]] = { + "vim_volume_id": vdu_volume["vim-volume-id"], + "image_id": vdu.get("sw-image-desc"), + } + + disk_list.append(persistent_root_disk[vsd["id"]]) + + # There can be only one root disk, when we find it, it will return the result + return disk_list, persistent_root_disk + + else: + + if root_disk.get("size-of-storage"): + persistent_root_disk[vsd["id"]] = { + "image_id": vdu.get("sw-image-desc"), + "size": root_disk.get("size-of-storage"), + } + + disk_list.append(persistent_root_disk[vsd["id"]]) + return disk_list, persistent_root_disk + + return disk_list, persistent_root_disk + + @staticmethod + def find_persistent_volumes( + persistent_root_disk: dict, + target_vdu: str, + vdu_instantiation_volumes_list: list, + disk_list: list, + ) -> list: + """Find the ordinary persistent volumes and add them to the disk_list + by parsing the instantiation parameters + + Args: + persistent_root_disk: persistent root disk dictionary + target_vdu: processed VDU + vdu_instantiation_volumes_list: instantiation parameters for the each VDU as a list + disk_list: to be filled up + + Returns: + disk_list: filled VDU list which is used for VDU creation + + """ + # Find the ordinary volumes which are not added to the persistent_root_disk + persistent_disk = {} + for disk in target_vdu.get("virtual-storages", {}): + if ( + disk.get("type-of-storage") == "persistent-storage:persistent-storage" + and disk["id"] not in persistent_root_disk.keys() + ): + for vdu_volume in vdu_instantiation_volumes_list: + + if vdu_volume["vim-volume-id"] and disk["id"] == vdu_volume["name"]: + + persistent_disk[disk["id"]] = { + "vim_volume_id": vdu_volume["vim-volume-id"], + } + disk_list.append(persistent_disk[disk["id"]]) + + else: + if disk["id"] not in persistent_disk.keys(): + persistent_disk[disk["id"]] = { + "size": disk.get("size-of-storage"), + } + disk_list.append(persistent_disk[disk["id"]]) + + return disk_list + @staticmethod def _process_vdu_params( target_vdu: Dict[str, Any], @@ -1103,33 +1230,59 @@ class Ns(object): cloud_config["key-pairs"] = ssh_keys persistent_root_disk = {} + vdu_instantiation_volumes_list = [] disk_list = [] vnfd_id = vnfr["vnfd-id"] vnfd = db.get_one("vnfds", {"_id": vnfd_id}) - for vdu in vnfd.get("vdu", ()): - if vdu["name"] == target_vdu["vdu-name"]: - for vsd in vnfd.get("virtual-storage-desc", ()): - if vsd.get("id") == vdu.get("virtual-storage-desc", [[]])[0]: - root_disk = vsd - if root_disk.get( - "type-of-storage" - ) == "persistent-storage:persistent-storage" and root_disk.get( - "size-of-storage" - ): - persistent_root_disk[vsd["id"]] = { - "image_id": vdu.get("sw-image-desc"), - "size": root_disk["size-of-storage"], - } - disk_list.append(persistent_root_disk[vsd["id"]]) - if target_vdu.get("virtual-storages"): - for disk in target_vdu["virtual-storages"]: - if ( - disk.get("type-of-storage") - == "persistent-storage:persistent-storage" - and disk["id"] not in persistent_root_disk.keys() - ): - disk_list.append({"size": disk["size-of-storage"]}) + if target_vdu.get("additionalParams"): + vdu_instantiation_volumes_list = ( + target_vdu.get("additionalParams").get("OSM").get("vdu_volumes") + ) + + if vdu_instantiation_volumes_list: + + # Find the root volumes and add to the disk_list + (disk_list, persistent_root_disk,) = Ns.find_persistent_root_volumes( + vnfd, target_vdu, vdu_instantiation_volumes_list, disk_list + ) + + # Find the ordinary volumes which are not added to the persistent_root_disk + # and put them to the disk list + disk_list = Ns.find_persistent_volumes( + persistent_root_disk, + target_vdu, + vdu_instantiation_volumes_list, + disk_list, + ) + + else: + + # vdu_instantiation_volumes_list is empty + for vdu in vnfd.get("vdu", ()): + if vdu["name"] == target_vdu["vdu-name"]: + for vsd in vnfd.get("virtual-storage-desc", ()): + if vsd.get("id") == vdu.get("virtual-storage-desc", [[]])[0]: + root_disk = vsd + if root_disk.get( + "type-of-storage" + ) == "persistent-storage:persistent-storage" and root_disk.get( + "size-of-storage" + ): + persistent_root_disk[vsd["id"]] = { + "image_id": vdu.get("sw-image-desc"), + "size": root_disk["size-of-storage"], + } + disk_list.append(persistent_root_disk[vsd["id"]]) + + if target_vdu.get("virtual-storages"): + for disk in target_vdu["virtual-storages"]: + if ( + disk.get("type-of-storage") + == "persistent-storage:persistent-storage" + and disk["id"] not in persistent_root_disk.keys() + ): + disk_list.append({"size": disk["size-of-storage"]}) affinity_group_list = [] @@ -1545,6 +1698,16 @@ class Ns(object): self.logger.warning( "ns.calculate_diff_items target_item={}".format(target_item) ) + if process_params == Ns._process_flavor_params: + kwargs.update( + { + "db": self.db, + } + ) + self.logger.warning( + "calculate_diff_items for flavor kwargs={}".format(kwargs) + ) + if process_params == Ns._process_vdu_params: self.logger.warning( "calculate_diff_items self.fs={}".format(self.fs) diff --git a/NG-RO/osm_ng_ro/ns_thread.py b/NG-RO/osm_ng_ro/ns_thread.py index 8bb8d4bc..8cd7d2cb 100644 --- a/NG-RO/osm_ng_ro/ns_thread.py +++ b/NG-RO/osm_ng_ro/ns_thread.py @@ -415,6 +415,7 @@ class VimInteractionVdu(VimInteractionBase): return "BUILD", ro_vim_item_update except (vimconn.VimConnException, NsWorkerException) as e: + self.logger.debug(traceback.format_exc()) self.logger.error( "task={} {} new-vm: {}".format(task_id, ro_task["target_id"], e) ) diff --git a/NG-RO/osm_ng_ro/tests/test_ns.py b/NG-RO/osm_ng_ro/tests/test_ns.py index c0ba4263..3dd699bc 100644 --- a/NG-RO/osm_ng_ro/tests/test_ns.py +++ b/NG-RO/osm_ng_ro/tests/test_ns.py @@ -1408,8 +1408,15 @@ class TestNs(unittest.TestCase): self, epa_params, ): + target_flavor = {} - indata = {} + indata = { + "vnf": [ + { + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + }, + ], + } vim_info = {} target_record_id = "" @@ -1428,6 +1435,7 @@ class TestNs(unittest.TestCase): self, epa_params, ): + target_flavor = { "no-target-flavor": "here", } @@ -1450,6 +1458,7 @@ class TestNs(unittest.TestCase): self, epa_params, ): + expected_result = { "find_params": { "flavor_data": { @@ -1494,6 +1503,7 @@ class TestNs(unittest.TestCase): self, epa_params, ): + expected_result = { "find_params": { "flavor_data": { @@ -1540,6 +1550,52 @@ class TestNs(unittest.TestCase): self, epa_params, ): + db = MagicMock(name="database mock") + kwargs = { + "db": db, + } + + db.get_one.return_value = { + "_id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + "df": [ + { + "id": "default-df", + "vdu-profile": [ + {"id": "without_volumes-VM", "min-number-of-instances": 1} + ], + } + ], + "id": "without_volumes-vnf", + "product-name": "without_volumes-vnf", + "vdu": [ + { + "id": "without_volumes-VM", + "name": "without_volumes-VM", + "sw-image-desc": "ubuntu20.04", + "alternative-sw-image-desc": [ + "ubuntu20.04-aws", + "ubuntu20.04-azure", + ], + "virtual-storage-desc": ["root-volume", "ephemeral-volume"], + } + ], + "version": "1.0", + "virtual-storage-desc": [ + {"id": "root-volume", "size-of-storage": "10"}, + { + "id": "ephemeral-volume", + "type-of-storage": "etsi-nfv-descriptors:ephemeral-storage", + "size-of-storage": "1", + }, + ], + "_admin": { + "storage": { + "fs": "mongo", + "path": "/app/storage/", + }, + "type": "vnfd", + }, + } expected_result = { "find_params": { "flavor_data": { @@ -1580,6 +1636,7 @@ class TestNs(unittest.TestCase): ], }, ], + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", }, ], } @@ -1593,6 +1650,7 @@ class TestNs(unittest.TestCase): indata=indata, vim_info=vim_info, target_record_id=target_record_id, + **kwargs, ) self.assertTrue(epa_params.called) @@ -1603,6 +1661,7 @@ class TestNs(unittest.TestCase): self, epa_params, ): + expected_result = { "find_params": { "flavor_data": { @@ -1643,6 +1702,118 @@ class TestNs(unittest.TestCase): ], }, ], + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + }, + ], + } + vim_info = {} + target_record_id = "" + + epa_params.return_value = {} + + result = Ns._process_flavor_params( + target_flavor=target_flavor, + indata=indata, + vim_info=vim_info, + target_record_id=target_record_id, + ) + + self.assertTrue(epa_params.called) + self.assertDictEqual(result, expected_result) + + @patch("osm_ng_ro.ns.Ns._process_epa_params") + def test__process_flavor_params_with_persistent_root_disk( + self, + epa_params, + ): + db = MagicMock(name="database mock") + + kwargs = { + "db": db, + } + + db.get_one.return_value = { + "_id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + "df": [ + { + "id": "default-df", + "vdu-profile": [ + {"id": "several_volumes-VM", "min-number-of-instances": 1} + ], + } + ], + "id": "several_volumes-vnf", + "product-name": "several_volumes-vnf", + "vdu": [ + { + "id": "several_volumes-VM", + "name": "several_volumes-VM", + "sw-image-desc": "ubuntu20.04", + "alternative-sw-image-desc": [ + "ubuntu20.04-aws", + "ubuntu20.04-azure", + ], + "virtual-storage-desc": [ + "persistent-root-volume", + ], + } + ], + "version": "1.0", + "virtual-storage-desc": [ + { + "id": "persistent-root-volume", + "type-of-storage": "persistent-storage:persistent-storage", + "size-of-storage": "10", + }, + ], + "_admin": { + "storage": { + "fs": "mongo", + "path": "/app/storage/", + }, + "type": "vnfd", + }, + } + expected_result = { + "find_params": { + "flavor_data": { + "disk": 0, + "ram": 1024, + "vcpus": 2, + }, + }, + "params": { + "flavor_data": { + "disk": 0, + "name": "test", + "ram": 1024, + "vcpus": 2, + }, + }, + } + target_flavor = { + "id": "test_id", + "name": "test", + "storage-gb": "10", + "memory-mb": "1024", + "vcpu-count": "2", + } + indata = { + "vnf": [ + { + "vdur": [ + { + "vdu-name": "several_volumes-VM", + "ns-flavor-id": "test_id", + "virtual-storages": [ + { + "type-of-storage": "persistent-storage:persistent-storage", + "size-of-storage": "10", + }, + ], + }, + ], + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", }, ], } @@ -1656,6 +1827,7 @@ class TestNs(unittest.TestCase): indata=indata, vim_info=vim_info, target_record_id=target_record_id, + **kwargs, ) self.assertTrue(epa_params.called) @@ -1666,6 +1838,7 @@ class TestNs(unittest.TestCase): self, epa_params, ): + expected_result = { "find_params": { "flavor_data": { @@ -1696,7 +1869,18 @@ class TestNs(unittest.TestCase): "memory-mb": "1024", "vcpu-count": "2", } - indata = {} + indata = { + "vnf": [ + { + "vdur": [ + { + "ns-flavor-id": "test_id", + }, + ], + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + }, + ], + } vim_info = {} target_record_id = "" @@ -1719,6 +1903,54 @@ class TestNs(unittest.TestCase): self, epa_params, ): + db = MagicMock(name="database mock") + + kwargs = { + "db": db, + } + + db.get_one.return_value = { + "_id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", + "df": [ + { + "id": "default-df", + "vdu-profile": [ + {"id": "without_volumes-VM", "min-number-of-instances": 1} + ], + } + ], + "id": "without_volumes-vnf", + "product-name": "without_volumes-vnf", + "vdu": [ + { + "id": "without_volumes-VM", + "name": "without_volumes-VM", + "sw-image-desc": "ubuntu20.04", + "alternative-sw-image-desc": [ + "ubuntu20.04-aws", + "ubuntu20.04-azure", + ], + "virtual-storage-desc": ["root-volume", "ephemeral-volume"], + } + ], + "version": "1.0", + "virtual-storage-desc": [ + {"id": "root-volume", "size-of-storage": "10"}, + { + "id": "ephemeral-volume", + "type-of-storage": "etsi-nfv-descriptors:ephemeral-storage", + "size-of-storage": "1", + }, + ], + "_admin": { + "storage": { + "fs": "mongo", + "path": "/app/storage/", + }, + "type": "vnfd", + }, + } + expected_result = { "find_params": { "flavor_data": { @@ -1771,6 +2003,7 @@ class TestNs(unittest.TestCase): ], }, ], + "vnfd-id": "ad6356e3-698c-43bf-9901-3aae9e9b9d18", }, ], } @@ -1786,6 +2019,7 @@ class TestNs(unittest.TestCase): indata=indata, vim_info=vim_info, target_record_id=target_record_id, + **kwargs, ) self.assertTrue(epa_params.called) diff --git a/RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py b/RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py index 0a390412..d9edb323 100644 --- a/RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py +++ b/RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py @@ -1369,7 +1369,6 @@ class vimconnector(vimconn.VimConnector): "Invalid mempage-size %s. Will be ignored", extended.get("mempage-size"), ) - # create flavor new_flavor = self.nova.flavors.create( name=name, @@ -1898,32 +1897,55 @@ class vimconnector(vimconn.VimConnector): if disk_list: block_device_mapping = {} for disk in disk_list: - if disk.get("vim_id"): - block_device_mapping["_vd" + chr(base_disk_index)] = disk[ - "vim_id" - ] - existing_vim_volumes.append({"id": disk["vim_id"]}) - else: - if "image_id" in disk: - base_disk_index = ord("a") + if "image_id" in disk: + # persistent root volume + base_disk_index = ord("a") + image_id = "" + if disk.get("vim_volume_id"): + + # use existing persistent root volume + block_device_mapping["vd" + chr(base_disk_index)] = disk[ + "vim_volume_id" + ] + existing_vim_volumes.append({"id": disk["vim_volume_id"]}) + + else: + # create persistent root volume volume = self.cinder.volumes.create( size=disk["size"], - name=name + "_vd" + chr(base_disk_index), + name=name + "vd" + chr(base_disk_index), imageRef=disk["image_id"], # Make sure volume is in the same AZ as the VM to be attached to availability_zone=vm_av_zone, ) boot_volume_id = volume.id + created_items["volume:" + str(volume.id)] = True + block_device_mapping[ + "vd" + chr(base_disk_index) + ] = volume.id + else: + # non-root persistent volume + if disk.get("vim_volume_id"): + + # use existing persistent volume + block_device_mapping["vd" + chr(base_disk_index)] = disk[ + "vim_volume_id" + ] + existing_vim_volumes.append({"id": disk["vim_volume_id"]}) + else: + + # create persistent volume volume = self.cinder.volumes.create( size=disk["size"], - name=name + "_vd" + chr(base_disk_index), + name=name + "vd" + chr(base_disk_index), # Make sure volume is in the same AZ as the VM to be attached to availability_zone=vm_av_zone, ) - - created_items["volume:" + str(volume.id)] = True - block_device_mapping["_vd" + chr(base_disk_index)] = volume.id + created_items["volume:" + str(volume.id)] = True + block_device_mapping[ + "vd" + chr(base_disk_index) + ] = volume.id base_disk_index += 1 @@ -1988,9 +2010,9 @@ class vimconnector(vimconn.VimConnector): ) ) server = self.nova.servers.create( - name, - image_id, - flavor_id, + name=name, + image=image_id, + flavor=flavor_id, nics=net_list_vim, security_groups=self.config.get("security_groups"), # TODO remove security_groups in future versions. Already at neutron port diff --git a/releasenotes/notes/fixing_bug_2012-c8553db48c29803d.yaml b/releasenotes/notes/fixing_bug_2012-c8553db48c29803d.yaml new file mode 100644 index 00000000..74573cc6 --- /dev/null +++ b/releasenotes/notes/fixing_bug_2012-c8553db48c29803d.yaml @@ -0,0 +1,24 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### +--- +fixes: + - | + Fix Bug 2012 use existing volumes as instantiation parameters targeting the Openstack VIM + RO processes the existing vim-volume-id which is provided through the + instantiation parameters, then they are consumed while creating the new VDUs. + +