From cc10343b861269fe9f336fadf54b8e9fba0a74a7 Mon Sep 17 00:00:00 2001 From: tierno Date: Fri, 19 Oct 2018 14:10:35 +0200 Subject: [PATCH] feature 1417 PDU: Instantiation parameters. Instantiation checking, look for proper PDU. Fix vnfr generation with vdu.count>1 Change-Id: I1ba1da03ac6a5fe15ecf5ceaab49049e5dca1106 Signed-off-by: tierno --- Makefile | 2 +- osm_nbi/instance_topics.py | 130 +++++++++++++++++++++++++++++++++++-- osm_nbi/nbi.py | 6 +- osm_nbi/tests/test.py | 100 ++++++++++++++++++++++++++-- osm_nbi/validation.py | 1 + 5 files changed, 224 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 5a2e3ce..475cbb0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ clean: - rm -rf dist deb_dist .build osm_nbi-*.tar.gz osm_nbi.egg-info eggs + rm -rf dist deb_dist .build osm_nbi-*.tar.gz osm_nbi.egg-info .eggs package: python3 setup.py --command-packages=stdeb.command sdist_dsc diff --git a/osm_nbi/instance_topics.py b/osm_nbi/instance_topics.py index e98f90f..29a391c 100644 --- a/osm_nbi/instance_topics.py +++ b/osm_nbi/instance_topics.py @@ -4,7 +4,7 @@ from uuid import uuid4 from http import HTTPStatus from time import time -from copy import copy +from copy import copy, deepcopy from validation import validate_input, ValidationError, ns_instantiate, ns_action, ns_scale from base_topic import BaseTopic, EngineException, get_iterable from descriptor_topics import DescriptorTopic @@ -170,9 +170,7 @@ class NsrTopic(BaseTopic): } vnfr_descriptor["connection-point"].append(vnf_cp) for vdu in vnfd["vdu"]: - vdur_id = str(uuid4()) vdur = { - "id": vdur_id, "vdu-id-ref": vdu["id"], # TODO "name": "" Name of the VDU in the VIM "ip-address": None, # mgmt-interface filled by LCM @@ -180,6 +178,8 @@ class NsrTopic(BaseTopic): "internal-connection-point": [], "interfaces": [], } + if vdu.get("pdu-type"): + vdur["pdu-type"] = vdu["pdu-type"] # TODO volumes: name, volume-id for icp in vdu.get("internal-connection-point", ()): vdu_icp = { @@ -196,8 +196,20 @@ class NsrTopic(BaseTopic): # "ip-address", "mac-address" # filled by LCM # vim-id # TODO it would be nice having a vim port id } + if iface.get("mgmt-interface"): + vdu_iface["mgmt-interface"] = True + vdur["interfaces"].append(vdu_iface) - vnfr_descriptor["vdur"].append(vdur) + count = vdu.get("count", 1) + if count is None: + count = 1 + count = int(count) # TODO remove when descriptor serialized with payngbind + for index in range(0, count): + if index: + vdur = deepcopy(vdur) + vdur["_id"] = str(uuid4()) + vdur["count-index"] = index + vnfr_descriptor["vdur"].append(vdur) step = "creating vnfr vnfd-id='{}' constituent-vnfd='{}' at database".format( member_vnf["vnfd-id-ref"], member_vnf["member-vnf-index"]) @@ -340,10 +352,11 @@ class NsLcmOpTopic(BaseTopic): if vim_account in vim_accounts: return try: - # TODO add _get_project_filter - self.db.get_one("vim_accounts", {"_id": vim_account}) + db_filter = self._get_project_filter(session, write=False, show_all=True) + db_filter["_id"] = vim_account + self.db.get_one("vim_accounts", db_filter) except Exception: - raise EngineException("Invalid vimAccountId='{}' not present".format(vim_account)) + raise EngineException("Invalid vimAccountId='{}' not present for the project".format(vim_account)) vim_accounts.append(vim_account) if operation == "action": @@ -400,6 +413,107 @@ class NsLcmOpTopic(BaseTopic): raise EngineException("Invalid parameter vld:name='{}' is not present at nsd:vld".format( in_vld["name"])) + def _look_for_pdu(self, session, rollback, vnfr, vim_account): + """ + Look for a free PDU in the catalog matching vdur type and interfaces. Fills vdur with ip_address information + :param vdur: vnfr:vdur descriptor. It is modified with pdu interface info if pdu is found + :param member_vnf_index: used just for logging. Target vnfd of nsd + :param vim_account: + :return: vder_update: dictionary to update vnfr:vdur with pdu info. In addition it modified choosen pdu to set + at status IN_USE + """ + vnfr_update = {} + rollback_vnfr = {} + for vdur_index, vdur in enumerate(get_iterable(vnfr.get("vdur"))): + if not vdur.get("pdu-type"): + continue + pdu_type = vdur.get("pdu-type") + pdu_filter = self._get_project_filter(session, write=True, show_all=True) + pdu_filter["vim.vim_accounts"] = vim_account + pdu_filter["type"] = pdu_type + pdu_filter["_admin.operationalState"] = "ENABLED" + pdu_filter["_admin.usageSate"] = "NOT_IN_USE", + # TODO feature 1417: "shared": True, + + available_pdus = self.db.get_list("pdus", pdu_filter) + for pdu in available_pdus: + # step 1 check if this pdu contains needed interfaces: + match_interfaces = True + for vdur_interface in vdur["interfaces"]: + for pdu_interface in pdu["interfaces"]: + if pdu_interface["name"] == vdur_interface["name"]: + # TODO feature 1417: match per mgmt type + break + else: # no interface found for name + match_interfaces = False + break + if match_interfaces: + break + else: + raise EngineException( + "No PDU of type={} found for member_vnf_index={} at vim_account={} matching interface " + "names".format(vdur["vdu-id-ref"], vnfr["member-vnf-index-ref"], pdu_type)) + + # step 2. Update pdu + rollback_pdu = { + "_admin.usageState": pdu["_admin"]["usageState"], + "_admin.usage.vnfr_id": None, + "_admin.usage.nsr_id": None, + "_admin.usage.vdur": None, + } + self.db.set_one("pdus", {"_id": pdu["_id"]}, + {"_admin.usageSate": "IN_USE", + "_admin.usage.vnfr_id": vnfr["_id"], + "_admin.usage.nsr_id": vnfr["nsr-id-ref"], + "_admin.usage.vdur": vdur["vdu-id-ref"]} + ) + rollback.append({"topic": "pdus", "_id": pdu["_id"], "operation": "set", "content": rollback_pdu}) + + # step 3. Fill vnfr info by filling vdur + vdu_text = "vdur.{}".format(vdur_index) + rollback_vnfr[vdu_text + ".pdu-id"] = None + vnfr_update[vdu_text + ".pdu-id"] = pdu["_id"] + for iface_index, vdur_interface in enumerate(vdur["interfaces"]): + for pdu_interface in pdu["interfaces"]: + if pdu_interface["name"] == vdur_interface["name"]: + iface_text = vdu_text + ".interfaces.{}".format(iface_index) + for k, v in pdu_interface.items(): + vnfr_update[iface_text + ".{}".format(k)] = v + rollback_vnfr[iface_text + ".{}".format(k)] = vdur_interface.get(v) + break + + if vnfr_update: + self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, vnfr_update) + rollback.append({"topic": "vnfrs", "_id": vnfr["_id"], "operation": "set", "content": rollback_vnfr}) + return + + def _update_vnfrs(self, session, rollback, nsr, indata): + vnfrs = None + # get vnfr + nsr_id = nsr["_id"] + vnfrs = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}) + + for vnfr in vnfrs: + vnfr_update = {} + vnfr_update_rollback = {} + member_vnf_index = vnfr["member-vnf-index-ref"] + # update vim-account-id + + vim_account = indata["vimAccountId"] + # check instantiate parameters + for vnf_inst_params in get_iterable(indata.get("vnf")): + if vnf_inst_params["member-vnf-index"] != member_vnf_index: + continue + if vnf_inst_params.get("vimAccountId"): + vim_account = vnf_inst_params.get("vimAccountId") + + vnfr_update["vim-account-id"] = vim_account + vnfr_update_rollback["vim-account-id"] = vnfr.get("vim-account-id") + + # get pdu + self._look_for_pdu(session, rollback, vnfr, vim_account) + # TODO change instantiation parameters to set network + def _create_nslcmop(self, session, nsInstanceId, operation, params): now = time() _id = str(uuid4()) @@ -460,6 +574,8 @@ class NsLcmOpTopic(BaseTopic): raise EngineException("ns_instance '{}' cannot be '{}' because it is already instantiated".format( nsInstanceId, operation), HTTPStatus.CONFLICT) self._check_ns_operation(session, nsr, operation, indata) + if operation == "instantiate": + self._update_vnfrs(session, rollback, nsr, indata) nslcmop_desc = self._create_nslcmop(session, nsInstanceId, operation, indata) self.format_on_new(nslcmop_desc, session["project_id"], make_public=make_public) _id = self.db.create("nslcmops", nslcmop_desc) diff --git a/osm_nbi/nbi.py b/osm_nbi/nbi.py index 0414dba..1d53396 100644 --- a/osm_nbi/nbi.py +++ b/osm_nbi/nbi.py @@ -750,7 +750,11 @@ class Server(object): rollback.reverse() for rollback_item in rollback: try: - self.engine.del_item(**rollback_item, session=session, force=True) + if rollback_item.get("operation") == "set": + self.engine.db.set_one(rollback_item["topic"], {"_id": rollback_item["_id"]}, + rollback_item["content"], fail_on_empty=False) + else: + self.engine.del_item(**rollback_item, session=session, force=True) except Exception as e2: rollback_error_text = "Rollback Exception {}: {}".format(rollback_item, e2) cherrypy.log(rollback_error_text) diff --git a/osm_nbi/tests/test.py b/osm_nbi/tests/test.py index 1869f54..0c08c22 100755 --- a/osm_nbi/tests/test.py +++ b/osm_nbi/tests/test.py @@ -583,6 +583,7 @@ class TestDeploy: self.ns_test = None self.ns_id = None self.vnfds_test = [] + self.vnfds_id = [] self.descriptor_url = "https://osm-download.etsi.org/ftp/osm-3.0-three/2nd-hackfest/packages/" self.vnfd_filenames = ("cirros_vnf.tar.gz",) self.nsd_filename = "cirros_2vnf_ns.tar.gz" @@ -619,17 +620,21 @@ class TestDeploy: headers = headers_zip_yaml if self.step % 2 == 0: # vnfd CREATE AND UPLOAD in one step: - engine.test("DEPLOY{}".format(self.step), "Onboard VNFD in one step", "POST", + test_name = "DEPLOY{}".format(self.step) + engine.test(test_name, "Onboard VNFD in one step", "POST", "/vnfpkgm/v1/vnf_packages_content", headers, "@b" + vnfd_filename_path, 201, {"Location": "/vnfpkgm/v1/vnf_packages_content/", "Content-Type": "application/yaml"}, yaml) - self.vnfds_test.append("DEPLOY" + str(self.step)) + self.vnfds_test.append(test_name) + self.vnfds_id.append(engine.test_ids[test_name]) self.step += 1 else: # vnfd CREATE AND UPLOAD ZIP - engine.test("DEPLOY{}".format(self.step), "Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages", + test_name = "DEPLOY{}".format(self.step) + engine.test(test_name, "Onboard VNFD step 1", "POST", "/vnfpkgm/v1/vnf_packages", headers_json, None, 201, {"Location": "/vnfpkgm/v1/vnf_packages/", "Content-Type": "application/json"}, "json") - self.vnfds_test.append("DEPLOY" + str(self.step)) + self.vnfds_test.append(test_name) + self.vnfds_id.append(engine.test_ids[test_name]) self.step += 1 # location = r.headers["Location"] # vnfd_id = location[location.rfind("/")+1:] @@ -722,7 +727,7 @@ class TestDeploy: raise TestException("NS instantiate is not done after {} seconds".format(timeout_deploy)) self.step += 1 - def _wait_nslcmop_ready(self, engine, nslcmop_test, timeout_deploy): + def _wait_nslcmop_ready(self, engine, nslcmop_test, timeout_deploy, expected_fail=False): wait = timeout while wait >= 0: r = engine.test("DEPLOY{}".format(self.step), "Wait to ns lcm operation complete", "GET", @@ -730,9 +735,14 @@ class TestDeploy: 200, r_header_json, "json") nslcmop = r.json() if "COMPLETED" in nslcmop["operationState"]: + if expected_fail: + raise TestException("NS terminate has success, expecting failing: {}".format( + nslcmop["detailed-status"])) break elif "FAILED" in nslcmop["operationState"]: - raise TestException("NS terminate has failed: {}".format(nslcmop["detailed-status"])) + if not expected_fail: + raise TestException("NS terminate has failed: {}".format(nslcmop["detailed-status"])) + break wait -= 5 sleep(5) else: @@ -907,6 +917,82 @@ class TestDeployHackfestCirros(TestDeploy): self.pss = {'1': "cubswin:)", '2': "cubswin:)"} +class TestDeployHackfest1(TestDeploy): + description = "Load and deploy Hackfest_1_vnfd example" + + def __init__(self): + super().__init__() + self.vnfd_filenames = ("hackfest_1_vnfd.tar.gz",) + self.nsd_filename = "hackfest_1_nsd.tar.gz" + # self.cmds = {'1': ['ls -lrt', ], '2': ['ls -lrt', ]} + # self.uss = {'1': "cirros", '2': "cirros"} + # self.pss = {'1': "cubswin:)", '2': "cubswin:)"} + + +class TestDeployHackfestCirrosScaling(TestDeploy): + description = "Load and deploy Hackfest cirros_2vnf_ns example with scaling modifications" + + def __init__(self): + super().__init__() + self.vnfd_filenames = ("cirros_vnf.tar.gz",) + self.nsd_filename = "cirros_2vnf_ns.tar.gz" + + def create_descriptors(self, engine): + super().create_descriptors(engine) + # Modify VNFD to add scaling and count=2 + payload = """ + vdu: + "$id: 'cirros_vnfd-VM'": + count: 2 + scaling-group-descriptor: + - name: "scale_cirros" + max-instance-count: 2 + vdu: + - vdu-id-ref: cirros_vnfd-VM + count: 2 + """ + engine.test("DEPLOY{}".format(self.step), "Edit VNFD ", "PATCH", + "/vnfpkgm/v1/vnf_packages/{}".format(self.vnfds_id[0]), + headers_yaml, payload, 204, None, None) + self.step += 1 + + def aditional_operations(self, engine, test_osm, manual_check): + if not test_osm: + return + # 2 perform scale out twice + payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_OUT, scaleByStepData: ' \ + '{scaling-group-descriptor: scale_cirros, member-vnf-index: "1"}}}' + for i in range(0, 2): + engine.test("DEPLOY{}".format(self.step), "Execute scale action over NS", "POST", + "/nslcm/v1/ns_instances/<{}>/scale".format(self.ns_test), headers_yaml, payload, + 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + nslcmop2_scale_out = "DEPLOY{}".format(self.step) + self._wait_nslcmop_ready(engine, nslcmop2_scale_out, timeout_deploy) + if manual_check: + input('NS scale out done. Check that two more vdus are there') + # TODO check automatic + + # 2 perform scale in + payload = '{scaleType: SCALE_VNF, scaleVnfData: {scaleVnfType: SCALE_IN, scaleByStepData: ' \ + '{scaling-group-descriptor: scale_cirros, member-vnf-index: "1"}}}' + for i in range(0, 2): + engine.test("DEPLOY{}".format(self.step), "Execute scale IN action over NS", "POST", + "/nslcm/v1/ns_instances/<{}>/scale".format(self.ns_test), headers_yaml, payload, + 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + nslcmop2_scale_in = "DEPLOY{}".format(self.step) + self._wait_nslcmop_ready(engine, nslcmop2_scale_in, timeout_deploy) + if manual_check: + input('NS scale in done. Check that two less vdus are there') + # TODO check automatic + + # perform scale in that must fail as reached limit + engine.test("DEPLOY{}".format(self.step), "Execute scale IN out of limit action over NS", "POST", + "/nslcm/v1/ns_instances/<{}>/scale".format(self.ns_test), headers_yaml, payload, + 201, {"Location": "nslcm/v1/ns_lcm_op_occs/", "Content-Type": "application/yaml"}, "yaml") + nslcmop2_scale_in = "DEPLOY{}".format(self.step) + self._wait_nslcmop_ready(engine, nslcmop2_scale_in, timeout_deploy, expected_fail=True) + + class TestDeployIpMac(TestDeploy): description = "Load and deploy descriptor examples setting mac, ip address at descriptor and instantiate params" @@ -1275,10 +1361,12 @@ if __name__ == "__main__": "VIM-SDN": TestVIMSDN, "Deploy-Custom": TestDeploy, "Deploy-Hackfest-Cirros": TestDeployHackfestCirros, + "Deploy-Hackfest-Cirros-Scaling": TestDeployHackfestCirrosScaling, "Deploy-Hackfest-3Charmed": TestDeployHackfest3Charmed, "Deploy-Hackfest-4": TestDeployHackfest4, "Deploy-CirrosMacIp": TestDeployIpMac, "TestDescriptors": TestDescriptors, + "TestDeployHackfest1": TestDeployHackfest1, # "Deploy-MultiVIM": TestDeployMultiVIM, } test_to_do = [] diff --git a/osm_nbi/validation.py b/osm_nbi/validation.py index 6999435..861bc74 100644 --- a/osm_nbi/validation.py +++ b/osm_nbi/validation.py @@ -196,6 +196,7 @@ ns_instantiate = { "vimAccountId": id_schema, "ssh_keys": {"type": "array", "items": {"type": "string"}}, "nsr_id": id_schema, + "vduImage": name_schema, "vnf": { "type": "array", "minItems": 1, -- 2.17.1