X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_lcm%2Fns.py;h=1adcdacf0f2b0045ba293d5f7e6f5d30222b1ea2;hb=b0e5fe4b18e985455efad8bd926d8c782aa49b61;hp=6a4b47f62444c59f428c947be58b7ec54a204f24;hpb=485b29173db188063464c74773f76ab4daf9d701;p=osm%2FLCM.git diff --git a/osm_lcm/ns.py b/osm_lcm/ns.py index 6a4b47f..1adcdac 100644 --- a/osm_lcm/ns.py +++ b/osm_lcm/ns.py @@ -32,6 +32,7 @@ from osm_common.dbbase import DbException from osm_common.fsbase import FsException from n2vc.n2vc_juju_conn import N2VCJujuConnector +from n2vc.exceptions import N2VCException from copy import copy, deepcopy from http import HTTPStatus @@ -454,8 +455,7 @@ class NsLcm(LcmBase): if isinstance(vld_params["ns-net"], dict): for vld_id, instance_scenario_id in vld_params["ns-net"].items(): RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id} - if RO_vld_ns_net: - populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net) + populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net) if "vnfd-connection-point-ref" in vld_params: for cp_params in vld_params["vnfd-connection-point-ref"]: # look for interface @@ -1117,6 +1117,20 @@ class NsLcm(LcmBase): raise Exception("{} {}".format(step, e)) from e # TODO raise N2VC exception with 'step' extra information + def _write_ns_status(self, nsr_id: str, ns_state: str, current_operation: str, current_operation_id: str, + error_description: str = None, error_detail: str = None): + try: + db_dict = dict() + if ns_state: + db_dict["nsState"] = ns_state + db_dict["currentOperation"] = current_operation + db_dict["currentOperationID"] = current_operation_id + db_dict["errorDescription"] = error_description + db_dict["errorDetail"] = error_detail + self.update_db_2("nsrs", nsr_id, db_dict) + except Exception as e: + self.logger.warn('Error writing NS status: {}'.format(e)) + async def instantiate(self, nsr_id, nslcmop_id): """ @@ -1155,6 +1169,7 @@ class NsLcm(LcmBase): db_vnfrs = {} # vnf's info indexed by member-index # n2vc_info = {} task_instantiation_list = [] + task_instantiation_info = {} # from task to info text exc = None try: # wait for any previous tasks in process @@ -1163,6 +1178,14 @@ class NsLcm(LcmBase): # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds) + # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id + self._write_ns_status( + nsr_id=nsr_id, + ns_state="BUILDING", + current_operation="INSTANTIATING", + current_operation_id=nslcmop_id + ) + # read from db: operation step = "Getting nslcmop={} from db".format(nslcmop_id) db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id}) @@ -1237,6 +1260,7 @@ class NsLcm(LcmBase): ) ) self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDUs", task_kdu) + task_instantiation_info[task_kdu] = "Deploy KDUs" task_instantiation_list.append(task_kdu) # n2vc_redesign STEP 1 Get VCA public ssh-key # feature 1429. Add n2vc public key to needed VMs @@ -1259,6 +1283,7 @@ class NsLcm(LcmBase): ) ) self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro) + task_instantiation_info[task_ro] = "Deploy at VIM" task_instantiation_list.append(task_ro) # n2vc_redesign STEP 3 to 6 Deploy N2VC @@ -1301,7 +1326,8 @@ class NsLcm(LcmBase): deploy_params=deploy_params, descriptor_config=descriptor_config, base_folder=base_folder, - task_instantiation_list=task_instantiation_list + task_instantiation_list=task_instantiation_list, + task_instantiation_info=task_instantiation_info ) # Deploy charms for each VDU that supports one. @@ -1342,7 +1368,8 @@ class NsLcm(LcmBase): deploy_params=deploy_params_vdu, descriptor_config=descriptor_config, base_folder=base_folder, - task_instantiation_list=task_instantiation_list + task_instantiation_list=task_instantiation_list, + task_instantiation_info=task_instantiation_info ) for kdud in get_iterable(vnfd, 'kdu'): kdu_name = kdud["name"] @@ -1377,7 +1404,8 @@ class NsLcm(LcmBase): deploy_params=deploy_params, descriptor_config=descriptor_config, base_folder=base_folder, - task_instantiation_list=task_instantiation_list + task_instantiation_list=task_instantiation_list, + task_instantiation_info=task_instantiation_info ) # Check if this NS has a charm configuration @@ -1412,33 +1440,64 @@ class NsLcm(LcmBase): deploy_params=deploy_params, descriptor_config=descriptor_config, base_folder=base_folder, - task_instantiation_list=task_instantiation_list + task_instantiation_list=task_instantiation_list, + task_instantiation_info=task_instantiation_info ) # Wait until all tasks of "task_instantiation_list" have been finished # while time() <= start_deploy + self.total_deploy_timeout: - error_text = None - timeout = 3600 # time() - start_deploy - task_instantiation_set = set(task_instantiation_list) # build a set with tasks - done = None - pending = None - if len(task_instantiation_set) > 0: - done, pending = await asyncio.wait(task_instantiation_set, timeout=timeout) - if pending: - error_text = "timeout" - for task in done: - if task.cancelled(): - if not error_text: - error_text = "cancelled" - elif task.done(): - exc = task.exception() - if exc: - error_text = str(exc) + error_text_list = [] + timeout = 3600 + + # let's begin with all OK + instantiated_ok = True + # let's begin with RO 'running' status (later we can change it) + db_nsr_update["operational-status"] = "running" + # let's begin with VCA 'configured' status (later we can change it) + db_nsr_update["config-status"] = "configured" + + if task_instantiation_list: + # wait for all tasks completion + done, pending = await asyncio.wait(task_instantiation_list, timeout=timeout) + + for task in pending: + instantiated_ok = False + if task == task_ro: + db_nsr_update["operational-status"] = "failed" + else: + db_nsr_update["config-status"] = "failed" + self.logger.error(logging_text + task_instantiation_info[task] + ": Timeout") + error_text_list.append(task_instantiation_info[task] + ": Timeout") + for task in done: + if task.cancelled(): + instantiated_ok = False + if task == task_ro: + db_nsr_update["operational-status"] = "failed" + else: + db_nsr_update["config-status"] = "failed" + self.logger.warn(logging_text + task_instantiation_info[task] + ": Cancelled") + error_text_list.append(task_instantiation_info[task] + ": Cancelled") + else: + exc = task.exception() + if exc: + instantiated_ok = False + if task == task_ro: + db_nsr_update["operational-status"] = "failed" + else: + db_nsr_update["config-status"] = "failed" + self.logger.error(logging_text + task_instantiation_info[task] + ": Failed") + if isinstance(exc, (N2VCException, ROclient.ROClientException)): + error_text_list.append(task_instantiation_info[task] + ": {}".format(exc)) + else: + exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__)) + self.logger.error(logging_text + task_instantiation_info[task] + exc_traceback) + error_text_list.append(task_instantiation_info[task] + ": " + exc_traceback) + else: + self.logger.debug(logging_text + task_instantiation_info[task] + ": Done") - if error_text: - db_nsr_update["config-status"] = "failed" - error_text = "fail configuring " + error_text + if error_text_list: + error_text = "\n".join(error_text_list) db_nsr_update["detailed-status"] = error_text db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP" db_nslcmop_update["detailed-status"] = error_text @@ -1448,7 +1507,6 @@ class NsLcm(LcmBase): db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED" db_nslcmop_update["statusEnteredTime"] = time() db_nslcmop_update["detailed-status"] = "done" - db_nsr_update["config-status"] = "configured" db_nsr_update["detailed-status"] = "done" except (ROclient.ROClientException, DbException, LcmException) as e: @@ -1466,6 +1524,7 @@ class NsLcm(LcmBase): if db_nsr: db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc) db_nsr_update["operational-status"] = "failed" + db_nsr_update["config-status"] = "failed" if db_nslcmop: db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc) db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED" @@ -1476,6 +1535,26 @@ class NsLcm(LcmBase): db_nsr_update["_admin.current-operation"] = None db_nsr_update["_admin.operation-type"] = None self.update_db_2("nsrs", nsr_id, db_nsr_update) + + # nsState="READY/BROKEN", currentOperation="IDLE", currentOperationID=None + ns_state = None + error_description = None + error_detail = None + if instantiated_ok: + ns_state = "READY" + else: + ns_state = "BROKEN" + error_description = 'Operation: INSTANTIATING.{}, step: {}'.format(nslcmop_id, step) + error_detail = error_text + self._write_ns_status( + nsr_id=nsr_id, + ns_state=ns_state, + current_operation="IDLE", + current_operation_id=None, + error_description=error_description, + error_detail=error_detail + ) + if db_nslcmop_update: self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update) except DbException as e: @@ -1595,7 +1674,7 @@ class NsLcm(LcmBase): def _deploy_n2vc(self, logging_text, db_nsr, db_vnfr, nslcmop_id, nsr_id, nsi_id, vnfd_id, vdu_id, kdu_name, member_vnf_index, vdu_index, vdu_name, deploy_params, descriptor_config, - base_folder, task_instantiation_list): + base_folder, task_instantiation_list, task_instantiation_info): # launch instantiate_N2VC in a asyncio task and register task object # Look where information of this charm is at database ._admin.deployed.VCA # if not found, create one entry and update database @@ -1644,6 +1723,7 @@ class NsLcm(LcmBase): ) ) self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_N2VC-{}".format(vca_index), task_n2vc) + task_instantiation_info[task_n2vc] = "Deploy VCA {}.{}".format(member_vnf_index or "", vdu_id or "") task_instantiation_list.append(task_n2vc) # Check if this VNFD has a configured terminate action @@ -1962,6 +2042,13 @@ class NsLcm(LcmBase): step = "Waiting for previous operations to terminate" await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id) + self._write_ns_status( + nsr_id=nsr_id, + ns_state="TERMINATING", + current_operation="TERMINATING", + current_operation_id=nslcmop_id + ) + step = "Getting nslcmop={} from db".format(nslcmop_id) db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id}) step = "Getting nsr={} from db".format(nsr_id) @@ -2136,6 +2223,7 @@ class NsLcm(LcmBase): self.logger.error(logging_text + failed_detail[-1]) if failed_detail: + terminate_ok = False self.logger.error(logging_text + " ;".join(failed_detail)) db_nsr_update["operational-status"] = "failed" db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail) @@ -2143,6 +2231,7 @@ class NsLcm(LcmBase): db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED" db_nslcmop_update["statusEnteredTime"] = time() else: + terminate_ok = True db_nsr_update["operational-status"] = "terminated" db_nsr_update["detailed-status"] = "Done" db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED" @@ -2174,6 +2263,25 @@ class NsLcm(LcmBase): db_nsr_update["_admin.current-operation"] = None db_nsr_update["_admin.operation-type"] = None self.update_db_2("nsrs", nsr_id, db_nsr_update) + + if terminate_ok: + ns_state = "IDLE" + error_description = None + error_detail = None + else: + ns_state = "BROKEN" + error_description = 'Operation: TERMINATING.{}, step: {}'.format(nslcmop_id, step) + error_detail = "; ".join(failed_detail) + + self._write_ns_status( + nsr_id=nsr_id, + ns_state=ns_state, + current_operation="IDLE", + current_operation_id=None, + error_description=error_description, + error_detail=error_detail + ) + except DbException as e: self.logger.error(logging_text + "Cannot update database: {}".format(e)) if nslcmop_operation_state: @@ -2320,6 +2428,13 @@ class NsLcm(LcmBase): step = "Waiting for previous operations to terminate" await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id) + self._write_ns_status( + nsr_id=nsr_id, + ns_state=None, + current_operation="RUNNING ACTION", + current_operation_id=nslcmop_id + ) + step = "Getting information from database" db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id}) db_nsr = self.db.get_one("nsrs", {"_id": nsr_id}) @@ -2363,7 +2478,7 @@ class NsLcm(LcmBase): break elif kdu_name: self.logger.debug(logging_text + "Checking actions in KDUs") - kdur = next((x for x in db_vnfr["kdur"] if x["kdu_name"] == kdu_name), None) + kdur = next((x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None) desc_params = self._format_additional_params(kdur.get("additionalParams")) or {} if primitive_params: desc_params.update(primitive_params) @@ -2512,6 +2627,12 @@ class NsLcm(LcmBase): db_nsr_update["_admin.nslcmop"] = None db_nsr_update["_admin.current-operation"] = None self.update_db_2("nsrs", nsr_id, db_nsr_update) + self._write_ns_status( + nsr_id=nsr_id, + ns_state=None, + current_operation="IDLE", + current_operation_id=None + ) except DbException as e: self.logger.error(logging_text + "Cannot update database: {}".format(e)) self.logger.debug(logging_text + "Exit") @@ -2555,6 +2676,13 @@ class NsLcm(LcmBase): step = "Waiting for previous operations to terminate" await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id) + self._write_ns_status( + nsr_id=nsr_id, + ns_state=None, + current_operation="SCALING", + current_operation_id=nslcmop_id + ) + step = "Getting nslcmop from database" self.logger.debug(step + " after having waited for previous tasks to be completed") db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id}) @@ -2889,7 +3017,7 @@ class NsLcm(LcmBase): # Post-scale reintent check: Check if this sub-operation has been executed before op_index = self._check_or_add_scale_suboperation( db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE') - if (op_index == self.SUBOPERATION_STATUS_SKIP): + if op_index == self.SUBOPERATION_STATUS_SKIP: # Skip sub-operation result = 'COMPLETED' result_detail = 'Done' @@ -2897,7 +3025,7 @@ class NsLcm(LcmBase): "vnf_config_primitive={} Skipped sub-operation, result {} {}". format(vnf_config_primitive, result, result_detail)) else: - if (op_index == self.SUBOPERATION_STATUS_NEW): + if op_index == self.SUBOPERATION_STATUS_NEW: # New sub-operation: Get index of this sub-operation op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation". @@ -2968,6 +3096,14 @@ class NsLcm(LcmBase): db_nsr_update["_admin.operation-type"] = None db_nsr_update["_admin.nslcmop"] = None self.update_db_2("nsrs", nsr_id, db_nsr_update) + + self._write_ns_status( + nsr_id=nsr_id, + ns_state=None, + current_operation="IDLE", + current_operation_id=None + ) + except DbException as e: self.logger.error(logging_text + "Cannot update database: {}".format(e)) if nslcmop_operation_state: