fix 1214: change timeout from 10min to 5min for k8s cluster creation
[osm/LCM.git] / osm_lcm / ns.py
index fb1478e..10d07ef 100644 (file)
@@ -629,11 +629,21 @@ class NsLcm(LcmBase):
                 continue
             vdu_id_ref = vdur["vdu-id-ref"]
             if vdu_create and vdu_create.get(vdu_id_ref):
+                vdur_copy = deepcopy(vdur)
+                vdur_copy["status"] = "BUILD"
+                vdur_copy["status-detailed"] = None
+                vdur_copy["ip_address"]: None
+                for iface in vdur_copy["interfaces"]:
+                    iface["ip-address"] = None
+                    iface["mac-address"] = None
+                    iface.pop("mgmt_vnf", None)  # only first vdu can be managment of vnf   # TODO ALF
                 for index in range(0, vdu_create[vdu_id_ref]):
-                    vdur = deepcopy(vdur)
-                    vdur["_id"] = str(uuid4())
-                    vdur["count-index"] += 1
-                    vdurs.insert(vdu_index+1+index, vdur)
+                    vdur_copy["_id"] = str(uuid4())
+                    vdur_copy["count-index"] += 1
+                    vdurs.insert(vdu_index+1+index, vdur_copy)
+                    self.logger.debug("scale out, adding vdu={}".format(vdur_copy))
+                    vdur_copy = deepcopy(vdur_copy)
+
                 del vdu_create[vdu_id_ref]
             if vdu_delete and vdu_delete.get(vdu_id_ref):
                 del vdurs[vdu_index]
@@ -1236,6 +1246,34 @@ class NsLcm(LcmBase):
             self.set_vnfr_at_error(db_vnfrs, str(e))
             raise
 
+    async def wait_kdu_up(self, logging_text, nsr_id, vnfr_id, kdu_name):
+        """
+        Wait for kdu to be up, get ip address
+        :param logging_text: prefix use for logging
+        :param nsr_id:
+        :param vnfr_id:
+        :param kdu_name:
+        :return: IP address
+        """
+
+        # self.logger.debug(logging_text + "Starting wait_kdu_up")
+        nb_tries = 0
+
+        while nb_tries < 360:
+            db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
+            kdur = next((x for x in get_iterable(db_vnfr, "kdur") if x.get("name") == kdu_name), None)
+            if not kdur:
+                raise LcmException("Not found vnfr_id={}, kdu_name={}".format(vnfr_id, kdu_name))
+            if kdur.get("status"):
+                if kdur["status"] in ("READY", "ENABLED"):
+                    return kdur.get("ip-address")
+                else:
+                    raise LcmException("target KDU={} is in error state".format(kdu_name))
+
+            await asyncio.sleep(10, loop=self.loop)
+            nb_tries += 1
+        raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
+
     async def wait_vm_up_insert_key_ro(self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None):
         """
         Wait for ip addres at RO, and optionally, insert public key in virtual machine
@@ -1581,8 +1619,11 @@ class NsLcm(LcmBase):
                 # n2vc_redesign STEP 5.1
                 # wait for RO (ip-address) Insert pub_key into VM
                 if vnfr_id:
-                    rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
-                                                                     user=user, pub_key=pub_key)
+                    if kdu_name:
+                        rw_mgmt_ip = await self.wait_kdu_up(logging_text, nsr_id, vnfr_id, kdu_name)
+                    else:
+                        rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id,
+                                                                         vdu_index, user=user, pub_key=pub_key)
                 else:
                     rw_mgmt_ip = None   # This is for a NS configuration
 
@@ -2349,10 +2390,13 @@ class NsLcm(LcmBase):
                     to_vca_endpoint = None
                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
                     for vca in vca_list:
-                        if vca.get('vdu_id') == r.get('entities')[0].get('id') and vca.get('config_sw_installed'):
+                        key_to_check = "vdu_id"
+                        if vca.get("vdu_id") is None:
+                            key_to_check = "vnfd_id"
+                        if vca.get(key_to_check) == r.get('entities')[0].get('id') and vca.get('config_sw_installed'):
                             from_vca_ee_id = vca.get('ee_id')
                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
-                        if vca.get('vdu_id') == r.get('entities')[1].get('id') and vca.get('config_sw_installed'):
+                        if vca.get(key_to_check) == r.get('entities')[1].get('id') and vca.get('config_sw_installed'):
                             to_vca_ee_id = vca.get('ee_id')
                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
                     if from_vca_ee_id and to_vca_ee_id:
@@ -2397,7 +2441,7 @@ class NsLcm(LcmBase):
             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
             return False
 
-    async def _install_kdu(self, nsr_id: str, nsr_db_path: str, vnfr_data: dict, kdu_index: int, kdur: dict, kdud: dict,
+    async def _install_kdu(self, nsr_id: str, nsr_db_path: str, vnfr_data: dict, kdu_index: int, kdud: dict,
                            vnfd: dict, k8s_instance_info: dict, k8params: dict = None, timeout: int = 600):
 
         try:
@@ -2425,8 +2469,9 @@ class NsLcm(LcmBase):
                 namespace=k8s_instance_info["namespace"])
 
             # Obtain management service info (if exists)
+            vnfr_update_dict = {}
             if services:
-                vnfr_update_dict = {"kdur.{}.services".format(kdu_index): services}
+                vnfr_update_dict["kdur.{}.services".format(kdu_index)] = services
                 mgmt_services = [service for service in kdud.get("service", []) if service.get("mgmt-service")]
                 for mgmt_service in mgmt_services:
                     for service in services:
@@ -2448,7 +2493,8 @@ class NsLcm(LcmBase):
                     else:
                         self.logger.warn("Mgmt service name: {} not found".format(mgmt_service["name"]))
 
-                self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
+            vnfr_update_dict["kdur.{}.status".format(kdu_index)] = "READY"
+            self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
 
             kdu_config = kdud.get("kdu-configuration")
             if kdu_config and kdu_config.get("initial-config-primitive") and kdu_config.get("juju") is None:
@@ -2470,6 +2516,7 @@ class NsLcm(LcmBase):
             # Prepare update db with error and raise exception
             try:
                 self.update_db_2("nsrs", nsr_id, {nsr_db_path + ".detailed-status": str(e)})
+                self.update_db_2("vnfrs", vnfr_data.get("_id"), {"kdur.{}.status".format(kdu_index): "ERROR"})
             except Exception:
                 # ignore to keep original exception
                 pass
@@ -2579,7 +2626,7 @@ class NsLcm(LcmBase):
                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 
                     task = asyncio.ensure_future(
-                        self._install_kdu(nsr_id, db_path, vnfr_data, kdu_index, kdur, kdud, db_vnfds[vnfd_id],
+                        self._install_kdu(nsr_id, db_path, vnfr_data, kdu_index, kdud, db_vnfds[vnfd_id],
                                           k8s_instance_info, k8params=desc_params, timeout=600))
                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
@@ -3219,7 +3266,6 @@ class NsLcm(LcmBase):
             self.logger.debug(logging_text + stage[1])
             # self.logger.debug("nsr_deployed: {}".format(nsr_deployed))
             for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
-                self.logger.debug("vca_index: {}, vca: {}".format(vca_index, vca))
                 config_descriptor = None
                 if not vca or not vca.get("ee_id"):
                     continue
@@ -3243,7 +3289,9 @@ class NsLcm(LcmBase):
                                              vca.get("needed_terminate"))
                 # For helm we must destroy_ee. Also for native_charm, as juju_model cannot be deleted if there are
                 # pending native charms
-                destroy_ee = "True" if vca_type in ("helm", "native_charm") else "False"
+                destroy_ee = True if vca_type in ("helm", "native_charm") else False
+                # self.logger.debug(logging_text + "vca_index: {}, ee_id: {}, vca_type: {} destroy_ee: {}".format(
+                #     vca_index, vca.get("ee_id"), vca_type, destroy_ee))
                 task = asyncio.ensure_future(
                     self.destroy_N2VC(logging_text, db_nslcmop, vca, config_descriptor, vca_index,
                                       destroy_ee, exec_terminate_primitives))
@@ -3251,13 +3299,13 @@ class NsLcm(LcmBase):
 
             # wait for pending tasks of terminate primitives
             if tasks_dict_info:
-                self.logger.debug(logging_text + 'Waiting for terminate primitive pending tasks...')
+                self.logger.debug(logging_text + 'Waiting for tasks {}'.format(list(tasks_dict_info.keys())))
                 error_list = await self._wait_for_tasks(logging_text, tasks_dict_info,
                                                         min(self.timeout_charm_delete, timeout_ns_terminate),
                                                         stage, nslcmop_id)
+                tasks_dict_info.clear()
                 if error_list:
                     return   # raise LcmException("; ".join(error_list))
-                tasks_dict_info.clear()
 
             # remove All execution environments at once
             stage[0] = "Stage 3/3 delete all."
@@ -3815,7 +3863,6 @@ class NsLcm(LcmBase):
         scale_process = None
         old_operational_status = ""
         old_config_status = ""
-        vnfr_scaled = False
         try:
             # wait for any previous tasks in process
             step = "Waiting for previous operations to terminate"
@@ -4060,7 +4107,7 @@ class NsLcm(LcmBase):
                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
 
                     RO_task_done = False
-                    step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
+                    step = detailed_status = "Waiting for VIM to scale. RO_task_id={}.".format(RO_nslcmop_id)
                     detailed_status_old = None
                     self.logger.debug(logging_text + step)
 
@@ -4080,11 +4127,16 @@ class NsLcm(LcmBase):
                                 detailed_status = step + "; {}".format(ns_status_info)
                             elif ns_status == "ACTIVE":
                                 RO_task_done = True
+                                self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
                                 self.logger.debug(logging_text + step)
                             else:
                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
                         else:
+                            desc = await self.RO.show("ns", RO_nsr_id)
+                            ns_status, ns_status_info = self.RO.check_ns_status(desc)
+                            # deploymentStatus
+                            self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
 
                             if ns_status == "ERROR":
                                 raise ROclient.ROClientException(ns_status_info)
@@ -4093,15 +4145,7 @@ class NsLcm(LcmBase):
                             elif ns_status == "ACTIVE":
                                 step = detailed_status = \
                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
-                                if not vnfr_scaled:
-                                    self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
-                                    vnfr_scaled = True
                                 try:
-                                    desc = await self.RO.show("ns", RO_nsr_id)
-
-                                    # deploymentStatus
-                                    self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
-
                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
                                     break