Provide more info upon instantiate task exceptions
[osm/LCM.git] / osm_lcm / ns.py
index f66c23a..8470086 100644 (file)
@@ -26,11 +26,13 @@ from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound,
 from osm_lcm import ROclient
 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase, deep_get
 from n2vc.k8s_helm_conn import K8sHelmConnector
+from n2vc.k8s_juju_conn import K8sJujuConnector
 
 from osm_common.dbbase import DbException
 from osm_common.fsbase import FsException
 
 from n2vc.n2vc_juju_conn import N2VCJujuConnector
+from n2vc.exceptions import N2VCException
 
 from copy import copy, deepcopy
 from http import HTTPStatus
@@ -125,6 +127,15 @@ class NsLcm(LcmBase):
             on_update_db=None,
         )
 
+        self.k8sclusterjuju = K8sJujuConnector(
+            kubectl_command=self.vca_config.get("kubectlpath"),
+            juju_command=self.vca_config.get("jujupath"),
+            fs=self.fs,
+            log=self.logger,
+            db=self.db,
+            on_update_db=None,
+        )
+
         # create RO client
         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 
@@ -871,9 +882,14 @@ class NsLcm(LcmBase):
         ip_address = None
         nb_tries = 0
         target_vdu_id = None
+        ro_retries = 0
 
         while True:
 
+            ro_retries += 1
+            if ro_retries >= 360:  # 1 hour
+                raise LcmException("Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id))
+
             await asyncio.sleep(10, loop=self.loop)
             # wait until NS is deployed at RO
             if not ro_nsr_id:
@@ -885,23 +901,29 @@ class NsLcm(LcmBase):
             # get ip address
             if not target_vdu_id:
                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
-                if not vdu_id:
+
+                if not vdu_id:  # for the VNF case
                     ip_address = db_vnfr.get("ip-address")
                     if not ip_address:
                         continue
-                for vdur in get_iterable(db_vnfr, "vdur"):
-                    if (vdur["vdu-id-ref"] == vdu_id and vdur["count-index"] == vdu_index) or \
-                            (ip_address and vdur.get("ip-address") == ip_address):
-                        if vdur.get("status") == "ACTIVE":
-                            target_vdu_id = vdur["vdu-id-ref"]
-                        elif vdur.get("status") == "ERROR":
-                            raise LcmException("Cannot inject ssh-key because target VM is in error state")
-                        break
-                else:
+                    vdur = next((x for x in get_iterable(db_vnfr, "vdur") if x.get("ip-address") == ip_address), None)
+                else:  # VDU case
+                    vdur = next((x for x in get_iterable(db_vnfr, "vdur")
+                                 if x.get("vdu-id-ref") == vdu_id and x.get("count-index") == vdu_index), None)
+
+                if not vdur:
                     raise LcmException("Not found vnfr_id={}, vdu_index={}, vdu_index={}".format(
                         vnfr_id, vdu_id, vdu_index
                     ))
 
+                if vdur.get("status") == "ACTIVE":
+                    ip_address = vdur.get("ip-address")
+                    if not ip_address:
+                        continue
+                    target_vdu_id = vdur["vdu-id-ref"]
+                elif vdur.get("status") == "ERROR":
+                    raise LcmException("Cannot inject ssh-key because target VM is in error state")
+
             if not target_vdu_id:
                 continue
 
@@ -968,7 +990,7 @@ class NsLcm(LcmBase):
                     namespace += ".{}-{}".format(vdu_id, vdu_index or 0)
 
             # Get artifact path
-            artifact_path = "/{}/{}/charms/{}".format(
+            artifact_path = "{}/{}/charms/{}".format(
                 base_folder["folder"],
                 base_folder["pkg-dir"],
                 config_descriptor["juju"]["charm"]
@@ -1134,6 +1156,7 @@ class NsLcm(LcmBase):
         db_vnfrs = {}     # vnf's info indexed by member-index
         # n2vc_info = {}
         task_instantiation_list = []
+        task_instantiation_info = {}  # from task to info text
         exc = None
         try:
             # wait for any previous tasks in process
@@ -1216,6 +1239,7 @@ class NsLcm(LcmBase):
                 )
             )
             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDUs", task_kdu)
+            task_instantiation_info[task_kdu] = "Deploy KDUs"
             task_instantiation_list.append(task_kdu)
             # n2vc_redesign STEP 1 Get VCA public ssh-key
             # feature 1429. Add n2vc public key to needed VMs
@@ -1238,6 +1262,7 @@ class NsLcm(LcmBase):
                 )
             )
             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
+            task_instantiation_info[task_ro] = "Deploy at VIM"
             task_instantiation_list.append(task_ro)
 
             # n2vc_redesign STEP 3 to 6 Deploy N2VC
@@ -1280,7 +1305,8 @@ class NsLcm(LcmBase):
                         deploy_params=deploy_params,
                         descriptor_config=descriptor_config,
                         base_folder=base_folder,
-                        task_instantiation_list=task_instantiation_list
+                        task_instantiation_list=task_instantiation_list,
+                        task_instantiation_info=task_instantiation_info
                     )
 
                 # Deploy charms for each VDU that supports one.
@@ -1321,7 +1347,8 @@ class NsLcm(LcmBase):
                                 deploy_params=deploy_params_vdu,
                                 descriptor_config=descriptor_config,
                                 base_folder=base_folder,
-                                task_instantiation_list=task_instantiation_list
+                                task_instantiation_list=task_instantiation_list,
+                                task_instantiation_info=task_instantiation_info
                             )
                 for kdud in get_iterable(vnfd, 'kdu'):
                     kdu_name = kdud["name"]
@@ -1356,7 +1383,8 @@ class NsLcm(LcmBase):
                             deploy_params=deploy_params,
                             descriptor_config=descriptor_config,
                             base_folder=base_folder,
-                            task_instantiation_list=task_instantiation_list
+                            task_instantiation_list=task_instantiation_list,
+                            task_instantiation_info=task_instantiation_info
                         )
 
             # Check if this NS has a charm configuration
@@ -1391,33 +1419,35 @@ class NsLcm(LcmBase):
                     deploy_params=deploy_params,
                     descriptor_config=descriptor_config,
                     base_folder=base_folder,
-                    task_instantiation_list=task_instantiation_list
+                    task_instantiation_list=task_instantiation_list,
+                    task_instantiation_info=task_instantiation_info
                 )
 
             # Wait until all tasks of "task_instantiation_list" have been finished
 
             # while time() <= start_deploy + self.total_deploy_timeout:
-            error_text = None
+            error_text_list = []
             timeout = 3600  # time() - start_deploy
-            task_instantiation_set = set(task_instantiation_list)    # build a set with tasks
-            done = None
-            pending = None
-            if len(task_instantiation_set) > 0:
-                done, pending = await asyncio.wait(task_instantiation_set, timeout=timeout)
-            if pending:
-                error_text = "timeout"
-            for task in done:
-                if task.cancelled():
-                    if not error_text:
-                        error_text = "cancelled"
-                elif task.done():
-                    exc = task.exception()
-                    if exc:
-                        error_text = str(exc)
+            if task_instantiation_list:
+                done, pending = await asyncio.wait(task_instantiation_list, timeout=timeout)
+                if pending:
+                    for task in pending:
+                        error_text_list.append(task_instantiation_info[task] + ": Timeout")
+                for task in done:
+                    if task.cancelled():
+                        error_text_list.append(task_instantiation_info[task] + ": Cancelled")
+                    elif task.done():
+                        exc = task.exception()
+                        if exc:
+                            if isinstance(exc, (N2VCException, ROclient.ROClientException)):
+                                error_text_list.append(task_instantiation_info[task] + ": {}".format(exc))
+                            else:
+                                error_text_list.append(task_instantiation_info[task] + ": " + "".
+                                                       join(traceback.format_exception(None, exc, exc.__traceback__)))
 
-            if error_text:
+            if error_text_list:
+                error_text = "\n".join(error_text_list)
                 db_nsr_update["config-status"] = "failed"
-                error_text = "fail configuring " + error_text
                 db_nsr_update["detailed-status"] = error_text
                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
                 db_nslcmop_update["detailed-status"] = error_text
@@ -1540,8 +1570,10 @@ class NsLcm(LcmBase):
                                                         params=desc_params, db_dict=db_dict, timeout=3600)
                         )
                     else:
-                        # TODO juju-bundle connector in place
-                        pass
+                        task = self.k8sclusterjuju.install(cluster_uuid=cluster_uuid, kdu_model=kdumodel,
+                                                           atomic=True, params=desc_params,
+                                                           db_dict=db_dict, timeout=600)
+
                     pending_tasks[task] = "_admin.deployed.K8s.{}.".format(index)
                     index += 1
             if not pending_tasks:
@@ -1572,7 +1604,7 @@ class NsLcm(LcmBase):
 
     def _deploy_n2vc(self, logging_text, db_nsr, db_vnfr, nslcmop_id, nsr_id, nsi_id, vnfd_id, vdu_id,
                      kdu_name, member_vnf_index, vdu_index, vdu_name, deploy_params, descriptor_config,
-                     base_folder, task_instantiation_list):
+                     base_folder, task_instantiation_list, task_instantiation_info):
         # launch instantiate_N2VC in a asyncio task and register task object
         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
         # if not found, create one entry and update database
@@ -1621,6 +1653,7 @@ class NsLcm(LcmBase):
             )
         )
         self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_N2VC-{}".format(vca_index), task_n2vc)
+        task_instantiation_info[task_n2vc] = "Deploy VCA {}.{}".format(member_vnf_index or "", vdu_id or "")
         task_instantiation_list.append(task_n2vc)
 
     # Check if this VNFD has a configured terminate action
@@ -1984,8 +2017,9 @@ class NsLcm(LcmBase):
                                 self.k8sclusterhelm.uninstall(cluster_uuid=kdu.get("k8scluster-uuid"),
                                                               kdu_instance=kdu_instance))
                         elif kdu.get("k8scluster-type") == "juju":
-                            # TODO Juju connector needed
-                            continue
+                            task_delete_kdu_instance = asyncio.ensure_future(
+                                self.k8sclusterjuju.uninstall(cluster_uuid=kdu.get("k8scluster-uuid"),
+                                                              kdu_instance=kdu_instance))
                         else:
                             self.error(logging_text + "Unknown k8s deployment type {}".
                                        format(kdu.get("k8scluster-type")))
@@ -2366,8 +2400,12 @@ class NsLcm(LcmBase):
                                                                            params=desc_params, db_dict=db_dict,
                                                                            timeout=300)
                             elif kdu.get("k8scluster-type") == "juju":
-                                # TODO Juju connector needed
-                                pass
+                                output = await self.k8sclusterjuju.upgrade(cluster_uuid=kdu.get("k8scluster-uuid"),
+                                                                           kdu_instance=kdu.get("kdu-instance"),
+                                                                           atomic=True, kdu_model=kdu_model,
+                                                                           params=desc_params, db_dict=db_dict,
+                                                                           timeout=300)
+
                             else:
                                 msg = "k8scluster-type not defined"
                                 raise LcmException(msg)
@@ -2380,8 +2418,9 @@ class NsLcm(LcmBase):
                                                                             kdu_instance=kdu.get("kdu-instance"),
                                                                             db_dict=db_dict)
                             elif kdu.get("k8scluster-type") == "juju":
-                                # TODO Juju connector needed
-                                pass
+                                output = await self.k8sclusterjuju.rollback(cluster_uuid=kdu.get("k8scluster-uuid"),
+                                                                            kdu_instance=kdu.get("kdu-instance"),
+                                                                            db_dict=db_dict)
                             else:
                                 msg = "k8scluster-type not defined"
                                 raise LcmException(msg)
@@ -2391,8 +2430,8 @@ class NsLcm(LcmBase):
                                 output = await self.k8sclusterhelm.status_kdu(cluster_uuid=kdu.get("k8scluster-uuid"),
                                                                               kdu_instance=kdu.get("kdu-instance"))
                             elif kdu.get("k8scluster-type") == "juju":
-                                # TODO Juju connector needed
-                                pass
+                                output = await self.k8sclusterjuju.status_kdu(cluster_uuid=kdu.get("k8scluster-uuid"),
+                                                                              kdu_instance=kdu.get("kdu-instance"))
                             else:
                                 msg = "k8scluster-type not defined"
                                 raise LcmException(msg)