Bug 1830 fixed
[osm/LCM.git] / osm_lcm / ns.py
index dd7ff3e..fda6722 100644 (file)
@@ -124,7 +124,7 @@ from copy import copy, deepcopy
 from time import time
 from uuid import uuid4
 
-from random import randint
+from random import SystemRandom
 
 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 
@@ -133,9 +133,18 @@ class NsLcm(LcmBase):
     SUBOPERATION_STATUS_NOT_FOUND = -1
     SUBOPERATION_STATUS_NEW = -2
     SUBOPERATION_STATUS_SKIP = -3
+    EE_TLS_NAME = "ee-tls"
     task_name_deploy_vca = "Deploying VCA"
-
-    def __init__(self, msg, lcm_tasks, config: LcmCfg, loop):
+    rel_operation_types = {
+        "GE": ">=",
+        "LE": "<=",
+        "GT": ">",
+        "LT": "<",
+        "EQ": "==",
+        "NE": "!=",
+    }
+
+    def __init__(self, msg, lcm_tasks, config: LcmCfg):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
@@ -145,7 +154,6 @@ class NsLcm(LcmBase):
 
         self.db = Database().instance.db
         self.fs = Filesystem().instance.fs
-        self.loop = loop
         self.lcm_tasks = lcm_tasks
         self.timeout = config.timeout
         self.ro_config = config.RO
@@ -154,7 +162,6 @@ class NsLcm(LcmBase):
         # create N2VC connector
         self.n2vc = N2VCJujuConnector(
             log=self.logger,
-            loop=self.loop,
             on_update_db=self._on_update_n2vc_db,
             fs=self.fs,
             db=self.db,
@@ -162,7 +169,6 @@ class NsLcm(LcmBase):
 
         self.conn_helm_ee = LCMHelmConn(
             log=self.logger,
-            loop=self.loop,
             vca_config=self.vca_config,
             on_update_db=self._on_update_n2vc_db,
         )
@@ -189,7 +195,6 @@ class NsLcm(LcmBase):
             kubectl_command=self.vca_config.kubectlpath,
             juju_command=self.vca_config.jujupath,
             log=self.logger,
-            loop=self.loop,
             on_update_db=self._on_update_k8s_db,
             fs=self.fs,
             db=self.db,
@@ -212,7 +217,7 @@ class NsLcm(LcmBase):
         }
 
         # create RO client
-        self.RO = NgRoClient(self.loop, **self.ro_config.to_dict())
+        self.RO = NgRoClient(**self.ro_config.to_dict())
 
         self.op_status_map = {
             "instantiation": self.RO.status,
@@ -899,7 +904,7 @@ class NsLcm(LcmBase):
                     )
                     vdur = next((vdur for vdur in target_vnf.get("vdur", ())), None)
                     if not vdur:
-                        return
+                        continue
                     for a_index, a_vld in enumerate(target["ns"]["vld"]):
                         target_vld = find_in_list(
                             get_iterable(vdur, "interfaces"),
@@ -967,6 +972,10 @@ class NsLcm(LcmBase):
             image["vim_info"] = {}
         for flavor in target["flavor"]:
             flavor["vim_info"] = {}
+        if db_nsr.get("shared-volumes"):
+            target["shared-volumes"] = deepcopy(db_nsr["shared-volumes"])
+            for shared_volumes in target["shared-volumes"]:
+                shared_volumes["vim_info"] = {}
         if db_nsr.get("affinity-or-anti-affinity-group"):
             target["affinity-or-anti-affinity-group"] = deepcopy(
                 db_nsr["affinity-or-anti-affinity-group"]
@@ -1237,6 +1246,15 @@ class NsLcm(LcmBase):
                         if target_vim not in ns_ags["vim_info"]:
                             ns_ags["vim_info"][target_vim] = {}
 
+                # shared-volumes
+                if vdur.get("shared-volumes-id"):
+                    for sv_id in vdur["shared-volumes-id"]:
+                        ns_sv = find_in_list(
+                            target["shared-volumes"], lambda sv: sv_id in sv["id"]
+                        )
+                        if ns_sv:
+                            ns_sv["vim_info"][target_vim] = {}
+
                 vdur["vim_info"] = {target_vim: {}}
                 # instantiation parameters
                 if vnf_params:
@@ -1250,6 +1268,9 @@ class NsLcm(LcmBase):
                             vdu_instantiation_params, vdud
                         )
                         vdur["additionalParams"]["OSM"]["vdu_volumes"] = vdu_volumes
+                        vdur["additionalParams"]["OSM"][
+                            "vim_flavor_id"
+                        ] = vdu_instantiation_params.get("vim-flavor-id")
                 vdur_list.append(vdur)
             target_vnf["vdur"] = vdur_list
             target["vnf"].append(target_vnf)
@@ -1315,7 +1336,7 @@ class NsLcm(LcmBase):
                 db_nsr_update["detailed-status"] = " ".join(stage)
                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
                 self._write_op_status(nslcmop_id, stage)
-            await asyncio.sleep(15, loop=self.loop)
+            await asyncio.sleep(15)
         else:  # timeout_ns_deploy
             raise NgRoException("Timeout waiting ns to deploy")
 
@@ -1501,7 +1522,7 @@ class NsLcm(LcmBase):
                         "target KDU={} is in error state".format(kdu_name)
                     )
 
-            await asyncio.sleep(10, loop=self.loop)
+            await asyncio.sleep(10)
             nb_tries += 1
         raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
 
@@ -1532,7 +1553,7 @@ class NsLcm(LcmBase):
                     "Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id)
                 )
 
-            await asyncio.sleep(10, loop=self.loop)
+            await asyncio.sleep(10)
 
             # get ip address
             if not target_vdu_id:
@@ -1818,7 +1839,7 @@ class NsLcm(LcmBase):
                     ee_id, credentials = await self.vca_map[
                         vca_type
                     ].create_execution_environment(
-                        namespace=namespace,
+                        namespace=nsr_id,
                         reuse_ee_id=ee_id,
                         db_dict=db_dict,
                         config=osm_config,
@@ -2293,9 +2314,7 @@ class NsLcm(LcmBase):
             self.logger.debug(
                 logging_text + "Invoke and wait for placement optimization"
             )
-            await self.msg.aiowrite(
-                "pla", "get_placement", {"nslcmopId": nslcmop_id}, loop=self.loop
-            )
+            await self.msg.aiowrite("pla", "get_placement", {"nslcmopId": nslcmop_id})
             db_poll_interval = 5
             wait = db_poll_interval * 10
             pla_result = None
@@ -2324,6 +2343,299 @@ class NsLcm(LcmBase):
                 vnfr["vim-account-id"] = pla_vnf["vimAccountId"]
         return modified
 
+    def _gather_vnfr_healing_alerts(self, vnfr, vnfd):
+        alerts = []
+        nsr_id = vnfr["nsr-id-ref"]
+        df = vnfd.get("df", [{}])[0]
+        # Checking for auto-healing configuration
+        if "healing-aspect" in df:
+            healing_aspects = df["healing-aspect"]
+            for healing in healing_aspects:
+                for healing_policy in healing.get("healing-policy", ()):
+                    vdu_id = healing_policy["vdu-id"]
+                    vdur = next(
+                        (vdur for vdur in vnfr["vdur"] if vdu_id == vdur["vdu-id-ref"]),
+                        {},
+                    )
+                    if not vdur:
+                        continue
+                    metric_name = "vm_status"
+                    vdu_name = vdur.get("name")
+                    vnf_member_index = vnfr["member-vnf-index-ref"]
+                    uuid = str(uuid4())
+                    name = f"healing_{uuid}"
+                    action = healing_policy
+                    # action_on_recovery = healing.get("action-on-recovery")
+                    # cooldown_time = healing.get("cooldown-time")
+                    # day1 = healing.get("day1")
+                    alert = {
+                        "uuid": uuid,
+                        "name": name,
+                        "metric": metric_name,
+                        "tags": {
+                            "ns_id": nsr_id,
+                            "vnf_member_index": vnf_member_index,
+                            "vdu_name": vdu_name,
+                        },
+                        "alarm_status": "ok",
+                        "action_type": "healing",
+                        "action": action,
+                    }
+                    alerts.append(alert)
+        return alerts
+
+    def _gather_vnfr_scaling_alerts(self, vnfr, vnfd):
+        alerts = []
+        nsr_id = vnfr["nsr-id-ref"]
+        df = vnfd.get("df", [{}])[0]
+        # Checking for auto-scaling configuration
+        if "scaling-aspect" in df:
+            scaling_aspects = df["scaling-aspect"]
+            all_vnfd_monitoring_params = {}
+            for ivld in vnfd.get("int-virtual-link-desc", ()):
+                for mp in ivld.get("monitoring-parameters", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for vdu in vnfd.get("vdu", ()):
+                for mp in vdu.get("monitoring-parameter", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for df in vnfd.get("df", ()):
+                for mp in df.get("monitoring-parameter", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for scaling_aspect in scaling_aspects:
+                scaling_group_name = scaling_aspect.get("name", "")
+                # Get monitored VDUs
+                all_monitored_vdus = set()
+                for delta in scaling_aspect.get("aspect-delta-details", {}).get(
+                    "deltas", ()
+                ):
+                    for vdu_delta in delta.get("vdu-delta", ()):
+                        all_monitored_vdus.add(vdu_delta.get("id"))
+                monitored_vdurs = list(
+                    filter(
+                        lambda vdur: vdur["vdu-id-ref"] in all_monitored_vdus,
+                        vnfr["vdur"],
+                    )
+                )
+                if not monitored_vdurs:
+                    self.logger.error(
+                        "Scaling criteria is referring to a vnf-monitoring-param that does not contain a reference to a vdu or vnf metric"
+                    )
+                    continue
+                for scaling_policy in scaling_aspect.get("scaling-policy", ()):
+                    if scaling_policy["scaling-type"] != "automatic":
+                        continue
+                    threshold_time = scaling_policy.get("threshold-time", "1")
+                    cooldown_time = scaling_policy.get("cooldown-time", "0")
+                    for scaling_criteria in scaling_policy["scaling-criteria"]:
+                        monitoring_param_ref = scaling_criteria.get(
+                            "vnf-monitoring-param-ref"
+                        )
+                        vnf_monitoring_param = all_vnfd_monitoring_params[
+                            monitoring_param_ref
+                        ]
+                        for vdur in monitored_vdurs:
+                            vdu_id = vdur["vdu-id-ref"]
+                            metric_name = vnf_monitoring_param.get("performance-metric")
+                            metric_name = f"osm_{metric_name}"
+                            vnf_member_index = vnfr["member-vnf-index-ref"]
+                            scalein_threshold = scaling_criteria.get(
+                                "scale-in-threshold"
+                            )
+                            scaleout_threshold = scaling_criteria.get(
+                                "scale-out-threshold"
+                            )
+                            # Looking for min/max-number-of-instances
+                            instances_min_number = 1
+                            instances_max_number = 1
+                            vdu_profile = df["vdu-profile"]
+                            if vdu_profile:
+                                profile = next(
+                                    item for item in vdu_profile if item["id"] == vdu_id
+                                )
+                                instances_min_number = profile.get(
+                                    "min-number-of-instances", 1
+                                )
+                                instances_max_number = profile.get(
+                                    "max-number-of-instances", 1
+                                )
+
+                            if scalein_threshold:
+                                uuid = str(uuid4())
+                                name = f"scalein_{uuid}"
+                                operation = scaling_criteria[
+                                    "scale-in-relational-operation"
+                                ]
+                                rel_operator = self.rel_operation_types.get(
+                                    operation, "<="
+                                )
+                                metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
+                                expression = f"(count ({metric_selector}) > {instances_min_number}) and (avg({metric_selector}) {rel_operator} {scalein_threshold})"
+                                labels = {
+                                    "ns_id": nsr_id,
+                                    "vnf_member_index": vnf_member_index,
+                                    "vdu_id": vdu_id,
+                                }
+                                prom_cfg = {
+                                    "alert": name,
+                                    "expr": expression,
+                                    "for": str(threshold_time) + "m",
+                                    "labels": labels,
+                                }
+                                action = scaling_policy
+                                action = {
+                                    "scaling-group": scaling_group_name,
+                                    "cooldown-time": cooldown_time,
+                                }
+                                alert = {
+                                    "uuid": uuid,
+                                    "name": name,
+                                    "metric": metric_name,
+                                    "tags": {
+                                        "ns_id": nsr_id,
+                                        "vnf_member_index": vnf_member_index,
+                                        "vdu_id": vdu_id,
+                                    },
+                                    "alarm_status": "ok",
+                                    "action_type": "scale_in",
+                                    "action": action,
+                                    "prometheus_config": prom_cfg,
+                                }
+                                alerts.append(alert)
+
+                            if scaleout_threshold:
+                                uuid = str(uuid4())
+                                name = f"scaleout_{uuid}"
+                                operation = scaling_criteria[
+                                    "scale-out-relational-operation"
+                                ]
+                                rel_operator = self.rel_operation_types.get(
+                                    operation, "<="
+                                )
+                                metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
+                                expression = f"(count ({metric_selector}) < {instances_max_number}) and (avg({metric_selector}) {rel_operator} {scaleout_threshold})"
+                                labels = {
+                                    "ns_id": nsr_id,
+                                    "vnf_member_index": vnf_member_index,
+                                    "vdu_id": vdu_id,
+                                }
+                                prom_cfg = {
+                                    "alert": name,
+                                    "expr": expression,
+                                    "for": str(threshold_time) + "m",
+                                    "labels": labels,
+                                }
+                                action = scaling_policy
+                                action = {
+                                    "scaling-group": scaling_group_name,
+                                    "cooldown-time": cooldown_time,
+                                }
+                                alert = {
+                                    "uuid": uuid,
+                                    "name": name,
+                                    "metric": metric_name,
+                                    "tags": {
+                                        "ns_id": nsr_id,
+                                        "vnf_member_index": vnf_member_index,
+                                        "vdu_id": vdu_id,
+                                    },
+                                    "alarm_status": "ok",
+                                    "action_type": "scale_out",
+                                    "action": action,
+                                    "prometheus_config": prom_cfg,
+                                }
+                                alerts.append(alert)
+        return alerts
+
+    def _gather_vnfr_alarm_alerts(self, vnfr, vnfd):
+        alerts = []
+        nsr_id = vnfr["nsr-id-ref"]
+        vnf_member_index = vnfr["member-vnf-index-ref"]
+
+        # Checking for VNF alarm configuration
+        for vdur in vnfr["vdur"]:
+            vdu_id = vdur["vdu-id-ref"]
+            vdu = next(filter(lambda vdu: vdu["id"] == vdu_id, vnfd["vdu"]))
+            if "alarm" in vdu:
+                # Get VDU monitoring params, since alerts are based on them
+                vdu_monitoring_params = {}
+                for mp in vdu.get("monitoring-parameter", []):
+                    vdu_monitoring_params[mp.get("id")] = mp
+                if not vdu_monitoring_params:
+                    self.logger.error(
+                        "VDU alarm refers to a VDU monitoring param, but there are no VDU monitoring params in the VDU"
+                    )
+                    continue
+                # Get alarms in the VDU
+                alarm_descriptors = vdu["alarm"]
+                # Create VDU alarms for each alarm in the VDU
+                for alarm_descriptor in alarm_descriptors:
+                    # Check that the VDU alarm refers to a proper monitoring param
+                    alarm_monitoring_param = alarm_descriptor.get(
+                        "vnf-monitoring-param-ref", ""
+                    )
+                    vdu_specific_monitoring_param = vdu_monitoring_params.get(
+                        alarm_monitoring_param, {}
+                    )
+                    if not vdu_specific_monitoring_param:
+                        self.logger.error(
+                            "VDU alarm refers to a VDU monitoring param not present in the VDU"
+                        )
+                        continue
+                    metric_name = vdu_specific_monitoring_param.get(
+                        "performance-metric"
+                    )
+                    if not metric_name:
+                        self.logger.error(
+                            "VDU alarm refers to a VDU monitoring param that has no associated performance-metric"
+                        )
+                        continue
+                    # Set params of the alarm to be created in Prometheus
+                    metric_name = f"osm_{metric_name}"
+                    metric_threshold = alarm_descriptor.get("value")
+                    uuid = str(uuid4())
+                    alert_name = f"vdu_alarm_{uuid}"
+                    operation = alarm_descriptor["operation"]
+                    rel_operator = self.rel_operation_types.get(operation, "<=")
+                    metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
+                    expression = f"{metric_selector} {rel_operator} {metric_threshold}"
+                    labels = {
+                        "ns_id": nsr_id,
+                        "vnf_member_index": vnf_member_index,
+                        "vdu_id": vdu_id,
+                        "vdu_name": "{{ $labels.vdu_name }}",
+                    }
+                    prom_cfg = {
+                        "alert": alert_name,
+                        "expr": expression,
+                        "for": "1m",  # default value. Ideally, this should be related to an IM param, but there is not such param
+                        "labels": labels,
+                    }
+                    alarm_action = dict()
+                    for action_type in ["ok", "insufficient-data", "alarm"]:
+                        if (
+                            "actions" in alarm_descriptor
+                            and action_type in alarm_descriptor["actions"]
+                        ):
+                            alarm_action[action_type] = alarm_descriptor["actions"][
+                                action_type
+                            ]
+                    alert = {
+                        "uuid": uuid,
+                        "name": alert_name,
+                        "metric": metric_name,
+                        "tags": {
+                            "ns_id": nsr_id,
+                            "vnf_member_index": vnf_member_index,
+                            "vdu_id": vdu_id,
+                        },
+                        "alarm_status": "ok",
+                        "action_type": "vdu_alarm",
+                        "action": alarm_action,
+                        "prometheus_config": prom_cfg,
+                    }
+                    alerts.append(alert)
+        return alerts
+
     def update_nsrs_with_pla_result(self, params):
         try:
             nslcmop_id = deep_get(params, ("placement", "nslcmopId"))
@@ -2533,13 +2845,16 @@ class NsLcm(LcmBase):
 
             # create namespace and certificate if any helm based EE is present in the NS
             if check_helm_ee_in_ns(db_vnfds):
-                # TODO: create EE namespace
+                await self.vca_map["helm-v3"].setup_ns_namespace(
+                    name=nsr_id,
+                )
                 # create TLS certificates
                 await self.vca_map["helm-v3"].create_tls_certificate(
-                    secret_name="ee-tls-{}".format(nsr_id),
+                    secret_name=self.EE_TLS_NAME,
                     dns_prefix="*",
                     nsr_id=nsr_id,
                     usage="server auth",
+                    namespace=nsr_id,
                 )
 
             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
@@ -2675,6 +2990,65 @@ class NsLcm(LcmBase):
                             stage=stage,
                         )
 
+            # Check if each vnf has exporter for metric collection if so update prometheus job records
+            if "exporters-endpoints" in vnfd.get("df")[0]:
+                exporter_config = vnfd.get("df")[0].get("exporters-endpoints")
+                self.logger.debug("exporter config :{}".format(exporter_config))
+                artifact_path = "{}/{}/{}".format(
+                    base_folder["folder"],
+                    base_folder["pkg-dir"],
+                    "exporter-endpoint",
+                )
+                ee_id = None
+                ee_config_descriptor = exporter_config
+                vnfr_id = db_vnfr["id"]
+                rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
+                    logging_text,
+                    nsr_id,
+                    vnfr_id,
+                    vdu_id=None,
+                    vdu_index=None,
+                    user=None,
+                    pub_key=None,
+                )
+                self.logger.debug("rw_mgmt_ip:{}".format(rw_mgmt_ip))
+                self.logger.debug("Artifact_path:{}".format(artifact_path))
+                db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
+                vdu_id_for_prom = None
+                vdu_index_for_prom = None
+                for x in get_iterable(db_vnfr, "vdur"):
+                    vdu_id_for_prom = x.get("vdu-id-ref")
+                    vdu_index_for_prom = x.get("count-index")
+                prometheus_jobs = await self.extract_prometheus_scrape_jobs(
+                    ee_id=ee_id,
+                    artifact_path=artifact_path,
+                    ee_config_descriptor=ee_config_descriptor,
+                    vnfr_id=vnfr_id,
+                    nsr_id=nsr_id,
+                    target_ip=rw_mgmt_ip,
+                    element_type="VDU",
+                    vdu_id=vdu_id_for_prom,
+                    vdu_index=vdu_index_for_prom,
+                )
+
+                self.logger.debug("Prometheus job:{}".format(prometheus_jobs))
+                if prometheus_jobs:
+                    db_nsr_update["_admin.deployed.prometheus_jobs"] = prometheus_jobs
+                    self.update_db_2(
+                        "nsrs",
+                        nsr_id,
+                        db_nsr_update,
+                    )
+
+                    for job in prometheus_jobs:
+                        self.db.set_one(
+                            "prometheus_jobs",
+                            {"job_name": job["job_name"]},
+                            job,
+                            upsert=True,
+                            fail_on_empty=False,
+                        )
+
             # Check if this NS has a charm configuration
             descriptor_config = nsd.get("ns-configuration")
             if descriptor_config and descriptor_config.get("juju"):
@@ -2798,7 +3172,27 @@ class NsLcm(LcmBase):
                 db_nsr_update["detailed-status"] = "Done"
                 db_nslcmop_update["detailed-status"] = "Done"
                 nslcmop_operation_state = "COMPLETED"
-
+                # Gather auto-healing and auto-scaling alerts for each vnfr
+                healing_alerts = []
+                scaling_alerts = []
+                for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
+                    vnfd = next(
+                        (sub for sub in db_vnfds if sub["_id"] == vnfr["vnfd-id"]), None
+                    )
+                    healing_alerts = self._gather_vnfr_healing_alerts(vnfr, vnfd)
+                    for alert in healing_alerts:
+                        self.logger.info(f"Storing healing alert in MongoDB: {alert}")
+                        self.db.create("alerts", alert)
+
+                    scaling_alerts = self._gather_vnfr_scaling_alerts(vnfr, vnfd)
+                    for alert in scaling_alerts:
+                        self.logger.info(f"Storing scaling alert in MongoDB: {alert}")
+                        self.db.create("alerts", alert)
+
+                    alarm_alerts = self._gather_vnfr_alarm_alerts(vnfr, vnfd)
+                    for alert in alarm_alerts:
+                        self.logger.info(f"Storing VNF alarm alert in MongoDB: {alert}")
+                        self.db.create("alerts", alert)
             if db_nsr:
                 self._write_ns_status(
                     nsr_id=nsr_id,
@@ -2826,8 +3220,13 @@ class NsLcm(LcmBase):
                             "nsr_id": nsr_id,
                             "nslcmop_id": nslcmop_id,
                             "operationState": nslcmop_operation_state,
+                            "startTime": db_nslcmop["startTime"],
+                            "links": db_nslcmop["links"],
+                            "operationParams": {
+                                "nsInstanceId": nsr_id,
+                                "nsdId": db_nsr["nsd-id"],
+                            },
                         },
-                        loop=self.loop,
                     )
                 except Exception as e:
                     self.logger.error(
@@ -4073,7 +4472,7 @@ class NsLcm(LcmBase):
         # TODO vdu_index_count
         for vca in vca_deployed_list:
             if vca["member-vnf-index"] == vnf_index and vca["vdu_id"] == vdu_id:
-                return vca["ee_id"]
+                return vca.get("ee_id")
 
     async def destroy_N2VC(
         self,
@@ -4355,9 +4754,12 @@ class NsLcm(LcmBase):
             # Delete Namespace and Certificates if necessary
             if check_helm_ee_in_ns(list(db_vnfds_from_member_index.values())):
                 await self.vca_map["helm-v3"].delete_tls_certificate(
-                    certificate_name=db_nslcmop["nsInstanceId"],
+                    namespace=db_nslcmop["nsInstanceId"],
+                    certificate_name=self.EE_TLS_NAME,
+                )
+                await self.vca_map["helm-v3"].delete_namespace(
+                    namespace=db_nslcmop["nsInstanceId"],
                 )
-                # TODO: Delete namespace
 
             # Delete from k8scluster
             stage[1] = "Deleting KDUs."
@@ -4514,12 +4916,13 @@ class NsLcm(LcmBase):
                             "operationState": nslcmop_operation_state,
                             "autoremove": autoremove,
                         },
-                        loop=self.loop,
                     )
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)
                     )
+                self.logger.debug(f"Deleting alerts: ns_id={nsr_id}")
+                self.db.del_list("alerts", {"tags.ns_id": nsr_id})
 
             self.logger.debug(logging_text + "Exit")
             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
@@ -4770,7 +5173,7 @@ class NsLcm(LcmBase):
                             )
                         )
                         # wait and retry
-                        await asyncio.sleep(retries_interval, loop=self.loop)
+                        await asyncio.sleep(retries_interval)
                     else:
                         if isinstance(e, asyncio.TimeoutError):
                             e = N2VCException(
@@ -5010,9 +5413,10 @@ class NsLcm(LcmBase):
                         del desc_params["kdu_model"]
                     else:
                         kdu_model = kdu.get("kdu-model")
-                        parts = kdu_model.split(sep=":")
-                        if len(parts) == 2:
-                            kdu_model = parts[0]
+                        if kdu_model.count("/") < 2:  # helm chart is not embedded
+                            parts = kdu_model.split(sep=":")
+                            if len(parts) == 2:
+                                kdu_model = parts[0]
                     if desc_params.get("kdu_atomic_upgrade"):
                         atomic_upgrade = desc_params.get(
                             "kdu_atomic_upgrade"
@@ -5181,7 +5585,6 @@ class NsLcm(LcmBase):
                             "nslcmop_id": nslcmop_id,
                             "operationState": nslcmop_operation_state,
                         },
-                        loop=self.loop,
                     )
                 except Exception as e:
                     self.logger.error(
@@ -6012,7 +6415,7 @@ class NsLcm(LcmBase):
                         and member_vnf_index
                     ):
                         msg.update({"vnf_member_index": member_vnf_index})
-                    await self.msg.aiowrite("ns", change_type, msg, loop=self.loop)
+                    await self.msg.aiowrite("ns", change_type, msg)
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)
@@ -7045,7 +7448,7 @@ class NsLcm(LcmBase):
                         "nslcmop_id": nslcmop_id,
                         "operationState": nslcmop_operation_state,
                     }
-                    await self.msg.aiowrite("ns", "scaled", msg, loop=self.loop)
+                    await self.msg.aiowrite("ns", "scaled", msg)
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)
@@ -7265,6 +7668,8 @@ class NsLcm(LcmBase):
         )
         if not job_file:
             return
+        self.logger.debug("Artifact path{}".format(artifact_path))
+        self.logger.debug("job file{}".format(job_file))
         with self.fs.file_open((artifact_path, job_file), "r") as f:
             job_data = f.read()
 
@@ -7303,7 +7708,7 @@ class NsLcm(LcmBase):
                         kdur_name = kdur.get("name")
                         break
 
-                await asyncio.sleep(10, loop=self.loop)
+                await asyncio.sleep(10)
             else:
                 if vdu_id and vdu_index is not None:
                     raise LcmException(
@@ -7314,22 +7719,35 @@ class NsLcm(LcmBase):
                         f"Timeout waiting KDU with name={kdu_name} and index={kdu_index} to be intantiated"
                     )
 
-        # TODO get_service
-        _, _, service = ee_id.partition(".")  # remove prefix   "namespace."
-        host_name = "{}-{}".format(service, ee_config_descriptor["metric-service"])
-        host_port = "80"
-        vnfr_id = vnfr_id.replace("-", "")
-        variables = {
-            "JOB_NAME": vnfr_id,
-            "TARGET_IP": target_ip,
-            "EXPORTER_POD_IP": host_name,
-            "EXPORTER_POD_PORT": host_port,
-            "NSR_ID": nsr_id,
-            "VNF_MEMBER_INDEX": vnf_member_index,
-            "VDUR_NAME": vdur_name,
-            "KDUR_NAME": kdur_name,
-            "ELEMENT_TYPE": element_type,
-        }
+        if ee_id is not None:
+            _, namespace, helm_id = get_ee_id_parts(
+                ee_id
+            )  # get namespace and EE gRPC service name
+            host_name = f'{helm_id}-{ee_config_descriptor["metric-service"]}.{namespace}.svc'  # svc_name.namespace.svc
+            host_port = "80"
+            vnfr_id = vnfr_id.replace("-", "")
+            variables = {
+                "JOB_NAME": vnfr_id,
+                "TARGET_IP": target_ip,
+                "EXPORTER_POD_IP": host_name,
+                "EXPORTER_POD_PORT": host_port,
+                "NSR_ID": nsr_id,
+                "VNF_MEMBER_INDEX": vnf_member_index,
+                "VDUR_NAME": vdur_name,
+                "KDUR_NAME": kdur_name,
+                "ELEMENT_TYPE": element_type,
+            }
+        else:
+            metric_path = ee_config_descriptor["metric-path"]
+            target_port = ee_config_descriptor["metric-port"]
+            vnfr_id = vnfr_id.replace("-", "")
+            variables = {
+                "JOB_NAME": vnfr_id,
+                "TARGET_IP": target_ip,
+                "TARGET_PORT": target_port,
+                "METRIC_PATH": metric_path,
+            }
+
         job_list = parse_job(job_data, variables)
         # ensure job_name is using the vnfr_id. Adding the metadata nsr_id
         for job in job_list:
@@ -7337,7 +7755,7 @@ class NsLcm(LcmBase):
                 not isinstance(job.get("job_name"), str)
                 or vnfr_id not in job["job_name"]
             ):
-                job["job_name"] = vnfr_id + "_" + str(randint(1, 10000))
+                job["job_name"] = vnfr_id + "_" + str(SystemRandom().randint(1, 10000))
             job["nsr_id"] = nsr_id
             job["vnfr_id"] = vnfr_id
         return job_list
@@ -7546,7 +7964,7 @@ class NsLcm(LcmBase):
                         "nslcmop_id": nslcmop_id,
                         "operationState": nslcmop_operation_state,
                     }
-                    await self.msg.aiowrite("ns", "migrated", msg, loop=self.loop)
+                    await self.msg.aiowrite("ns", "migrated", msg)
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)
@@ -7670,9 +8088,9 @@ class NsLcm(LcmBase):
                     for target_vdu in target_vdu_list:
                         deploy_params_vdu = target_vdu
                         # Set run-day1 vnf level value if not vdu level value exists
-                        if not deploy_params_vdu.get("run-day1") and target_vnf[
-                            "additionalParams"
-                        ].get("run-day1"):
+                        if not deploy_params_vdu.get("run-day1") and target_vnf.get(
+                            "additionalParams", {}
+                        ).get("run-day1"):
                             deploy_params_vdu["run-day1"] = target_vnf[
                                 "additionalParams"
                             ].get("run-day1")
@@ -7832,7 +8250,7 @@ class NsLcm(LcmBase):
                         "nslcmop_id": nslcmop_id,
                         "operationState": nslcmop_operation_state,
                     }
-                    await self.msg.aiowrite("ns", "healed", msg, loop=self.loop)
+                    await self.msg.aiowrite("ns", "healed", msg)
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)
@@ -8513,7 +8931,7 @@ class NsLcm(LcmBase):
             self.logger.debug("Wait Heal RO > {}".format(operational_status_ro))
             if operational_status_ro != "healing":
                 break
-            await asyncio.sleep(15, loop=self.loop)
+            await asyncio.sleep(15)
         else:  # timeout_ns_deploy
             raise NgRoException("Timeout waiting ns to deploy")
 
@@ -8611,7 +9029,7 @@ class NsLcm(LcmBase):
                         "nslcmop_id": nslcmop_id,
                         "operationState": nslcmop_operation_state,
                     }
-                    await self.msg.aiowrite("ns", "verticalscaled", msg, loop=self.loop)
+                    await self.msg.aiowrite("ns", "verticalscaled", msg)
                 except Exception as e:
                     self.logger.error(
                         logging_text + "kafka_write notification Exception {}".format(e)