Feature 8720: Add scale support for charms

[osm/LCM.git] / osm_lcm / ns.py

diff --git a/osm_lcm/ns.py b/osm_lcm/ns.py
index bcd1af9..9514768 100644 (file)
--- a/osm_lcm/ns.py
+++ b/osm_lcm/ns.py
@@ -33,12 +33,13 @@ from osm_common.dbbase import DbException
 from osm_common.fsbase import FsException
 
 from n2vc.n2vc_juju_conn import N2VCJujuConnector
 from osm_common.fsbase import FsException
 
 from n2vc.n2vc_juju_conn import N2VCJujuConnector

-from n2vc.exceptions import N2VCException
+from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException

 
 from copy import copy, deepcopy
 from http import HTTPStatus
 from time import time
 from uuid import uuid4
 
 from copy import copy, deepcopy
 from http import HTTPStatus
 from time import time
 from uuid import uuid4

+from functools import partial

 
 __author__ = "Alfonso Tierno"
 
 
 __author__ = "Alfonso Tierno"
 


@@ -48,8 +49,8 @@ class NsLcm(LcmBase):
     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
     timeout_charm_delete = 10 * 60
     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
     timeout_charm_delete = 10 * 60

-    timeout_primitive = 10 * 60  # timeout for primitive execution
-    timeout_progress_primitive = 2 * 60  # timeout for some progress in a primitive execution
+    timeout_primitive = 30 * 60  # timeout for primitive execution
+    timeout_progress_primitive = 10 * 60  # timeout for some progress in a primitive execution

 
     SUBOPERATION_STATUS_NOT_FOUND = -1
     SUBOPERATION_STATUS_NEW = -2
 
     SUBOPERATION_STATUS_NOT_FOUND = -1
     SUBOPERATION_STATUS_NEW = -2


@@ -277,10 +278,12 @@ class NsLcm(LcmBase):
             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
                                format(vnfd["id"], vdu["id"], e))
 
             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
                                format(vnfd["id"], vdu["id"], e))
 

-    def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
+    def _ns_params_2_RO(self, ns_params, nsd, vnfd_dict, db_vnfrs, n2vc_key_list):

         """
         Creates a RO ns descriptor from OSM ns_instantiate params
         :param ns_params: OSM instantiate params
         """
         Creates a RO ns descriptor from OSM ns_instantiate params
         :param ns_params: OSM instantiate params

+        :param vnfd_dict: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
+        :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index. {member-vnf-index: {vnfr_object}, ...}

         :return: The RO ns descriptor
         """
         vim_2_RO = {}
         :return: The RO ns descriptor
         """
         vim_2_RO = {}


@@ -342,6 +345,13 @@ class NsLcm(LcmBase):
             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
             # "scenario": ns_params["nsdId"],
         }
             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
             # "scenario": ns_params["nsdId"],
         }

+        # set vim_account of each vnf if different from general vim_account.
+        # Get this information from <vnfr> database content, key vim-account-id
+        # Vim account can be set by placement_engine and it may be different from
+        # the instantiate parameters (vnfs.member-vnf-index.datacenter).
+        for vnf_index, vnfr in db_vnfrs.items():
+            if vnfr.get("vim-account-id") and vnfr["vim-account-id"] != ns_params["vimAccountId"]:
+                populate_dict(RO_ns_params, ("vnfs", vnf_index, "datacenter"), vim_account_2_RO(vnfr["vim-account-id"]))

 
         n2vc_key_list = n2vc_key_list or []
         for vnfd_ref, vnfd in vnfd_dict.items():
 
         n2vc_key_list = n2vc_key_list or []
         for vnfd_ref, vnfd in vnfd_dict.items():


@@ -390,9 +400,6 @@ class NsLcm(LcmBase):
             else:
                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
             else:
                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))

-            if vnf_params.get("vimAccountId"):
-                populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
-                              vim_account_2_RO(vnf_params["vimAccountId"]))

 
             for vdu_params in get_iterable(vnf_params, "vdu"):
                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 
             for vdu_params in get_iterable(vnf_params, "vdu"):
                 # TODO feature 1417: check that this VDU exist and it is not a PDU


@@ -751,6 +758,19 @@ class NsLcm(LcmBase):
 
     async def instantiate_RO(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
                              n2vc_key_list, stage):
 
     async def instantiate_RO(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
                              n2vc_key_list, stage):

+        """
+        Instantiate at RO
+        :param logging_text: preffix text to use at logging
+        :param nsr_id: nsr identity
+        :param nsd: database content of ns descriptor
+        :param db_nsr: database content of ns record
+        :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
+        :param db_vnfrs:
+        :param db_vnfds_ref: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
+        :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
+        :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
+        :return: None or exception
+        """

         try:
             db_nsr_update = {}
             RO_descriptor_number = 0   # number of descriptors created at RO
         try:
             db_nsr_update = {}
             RO_descriptor_number = 0   # number of descriptors created at RO


@@ -765,10 +785,15 @@ class NsLcm(LcmBase):
 
             # Check for and optionally request placement optimization. Database will be updated if placement activated
             stage[2] = "Waiting for Placement."
 
             # Check for and optionally request placement optimization. Database will be updated if placement activated
             stage[2] = "Waiting for Placement."

-            await self.do_placement(logging_text, db_nslcmop, db_vnfrs)
+            if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
+                # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
+                for vnfr in db_vnfrs.values():
+                    if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
+                        break
+                else:
+                    ns_params["vimAccountId"] == vnfr["vim-account-id"]

 
             # deploy RO
 
             # deploy RO

-

             # get vnfds, instantiate at RO
             for c_vnf in nsd.get("constituent-vnfd", ()):
                 member_vnf_index = c_vnf["member-vnf-index"]
             # get vnfds, instantiate at RO
             for c_vnf in nsd.get("constituent-vnfd", ()):
                 member_vnf_index = c_vnf["member-vnf-index"]


@@ -898,7 +923,7 @@ class NsLcm(LcmBase):
                             await asyncio.wait(task_dependency, timeout=3600)
 
                 stage[2] = "Checking instantiation parameters."
                             await asyncio.wait(task_dependency, timeout=3600)
 
                 stage[2] = "Checking instantiation parameters."

-                RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
+                RO_ns_params = self._ns_params_2_RO(ns_params, nsd, db_vnfds_ref, db_vnfrs, n2vc_key_list)

                 stage[2] = "Deploying ns at VIM."
                 db_nsr_update["detailed-status"] = " ".join(stage)
                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
                 stage[2] = "Deploying ns at VIM."
                 db_nsr_update["detailed-status"] = " ".join(stage)
                 self.update_db_2("nsrs", nsr_id, db_nsr_update)


@@ -1143,7 +1168,6 @@ class NsLcm(LcmBase):
                     element_under_configuration = kdu_name
 
             # Get artifact path
                     element_under_configuration = kdu_name
 
             # Get artifact path

-            self.fs.sync()  # Sync from FSMongo

             artifact_path = "{}/{}/charms/{}".format(
                 base_folder["folder"],
                 base_folder["pkg-dir"],
             artifact_path = "{}/{}/charms/{}".format(
                 base_folder["folder"],
                 base_folder["pkg-dir"],


@@ -1247,11 +1271,24 @@ class NsLcm(LcmBase):
                                 deploy_params
                             )
                             break
                                 deploy_params
                             )
                             break

+            num_units = 1
+            if is_proxy_charm:
+                if element_type == "NS":
+                    num_units = db_nsr.get("config-units") or 1
+                elif element_type == "VNF":
+                    num_units = db_vnfr.get("config-units") or 1
+                elif element_type == "VDU":
+                    for v in db_vnfr["vdur"]:
+                        if vdu_id == v["vdu-id-ref"]:
+                            num_units = v.get("config-units") or 1
+                            break
+

             await self.n2vc.install_configuration_sw(
                 ee_id=ee_id,
                 artifact_path=artifact_path,
                 db_dict=db_dict,
             await self.n2vc.install_configuration_sw(
                 ee_id=ee_id,
                 artifact_path=artifact_path,
                 db_dict=db_dict,

-                config=config
+                config=config,
+                num_units=num_units

             )
 
             # write in db flag of configuration_sw already installed
             )
 
             # write in db flag of configuration_sw already installed


@@ -1460,29 +1497,44 @@ class NsLcm(LcmBase):
             self.logger.warn('Error writing configuration status={}, ns={}, vca_index={}: {}'
                              .format(status, nsr_id, vca_index, e))
 
             self.logger.warn('Error writing configuration status={}, ns={}, vca_index={}: {}'
                              .format(status, nsr_id, vca_index, e))
 

-    async def do_placement(self, logging_text, db_nslcmop, db_vnfrs):
+    async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
+        """
+        Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
+        sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
+        Database is used because the result can be obtained from a different LCM worker in case of HA.
+        :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
+        :param db_nslcmop: database content of nslcmop
+        :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
+        :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
+            computed 'vim-account-id'
+        """
+        modified = False
+        nslcmop_id = db_nslcmop['_id']

         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
         if placement_engine == "PLA":
         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
         if placement_engine == "PLA":

-            self.logger.debug(logging_text + "Invoke placement optimization for nslcmopId={}".format(db_nslcmop['id']))
-            await self.msg.aiowrite("pla", "get_placement", {'nslcmopId': db_nslcmop['_id']}, loop=self.loop)
+            self.logger.debug(logging_text + "Invoke and wait for placement optimization")
+            await self.msg.aiowrite("pla", "get_placement", {'nslcmopId': nslcmop_id}, loop=self.loop)

             db_poll_interval = 5
             db_poll_interval = 5

-            wait = db_poll_interval * 4
+            wait = db_poll_interval * 10

             pla_result = None
             while not pla_result and wait >= 0:
                 await asyncio.sleep(db_poll_interval)
                 wait -= db_poll_interval
             pla_result = None
             while not pla_result and wait >= 0:
                 await asyncio.sleep(db_poll_interval)
                 wait -= db_poll_interval

-                db_nslcmop = self.db.get_one("nslcmops", {"_id": db_nslcmop["_id"]})
+                db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})

                 pla_result = deep_get(db_nslcmop, ('_admin', 'pla'))
 
             if not pla_result:
                 pla_result = deep_get(db_nslcmop, ('_admin', 'pla'))
 
             if not pla_result:

-                raise LcmException("Placement timeout for nslcmopId={}".format(db_nslcmop['id']))
+                raise LcmException("Placement timeout for nslcmopId={}".format(nslcmop_id))

 
             for pla_vnf in pla_result['vnf']:
                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
                 if not pla_vnf.get('vimAccountId') or not vnfr:
                     continue
 
             for pla_vnf in pla_result['vnf']:
                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
                 if not pla_vnf.get('vimAccountId') or not vnfr:
                     continue

+                modified = True

                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})
                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})

-        return
+                # Modifies db_vnfrs
+                vnfr["vim-account-id"] = pla_vnf['vimAccountId']
+        return modified

 
     def update_nsrs_with_pla_result(self, params):
         try:
 
     def update_nsrs_with_pla_result(self, params):
         try:


@@ -1508,6 +1560,9 @@ class NsLcm(LcmBase):
         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
         self.logger.debug(logging_text + "Enter")
 
         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
         self.logger.debug(logging_text + "Enter")
 

+        # Sync from FSMongo
+        self.fs.sync()
+

         # get all needed from database
 
         # database nsrs record
         # get all needed from database
 
         # database nsrs record


@@ -1563,8 +1618,9 @@ class NsLcm(LcmBase):
             # read from db: ns
             stage[1] = "Getting nsr={} from db".format(nsr_id)
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
             # read from db: ns
             stage[1] = "Getting nsr={} from db".format(nsr_id)
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})

-            # nsd is replicated into ns (no db read)
-            nsd = db_nsr["nsd"]
+            stage[1] = "Getting nsd={} from db".format(db_nsr["nsd-id"])
+            nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
+            db_nsr["nsd"] = nsd

             # nsr_name = db_nsr["name"]   # TODO short-name??
 
             # read from db: vnf's of this ns
             # nsr_name = db_nsr["name"]   # TODO short-name??
 
             # read from db: vnf's of this ns


@@ -1892,14 +1948,13 @@ class NsLcm(LcmBase):
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )

-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )

 
             if nslcmop_operation_state:
                 try:
 
             if nslcmop_operation_state:
                 try:


@@ -1925,13 +1980,14 @@ class NsLcm(LcmBase):
 
             # read nsr record
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 
             # read nsr record
             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})

+            nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})

 
             # this VCA data
             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
 
             # read all ns-configuration relations
             ns_relations = list()
 
             # this VCA data
             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
 
             # read all ns-configuration relations
             ns_relations = list()

-            db_ns_relations = deep_get(db_nsr, ('nsd', 'ns-configuration', 'relation'))
+            db_ns_relations = deep_get(nsd, ('ns-configuration', 'relation'))

             if db_ns_relations:
                 for r in db_ns_relations:
                     # check if this VCA is in the relation
             if db_ns_relations:
                 for r in db_ns_relations:
                     # check if this VCA is in the relation


@@ -2072,6 +2128,25 @@ class NsLcm(LcmBase):
             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
             return False
 
             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
             return False
 

+    def _write_db_callback(self, task, item, _id, on_done=None, on_exc=None):
+        """
+        callback for kdu install intended to store the returned kdu_instance at database
+        :return: None
+        """
+        db_update = {}
+        try:
+            result = task.result()
+            if on_done:
+                db_update[on_done] = str(result)
+        except Exception as e:
+            if on_exc:
+                db_update[on_exc] = str(e)
+        if db_update:
+            try:
+                self.update_db_2(item, _id, db_update)
+            except Exception:
+                pass
+

     async def deploy_kdus(self, logging_text, nsr_id, nslcmop_id, db_vnfrs, db_vnfds, task_instantiation_info):
         # Launch kdus if present in the descriptor
 
     async def deploy_kdus(self, logging_text, nsr_id, nslcmop_id, db_vnfrs, db_vnfds, task_instantiation_info):
         # Launch kdus if present in the descriptor
 


@@ -2104,6 +2179,7 @@ class NsLcm(LcmBase):
                 for kdur in get_iterable(vnfr_data, "kdur"):
                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
                     vnfd_id = vnfr_data.get('vnfd-id')
                 for kdur in get_iterable(vnfr_data, "kdur"):
                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
                     vnfd_id = vnfr_data.get('vnfd-id')

+                    namespace = kdur.get("k8s-namespace")

                     if kdur.get("helm-chart"):
                         kdumodel = kdur["helm-chart"]
                         k8sclustertype = "helm-chart"
                     if kdur.get("helm-chart"):
                         kdumodel = kdur["helm-chart"]
                         k8sclustertype = "helm-chart"


@@ -2119,7 +2195,7 @@ class NsLcm(LcmBase):
                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel

-                            filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["'pkg-dir"], k8sclustertype,
+                            filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["pkg-dir"], k8sclustertype,

                                                              kdumodel)
                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
                                 kdumodel = self.fs.path + filename
                                                              kdumodel)
                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
                                 kdumodel = self.fs.path + filename


@@ -2153,20 +2229,25 @@ class NsLcm(LcmBase):
                                         "k8scluster-type": k8sclustertype,
                                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
                                         "kdu-name": kdur["kdu-name"],
                                         "k8scluster-type": k8sclustertype,
                                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
                                         "kdu-name": kdur["kdu-name"],

-                                        "kdu-model": kdumodel}
-                    db_nsr_update["_admin.deployed.K8s.{}".format(index)] = k8s_instace_info
+                                        "kdu-model": kdumodel,
+                                        "namespace": namespace}
+                    db_path = "_admin.deployed.K8s.{}".format(index)
+                    db_nsr_update[db_path] = k8s_instace_info

                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 
                     db_dict = {"collection": "nsrs",
                                "filter": {"_id": nsr_id},
                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 
                     db_dict = {"collection": "nsrs",
                                "filter": {"_id": nsr_id},

-                               "path": "_admin.deployed.K8s.{}".format(index)}
+                               "path": db_path}

 
                     task = asyncio.ensure_future(
                         self.k8scluster_map[k8sclustertype].install(cluster_uuid=cluster_uuid, kdu_model=kdumodel,
                                                                     atomic=True, params=desc_params,
                                                                     db_dict=db_dict, timeout=600,
 
                     task = asyncio.ensure_future(
                         self.k8scluster_map[k8sclustertype].install(cluster_uuid=cluster_uuid, kdu_model=kdumodel,
                                                                     atomic=True, params=desc_params,
                                                                     db_dict=db_dict, timeout=600,

-                                                                    kdu_name=kdur["kdu-name"]))
+                                                                    kdu_name=kdur["kdu-name"], namespace=namespace))

 
 

+                    task.add_done_callback(partial(self._write_db_callback, item="nsrs", _id=nsr_id,
+                                                   on_done=db_path + ".kdu-instance",
+                                                   on_exc=db_path + ".detailed-status"))

                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
 
                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
 


@@ -2327,9 +2408,9 @@ class NsLcm(LcmBase):
             # _ns_execute_primitive() or RO.create_action() will NOT be executed
             return self.SUBOPERATION_STATUS_SKIP
         else:
             # _ns_execute_primitive() or RO.create_action() will NOT be executed
             return self.SUBOPERATION_STATUS_SKIP
         else:

-            # c. Reintent executing sub-operation
+            # c. retry executing sub-operation

             # The sub-operation exists, and operationState != 'COMPLETED'
             # The sub-operation exists, and operationState != 'COMPLETED'

-            # Update operationState = 'PROCESSING' to indicate a reintent.
+            # Update operationState = 'PROCESSING' to indicate a retry.

             operationState = 'PROCESSING'
             detailed_status = 'In progress'
             self._update_suboperation_status(
             operationState = 'PROCESSING'
             detailed_status = 'In progress'
             self._update_suboperation_status(


@@ -2342,7 +2423,7 @@ class NsLcm(LcmBase):
     # Find a sub-operation where all keys in a matching dictionary must match
     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
     def _find_suboperation(self, db_nslcmop, match):
     # Find a sub-operation where all keys in a matching dictionary must match
     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
     def _find_suboperation(self, db_nslcmop, match):

-        if (db_nslcmop and match):
+        if db_nslcmop and match:

             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
             for i, op in enumerate(op_list):
                 if all(op.get(k) == match[k] for k in match):
             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
             for i, op in enumerate(op_list):
                 if all(op.get(k) == match[k] for k in match):


@@ -2410,11 +2491,11 @@ class NsLcm(LcmBase):
     # Check for 3 different cases:
     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
     # Check for 3 different cases:
     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP

-    # c. Reintent: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
+    # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute

     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
         # Find this sub-operation
     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
         # Find this sub-operation

-        if (RO_nsr_id and RO_scaling_info):
+        if RO_nsr_id and RO_scaling_info:

             operationType = 'SCALE-RO'
             match = {
                 'member_vnf_index': vnf_index,
             operationType = 'SCALE-RO'
             match = {
                 'member_vnf_index': vnf_index,


@@ -2533,7 +2614,10 @@ class NsLcm(LcmBase):
     async def _delete_all_N2VC(self, db_nsr: dict):
         self._write_all_config_status(db_nsr=db_nsr, status='TERMINATING')
         namespace = "." + db_nsr["_id"]
     async def _delete_all_N2VC(self, db_nsr: dict):
         self._write_all_config_status(db_nsr=db_nsr, status='TERMINATING')
         namespace = "." + db_nsr["_id"]

-        await self.n2vc.delete_namespace(namespace=namespace, total_timeout=self.timeout_charm_delete)
+        try:
+            await self.n2vc.delete_namespace(namespace=namespace, total_timeout=self.timeout_charm_delete)
+        except N2VCNotFound:  # already deleted. Skip
+            pass

         self._write_all_config_status(db_nsr=db_nsr, status='DELETED')
 
     async def _terminate_RO(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
         self._write_all_config_status(db_nsr=db_nsr, status='DELETED')
 
     async def _terminate_RO(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):


@@ -2690,6 +2774,7 @@ class NsLcm(LcmBase):
         timeout_ns_terminate = self.timeout_ns_terminate
         db_nsr = None
         db_nslcmop = None
         timeout_ns_terminate = self.timeout_ns_terminate
         db_nsr = None
         db_nslcmop = None

+        operation_params = None

         exc = None
         error_list = []   # annotates all failed error messages
         db_nslcmop_update = {}
         exc = None
         error_list = []   # annotates all failed error messages
         db_nslcmop_update = {}


@@ -2874,14 +2959,14 @@ class NsLcm(LcmBase):
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )
                     error_detail=error_detail,
                     other_update=db_nsr_update
                 )

-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
+            if operation_params:

                 autoremove = operation_params.get("autoremove", False)
             if nslcmop_operation_state:
                 try:
                 autoremove = operation_params.get("autoremove", False)
             if nslcmop_operation_state:
                 try:


@@ -2927,7 +3012,8 @@ class NsLcm(LcmBase):
                     new_error = created_tasks_info[task] + ": {}".format(exc)
                     error_list.append(created_tasks_info[task])
                     error_detail_list.append(new_error)
                     new_error = created_tasks_info[task] + ": {}".format(exc)
                     error_list.append(created_tasks_info[task])
                     error_detail_list.append(new_error)

-                    if isinstance(exc, (str, DbException, N2VCException, ROclient.ROClientException, LcmException)):
+                    if isinstance(exc, (str, DbException, N2VCException, ROclient.ROClientException, LcmException,
+                                        K8sException)):

                         self.logger.error(logging_text + new_error)
                     else:
                         exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__))
                         self.logger.error(logging_text + new_error)
                     else:
                         exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__))


@@ -3151,8 +3237,11 @@ class NsLcm(LcmBase):
             else:
                 desc_params = self._format_additional_params(db_nsr.get("additionalParamsForNs"))
 
             else:
                 desc_params = self._format_additional_params(db_nsr.get("additionalParamsForNs"))
 

+            if kdu_name:
+                kdu_action = True if not deep_get(kdu, ("kdu-configuration", "juju")) else False
+

             # TODO check if ns is in a proper status
             # TODO check if ns is in a proper status

-            if kdu_name and primitive in ("upgrade", "rollback", "status"):
+            if kdu_name and (primitive in ("upgrade", "rollback", "status") or kdu_action):

                 # kdur and desc_params already set from before
                 if primitive_params:
                     desc_params.update(primitive_params)
                 # kdur and desc_params already set from before
                 if primitive_params:
                     desc_params.update(primitive_params)


@@ -3204,13 +3293,24 @@ class NsLcm(LcmBase):
                             cluster_uuid=kdu.get("k8scluster-uuid"),
                             kdu_instance=kdu.get("kdu-instance")),
                         timeout=timeout_ns_action)
                             cluster_uuid=kdu.get("k8scluster-uuid"),
                             kdu_instance=kdu.get("kdu-instance")),
                         timeout=timeout_ns_action)

+                else:
+                    kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(kdu["kdu-name"], nsr_id)
+                    params = self._map_primitive_params(config_primitive_desc, primitive_params, desc_params)
+
+                    detailed_status = await asyncio.wait_for(
+                        self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
+                            cluster_uuid=kdu.get("k8scluster-uuid"),
+                            kdu_instance=kdu_instance,
+                            primitive_name=primitive,
+                            params=params, db_dict=db_dict,
+                            timeout=timeout_ns_action),
+                        timeout=timeout_ns_action)

 
                 if detailed_status:
                     nslcmop_operation_state = 'COMPLETED'
                 else:
                     detailed_status = ''
                     nslcmop_operation_state = 'FAILED'
 
                 if detailed_status:
                     nslcmop_operation_state = 'COMPLETED'
                 else:
                     detailed_status = ''
                     nslcmop_operation_state = 'FAILED'

-

             else:
                 nslcmop_operation_state, detailed_status = await self._ns_execute_primitive(
                     self._look_for_deployed_vca(nsr_deployed["VCA"],
             else:
                 nslcmop_operation_state, detailed_status = await self._ns_execute_primitive(
                     self._look_for_deployed_vca(nsr_deployed["VCA"],


@@ -3227,7 +3327,7 @@ class NsLcm(LcmBase):
                                                                                    detailed_status))
             return  # database update is called inside finally
 
                                                                                    detailed_status))
             return  # database update is called inside finally
 

-        except (DbException, LcmException, N2VCException) as e:
+        except (DbException, LcmException, N2VCException, K8sException) as e:

             self.logger.error(logging_text + "Exit Exception {}".format(e))
             exc = e
         except asyncio.CancelledError:
             self.logger.error(logging_text + "Exit Exception {}".format(e))
             exc = e
         except asyncio.CancelledError:


@@ -3255,14 +3355,13 @@ class NsLcm(LcmBase):
                     other_update=db_nsr_update
                 )
 
                     other_update=db_nsr_update
                 )
 

-            if db_nslcmop:
-                self._write_op_status(
-                    op_id=nslcmop_id,
-                    stage="",
-                    error_message=error_description_nslcmop,
-                    operation_state=nslcmop_operation_state,
-                    other_update=db_nslcmop_update,
-                )
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )

 
             if nslcmop_operation_state:
                 try:
 
             if nslcmop_operation_state:
                 try:


@@ -3458,10 +3557,10 @@ class NsLcm(LcmBase):
                         db_nsr_update["config-status"] = "configuring pre-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 
                         db_nsr_update["config-status"] = "configuring pre-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 

-                        # Pre-scale reintent check: Check if this sub-operation has been executed before
+                        # Pre-scale retry check: Check if this sub-operation has been executed before

                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')

-                        if (op_index == self.SUBOPERATION_STATUS_SKIP):
+                        if op_index == self.SUBOPERATION_STATUS_SKIP:

                             # Skip sub-operation
                             result = 'COMPLETED'
                             result_detail = 'Done'
                             # Skip sub-operation
                             result = 'COMPLETED'
                             result_detail = 'Done'


@@ -3469,20 +3568,20 @@ class NsLcm(LcmBase):
                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
                                                   vnf_config_primitive, result, result_detail))
                         else:
                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
                                                   vnf_config_primitive, result, result_detail))
                         else:

-                            if (op_index == self.SUBOPERATION_STATUS_NEW):
+                            if op_index == self.SUBOPERATION_STATUS_NEW:

                                 # New sub-operation: Get index of this sub-operation
                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:
                                 # New sub-operation: Get index of this sub-operation
                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:

-                                # Reintent:  Get registered params for this existing sub-operation
+                                # retry:  Get registered params for this existing sub-operation

                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')
                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')

-                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
+                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".

                                                   format(vnf_config_primitive))
                                                   format(vnf_config_primitive))

-                            # Execute the primitive, either with new (first-time) or registered (reintent) args
+                            # Execute the primitive, either with new (first-time) or registered (retry) args

                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,
                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,


@@ -3506,26 +3605,26 @@ class NsLcm(LcmBase):
             # if (RO_nsr_id and RO_scaling_info):
             if RO_scaling_info:
                 scale_process = "RO"
             # if (RO_nsr_id and RO_scaling_info):
             if RO_scaling_info:
                 scale_process = "RO"

-                # Scale RO reintent check: Check if this sub-operation has been executed before
+                # Scale RO retry check: Check if this sub-operation has been executed before

                 op_index = self._check_or_add_scale_suboperation(
                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
                 op_index = self._check_or_add_scale_suboperation(
                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)

-                if (op_index == self.SUBOPERATION_STATUS_SKIP):
+                if op_index == self.SUBOPERATION_STATUS_SKIP:

                     # Skip sub-operation
                     result = 'COMPLETED'
                     result_detail = 'Done'
                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
                         result, result_detail))
                 else:
                     # Skip sub-operation
                     result = 'COMPLETED'
                     result_detail = 'Done'
                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
                         result, result_detail))
                 else:

-                    if (op_index == self.SUBOPERATION_STATUS_NEW):
+                    if op_index == self.SUBOPERATION_STATUS_NEW:

                         # New sub-operation: Get index of this sub-operation
                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                         self.logger.debug(logging_text + "New sub-operation RO")
                     else:
                         # New sub-operation: Get index of this sub-operation
                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
                         self.logger.debug(logging_text + "New sub-operation RO")
                     else:

-                        # Reintent:  Get registered params for this existing sub-operation
+                        # retry:  Get registered params for this existing sub-operation

                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                         RO_nsr_id = op.get('RO_nsr_id')
                         RO_scaling_info = op.get('RO_scaling_info')
                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                         RO_nsr_id = op.get('RO_nsr_id')
                         RO_scaling_info = op.get('RO_scaling_info')

-                        self.logger.debug(logging_text + "Sub-operation RO reintent".format(
+                        self.logger.debug(logging_text + "Sub-operation RO retry for primitive {}".format(

                             vnf_config_primitive))
 
                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
                             vnf_config_primitive))
 
                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})


@@ -3651,7 +3750,7 @@ class NsLcm(LcmBase):
                         db_nsr_update["config-status"] = "configuring post-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 
                         db_nsr_update["config-status"] = "configuring post-scaling"
                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
 

-                        # Post-scale reintent check: Check if this sub-operation has been executed before
+                        # Post-scale retry check: Check if this sub-operation has been executed before

                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
                         if op_index == self.SUBOPERATION_STATUS_SKIP:
                         op_index = self._check_or_add_scale_suboperation(
                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
                         if op_index == self.SUBOPERATION_STATUS_SKIP:


@@ -3668,14 +3767,14 @@ class NsLcm(LcmBase):
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:
                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
                                                   format(vnf_config_primitive))
                             else:

-                                # Reintent:  Get registered params for this existing sub-operation
+                                # retry:  Get registered params for this existing sub-operation

                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')
                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
                                 vnf_index = op.get('member_vnf_index')
                                 vnf_config_primitive = op.get('primitive')
                                 primitive_params = op.get('primitive_params')

-                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
+                                self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".

                                                   format(vnf_config_primitive))
                                                   format(vnf_config_primitive))

-                            # Execute the primitive, either with new (first-time) or registered (reintent) args
+                            # Execute the primitive, either with new (first-time) or registered (retry) args

                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,
                             result, result_detail = await self._ns_execute_primitive(
                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
                                                             member_vnf_index=vnf_index,


@@ -3694,9 +3793,6 @@ class NsLcm(LcmBase):
                         scale_process = None
             # POST-SCALE END
 
                         scale_process = None
             # POST-SCALE END
 

-            db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
-            db_nslcmop_update["statusEnteredTime"] = time()
-            db_nslcmop_update["detailed-status"] = "done"

             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
                 else old_operational_status
             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
                 else old_operational_status


@@ -3719,10 +3815,8 @@ class NsLcm(LcmBase):
                 current_operation_id=None
             )
             if exc:
                 current_operation_id=None
             )
             if exc:

-                if db_nslcmop:
-                    db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
-                    db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
-                    db_nslcmop_update["statusEnteredTime"] = time()
+                db_nslcmop_update["detailed-status"] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
+                nslcmop_operation_state = "FAILED"

                 if db_nsr:
                     db_nsr_update["operational-status"] = old_operational_status
                     db_nsr_update["config-status"] = old_config_status
                 if db_nsr:
                     db_nsr_update["operational-status"] = old_operational_status
                     db_nsr_update["config-status"] = old_config_status


@@ -3734,20 +3828,27 @@ class NsLcm(LcmBase):
                             db_nsr_update["operational-status"] = "failed"
                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
                                                                                                      exc)
                             db_nsr_update["operational-status"] = "failed"
                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
                                                                                                      exc)

-            try:
-                if db_nslcmop and db_nslcmop_update:
-                    self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
-                if db_nsr:
-                    self._write_ns_status(
-                        nsr_id=nsr_id,
-                        ns_state=None,
-                        current_operation="IDLE",
-                        current_operation_id=None,
-                        other_update=db_nsr_update
-                    )
+            else:
+                error_description_nslcmop = None
+                nslcmop_operation_state = "COMPLETED"
+                db_nslcmop_update["detailed-status"] = "Done"
+
+            self._write_op_status(
+                op_id=nslcmop_id,
+                stage="",
+                error_message=error_description_nslcmop,
+                operation_state=nslcmop_operation_state,
+                other_update=db_nslcmop_update,
+            )
+            if db_nsr:
+                self._write_ns_status(
+                    nsr_id=nsr_id,
+                    ns_state=None,
+                    current_operation="IDLE",
+                    current_operation_id=None,
+                    other_update=db_nsr_update
+                )

 
 

-            except DbException as e:
-                self.logger.error(logging_text + "Cannot update database: {}".format(e))

             if nslcmop_operation_state:
                 try:
                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
             if nslcmop_operation_state:
                 try:
                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,