osm_lcm/k8s.py - osm/LCM - Gitiles

 # -*- coding: utf-8 -*-

 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 __author__ = (
     "Shrinithi R <shrinithi.r@tataelxsi.co.in>",
     "Shahithya Y <shahithya.y@tataelxsi.co.in>",
 )

 import os
 from time import time
 import traceback
 import yaml
 from copy import deepcopy
 from osm_lcm import vim_sdn
 from osm_lcm.gitops import GitOpsLcm
 from osm_lcm.lcm_utils import LcmException

 MAP_PROFILE = {
     "infra_controller_profiles": "infra-controllers",
     "infra_config_profiles": "infra-configs",
     "resource_profiles": "managed_resources",
     "app_profiles": "apps",
 }


 class NodeGroupLcm(GitOpsLcm):
     db_collection = "nodegroups"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)
         self._workflows = {
             "add_nodegroup": {
                 "check_resource_function": self.check_add_nodegroup,
             },
             "scale_nodegroup": {
                 "check_resource_function": self.check_scale_nodegroup,
             },
             "delete_nodegroup": {
                 "check_resource_function": self.check_delete_nodegroup,
             },
         }

     async def create(self, params, order_id):
         self.logger.info("Add NodeGroup Enter")

         # To get the nodegroup and op ids
         nodegroup_id = params["nodegroup_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(nodegroup_id, op_id)

         # To get the nodegroup details and control plane from DB
         db_nodegroup = self.db.get_one(self.db_collection, {"_id": nodegroup_id})
         db_cluster = self.db.get_one("clusters", {"_id": db_nodegroup["cluster_id"]})

         # To get the operation params details
         op_params = self.get_operation_params(db_nodegroup, op_id)
         self.logger.info(f"Operations Params: {op_params}")

         db_vim = self.db.get_one("vim_accounts", {"name": db_cluster["vim_account"]})

         # To copy the cluster content and decrypting fields to use in workflows
         workflow_content = {
             "nodegroup": db_nodegroup,
             "cluster": db_cluster,
             "vim_account": db_vim,
         }
         self.logger.info(f"Workflow content: {workflow_content}")

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "add_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_nodegroup
         )

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "add_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "add_nodegroup", op_id, op_params, db_nodegroup
             )
         self.db.set_one(self.db_collection, {"_id": db_nodegroup["_id"]}, db_nodegroup)
         self.logger.info(f"Add NodeGroup Exit with resource status: {resource_status}")
         return

     async def check_add_nodegroup(self, op_id, op_params, content):
         self.logger.info(f"check_add_nodegroup Operation {op_id}. Params: {op_params}.")
         self.logger.info(f"Content: {content}")
         db_nodegroup = content
         nodegroup_name = db_nodegroup["git_name"].lower()
         nodegroup_kustomization_name = nodegroup_name
         checkings_list = [
             {
                 "item": "kustomization",
                 "name": nodegroup_kustomization_name,
                 "namespace": "managed-resources",
                 "condition": {
                     "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                     "value": "True",
                 },
                 "timeout": self._checkloop_kustomization_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
             },
             {
                 "item": "nodegroup_aws",
                 "name": nodegroup_name,
                 "namespace": "",
                 "condition": {
                     "jsonpath_filter": "status.conditions[?(@.type=='Synced')].status",
                     "value": "True",
                 },
                 "timeout": self._checkloop_resource_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.RESOURCE_SYNCED.NODEGROUP",
             },
             {
                 "item": "nodegroup_aws",
                 "name": nodegroup_name,
                 "namespace": "",
                 "condition": {
                     "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                     "value": "True",
                 },
                 "timeout": self._checkloop_resource_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.RESOURCE_READY.NODEGROUP",
             },
         ]
         self.logger.info(f"Checking list: {checkings_list}")
         result, message = await self.common_check_list(
             op_id, checkings_list, "nodegroups", db_nodegroup
         )
         if not result:
             return False, message
         return True, "OK"

     async def scale(self, params, order_id):
         self.logger.info("Scale nodegroup Enter")

         op_id = params["operation_id"]
         nodegroup_id = params["nodegroup_id"]

         # To initialize the operation states
         self.initialize_operation(nodegroup_id, op_id)

         db_nodegroup = self.db.get_one(self.db_collection, {"_id": nodegroup_id})
         db_cluster = self.db.get_one("clusters", {"_id": db_nodegroup["cluster_id"]})
         op_params = self.get_operation_params(db_nodegroup, op_id)
         db_vim = self.db.get_one("vim_accounts", {"name": db_cluster["vim_account"]})

         workflow_content = {
             "nodegroup": db_nodegroup,
             "cluster": db_cluster,
             "vim_account": db_vim,
         }
         self.logger.info(f"Workflow content: {workflow_content}")

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "scale_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_nodegroup
         )

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "scale_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "scale_nodegroup", op_id, op_params, db_nodegroup
             )

         if resource_status:
             db_nodegroup["state"] = "READY"
             self.db.set_one(
                 self.db_collection, {"_id": db_nodegroup["_id"]}, db_nodegroup
             )
         self.logger.info(
             f"Nodegroup Scale Exit with resource status: {resource_status}"
         )
         return

     async def check_scale_nodegroup(self, op_id, op_params, content):
         self.logger.info(
             f"check_scale_nodegroup Operation {op_id}. Params: {op_params}."
         )
         self.logger.debug(f"Content: {content}")
         db_nodegroup = content
         nodegroup_name = db_nodegroup["git_name"].lower()
         nodegroup_kustomization_name = nodegroup_name
         checkings_list = [
             {
                 "item": "kustomization",
                 "name": nodegroup_kustomization_name,
                 "namespace": "managed-resources",
                 "condition": {
                     "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                     "value": "True",
                 },
                 "timeout": self._checkloop_kustomization_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
             },
             {
                 "item": "nodegroup_aws",
                 "name": nodegroup_name,
                 "namespace": "",
                 "condition": {
                     "jsonpath_filter": "status.atProvider.scalingConfig[0].desiredSize",
                     "value": f"{op_params['node_count']}",
                 },
                 "timeout": self._checkloop_resource_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.RESOURCE_SYNCED.NODEGROUP",
             },
         ]
         self.logger.info(f"Checking list: {checkings_list}")
         return await self.common_check_list(
             op_id, checkings_list, "nodegroups", db_nodegroup
         )

     async def delete(self, params, order_id):
         self.logger.info("Delete nodegroup Enter")

         op_id = params["operation_id"]
         nodegroup_id = params["nodegroup_id"]

         # To initialize the operation states
         self.initialize_operation(nodegroup_id, op_id)

         db_nodegroup = self.db.get_one(self.db_collection, {"_id": nodegroup_id})
         db_cluster = self.db.get_one("clusters", {"_id": db_nodegroup["cluster_id"]})
         op_params = self.get_operation_params(db_nodegroup, op_id)

         workflow_content = {"nodegroup": db_nodegroup, "cluster": db_cluster}

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_nodegroup
         )

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "delete_nodegroup", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "delete_nodegroup", op_id, op_params, db_nodegroup
             )

         if resource_status:
             node_count = db_cluster.get("node_count")
             new_node_count = node_count - 1
             self.logger.info(f"New Node count: {new_node_count}")
             db_cluster["node_count"] = new_node_count
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             db_nodegroup["state"] = "DELETED"
             self.db.set_one(
                 self.db_collection, {"_id": db_nodegroup["_id"]}, db_nodegroup
             )
             self.db.del_one(self.db_collection, {"_id": db_nodegroup["_id"]})
         self.logger.info(
             f"Nodegroup Delete Exit with resource status: {resource_status}"
         )
         return

     async def check_delete_nodegroup(self, op_id, op_params, content):
         self.logger.info(
             f"check_delete_nodegroup Operation {op_id}. Params: {op_params}."
         )
         db_nodegroup = content
         nodegroup_name = db_nodegroup["git_name"].lower()
         nodegroup_kustomization_name = nodegroup_name
         checkings_list = [
             {
                 "item": "kustomization",
                 "name": nodegroup_kustomization_name,
                 "namespace": "managed-resources",
                 "deleted": True,
                 "timeout": self._checkloop_kustomization_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.KUSTOMIZATION_DELETED",
             },
             {
                 "item": "nodegroup_aws",
                 "name": nodegroup_name,
                 "namespace": "",
                 "deleted": True,
                 "timeout": self._checkloop_resource_timeout,
                 "enable": True,
                 "resourceState": "IN_PROGRESS.RESOURCE_DELETED.NODEGROUP",
             },
         ]
         self.logger.info(f"Checking list: {checkings_list}")
         return await self.common_check_list(
             op_id, checkings_list, "nodegroups", db_nodegroup
         )


 class ClusterLcm(GitOpsLcm):
     db_collection = "clusters"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)
         self._workflows = {
             "create_cluster": {
                 "check_resource_function": self.check_create_cluster,
             },
             "register_cluster": {
                 "check_resource_function": self.check_register_cluster,
             },
             "update_cluster": {
                 "check_resource_function": self.check_update_cluster,
             },
             "delete_cluster": {
                 "check_resource_function": self.check_delete_cluster,
             },
         }
         self.regist = vim_sdn.K8sClusterLcm(msg, self.lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("cluster Create Enter")
         workflow_status = None
         resource_status = None

         # To get the cluster and op ids
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(cluster_id, op_id)

         # To get the cluster
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         db_cluster_copy = self.decrypted_copy(db_cluster)
         workflow_content = {
             "cluster": db_cluster_copy,
         }

         # To get the vim account details
         db_vim = self.db.get_one("vim_accounts", {"name": db_cluster["vim_account"]})
         workflow_content["vim_account"] = db_vim

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["state"] = "FAILED_CREATION"
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             # Clean items used in the workflow, no matter if the workflow succeeded
             clean_status, clean_msg = await self.odu.clean_items_workflow(
                 "create_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
             )
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["state"] = "CREATED"
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["state"] = "FAILED_CREATION"
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "create_cluster", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "create_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster["operatingState"] = "IDLE"
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         db_cluster["current_operation"] = None

         # Retrieve credentials and subnets and register the cluster in k8sclusters collection
         cluster_creds = None
         db_register = self.db.get_one("k8sclusters", {"name": db_cluster["name"]})
         if db_cluster["resourceState"] == "READY" and db_cluster["state"] == "CREATED":
             # Retrieve credentials
             result, cluster_creds = await self.odu.get_cluster_credentials(db_cluster)
             # TODO: manage the case where the credentials are not available
             if result:
                 db_cluster["credentials"] = cluster_creds

             # Retrieve subnets
             if op_params.get("private_subnet") and op_params.get("public_subnet"):
                 db_cluster["private_subnet"] = op_params["private_subnet"]
                 db_cluster["public_subnet"] = op_params["public_subnet"]
             else:
                 if db_vim["vim_type"] == "aws":
                     generic_object = await self.odu.list_object(
                         api_group="ec2.aws.upbound.io",
                         api_plural="subnets",
                         api_version="v1beta1",
                     )
                     private_subnet = []
                     public_subnet = []
                     for subnet in generic_object:
                         labels = subnet.get("metadata", {}).get("labels", {})
                         status = subnet.get("status", {}).get("atProvider", {})
                         # Extract relevant label values
                         cluster_label = labels.get("cluster")
                         access_label = labels.get("access")
                         subnet_id = status.get("id")
                         # Apply filtering
                         if cluster_label == db_cluster["name"] and subnet_id:
                             if access_label == "private":
                                 private_subnet.append(subnet_id)
                             elif access_label == "public":
                                 public_subnet.append(subnet_id)
                     # Update db_cluster
                     db_cluster["private_subnet"] = private_subnet
                     db_cluster["public_subnet"] = public_subnet
                     self.logger.info("DB cluster: {}".format(db_cluster))

             # Register the cluster in k8sclusters collection
             db_register["credentials"] = cluster_creds
             # To call the lcm.py for registering the cluster in k8scluster lcm.
             self.db.set_one("k8sclusters", {"_id": db_register["_id"]}, db_register)
             register = await self.regist.create(db_register, order_id)
             self.logger.debug(f"Register is : {register}")
         else:
             db_register["_admin"]["operationalState"] = "ERROR"
             self.db.set_one("k8sclusters", {"_id": db_register["_id"]}, db_register)

         # Update db_cluster
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
         self.update_default_profile_agekeys(db_cluster_copy)
         self.update_profile_state(db_cluster, workflow_status, resource_status)

         return

     async def check_create_cluster(self, op_id, op_params, content):
         self.logger.info(
             f"check_create_cluster Operation {op_id}. Params: {op_params}."
         )
         db_cluster = content["cluster"]
         cluster_name = db_cluster["git_name"].lower()
         cluster_kustomization_name = cluster_name
         db_vim_account = content["vim_account"]
         cloud_type = db_vim_account["vim_type"]
         nodegroup_name = ""
         if cloud_type == "aws":
             nodegroup_name = f"{cluster_name}-nodegroup"
             cluster_name = f"{cluster_name}-cluster"
         elif cloud_type == "gcp":
             nodegroup_name = f"nodepool-{cluster_name}"
         bootstrap = op_params.get("bootstrap", True)
         if cloud_type in ("azure", "gcp", "aws"):
             checkings_list = [
                 {
                     "item": "kustomization",
                     "name": cluster_kustomization_name,
                     "namespace": "managed-resources",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                         "value": "True",
                     },
                     "timeout": 1500,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
                 },
                 {
                     "item": f"cluster_{cloud_type}",
                     "name": cluster_name,
                     "namespace": "",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Synced')].status",
                         "value": "True",
                     },
                     "timeout": self._checkloop_resource_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.RESOURCE_SYNCED.CLUSTER",
                 },
                 {
                     "item": f"cluster_{cloud_type}",
                     "name": cluster_name,
                     "namespace": "",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                         "value": "True",
                     },
                     "timeout": self._checkloop_resource_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.RESOURCE_READY.CLUSTER",
                 },
                 {
                     "item": "kustomization",
                     "name": f"{cluster_kustomization_name}-bstrp-fluxctrl",
                     "namespace": "managed-resources",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                         "value": "True",
                     },
                     "timeout": self._checkloop_resource_timeout,
                     "enable": bootstrap,
                     "resourceState": "IN_PROGRESS.BOOTSTRAP_OK",
                 },
             ]
         else:
             return False, "Not suitable VIM account to check cluster status"
         if cloud_type != "aws":
             if nodegroup_name:
                 nodegroup_check = {
                     "item": f"nodegroup_{cloud_type}",
                     "name": nodegroup_name,
                     "namespace": "",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                         "value": "True",
                     },
                     "timeout": self._checkloop_resource_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.RESOURCE_READY.NODEGROUP",
                 }
                 checkings_list.insert(3, nodegroup_check)
         return await self.common_check_list(
             op_id, checkings_list, "clusters", db_cluster
         )

     def update_default_profile_agekeys(self, db_cluster):
         profiles = [
             "infra_controller_profiles",
             "infra_config_profiles",
             "app_profiles",
             "resource_profiles",
         ]
         self.logger.debug("the db_cluster is :{}".format(db_cluster))
         for profile_type in profiles:
             profile_id = db_cluster[profile_type]
             db_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(db_collection, {"_id": profile_id})
             db_profile["age_pubkey"] = db_cluster["age_pubkey"]
             db_profile["age_privkey"] = db_cluster["age_privkey"]
             self.encrypt_age_keys(db_profile)
             self.db.set_one(db_collection, {"_id": db_profile["_id"]}, db_profile)

     def update_profile_state(self, db_cluster, workflow_status, resource_status):
         profiles = [
             "infra_controller_profiles",
             "infra_config_profiles",
             "app_profiles",
             "resource_profiles",
         ]
         self.logger.debug("the db_cluster is :{}".format(db_cluster))
         for profile_type in profiles:
             profile_id = db_cluster[profile_type]
             db_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(db_collection, {"_id": profile_id})
             op_id = db_profile["operationHistory"][-1].get("op_id")
             db_profile["state"] = db_cluster["state"]
             db_profile["resourceState"] = db_cluster["resourceState"]
             db_profile["operatingState"] = db_cluster["operatingState"]
             db_profile = self.update_operation_history(
                 db_profile, op_id, workflow_status, resource_status
             )
             self.db.set_one(db_collection, {"_id": db_profile["_id"]}, db_profile)

     async def delete(self, params, order_id):
         self.logger.info("cluster delete Enter")

         try:
             # To get the cluster and op ids
             cluster_id = params["cluster_id"]
             op_id = params["operation_id"]

             # To initialize the operation states
             self.initialize_operation(cluster_id, op_id)

             # To get the cluster
             db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

             # To get the operation params details
             op_params = self.get_operation_params(db_cluster, op_id)

             # To copy the cluster content and decrypting fields to use in workflows
             workflow_content = {
                 "cluster": self.decrypted_copy(db_cluster),
             }

             # To get the vim account details
             db_vim = self.db.get_one(
                 "vim_accounts", {"name": db_cluster["vim_account"]}
             )
             workflow_content["vim_account"] = db_vim
         except Exception as e:
             self.logger.debug(traceback.format_exc())
             self.logger.debug(f"Exception: {e}", exc_info=True)
             raise e

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["state"] = "FAILED_DELETION"
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             # Clean items used in the workflow, no matter if the workflow succeeded
             clean_status, clean_msg = await self.odu.clean_items_workflow(
                 "delete_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
             )
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["state"] = "DELETED"
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["state"] = "FAILED_DELETION"
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         # Clean items used in the workflow or in the cluster, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "delete_cluster", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "delete_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster["operatingState"] = "IDLE"
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 cluster_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         # To delete it from DB
         if db_cluster["state"] == "DELETED":
             self.delete_cluster(db_cluster)

         # To delete it from k8scluster collection
         self.db.del_one("k8sclusters", {"name": db_cluster["name"]})

         return

     async def check_delete_cluster(self, op_id, op_params, content):
         self.logger.info(
             f"check_delete_cluster Operation {op_id}. Params: {op_params}."
         )
         self.logger.debug(f"Content: {content}")
         db_cluster = content["cluster"]
         cluster_name = db_cluster["git_name"].lower()
         cluster_kustomization_name = cluster_name
         db_vim_account = content["vim_account"]
         cloud_type = db_vim_account["vim_type"]
         if cloud_type == "aws":
             cluster_name = f"{cluster_name}-cluster"
         if cloud_type in ("azure", "gcp", "aws"):
             checkings_list = [
                 {
                     "item": "kustomization",
                     "name": cluster_kustomization_name,
                     "namespace": "managed-resources",
                     "deleted": True,
                     "timeout": self._checkloop_kustomization_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.KUSTOMIZATION_DELETED",
                 },
                 {
                     "item": f"cluster_{cloud_type}",
                     "name": cluster_name,
                     "namespace": "",
                     "deleted": True,
                     "timeout": self._checkloop_resource_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.RESOURCE_DELETED.CLUSTER",
                 },
             ]
         else:
             return False, "Not suitable VIM account to check cluster status"
         return await self.common_check_list(
             op_id, checkings_list, "clusters", db_cluster
         )

     def delete_cluster(self, db_cluster):
         # Actually, item_content is equal to db_cluster
         # detach profiles
         update_dict = None
         profiles_to_detach = [
             "infra_controller_profiles",
             "infra_config_profiles",
             "app_profiles",
             "resource_profiles",
         ]
         """
         profiles_collection = {
             "infra_controller_profiles": "k8sinfra_controller",
             "infra_config_profiles": "k8sinfra_config",
             "app_profiles": "k8sapp",
             "resource_profiles": "k8sresource",
         }
         """
         for profile_type in profiles_to_detach:
             if db_cluster.get(profile_type):
                 profile_ids = db_cluster[profile_type]
                 profile_ids_copy = deepcopy(profile_ids)
                 for profile_id in profile_ids_copy:
                     db_collection = self.profile_collection_mapping[profile_type]
                     db_profile = self.db.get_one(db_collection, {"_id": profile_id})
                     self.logger.debug("the db_profile is :{}".format(db_profile))
                     self.logger.debug(
                         "the item_content name is :{}".format(db_cluster["name"])
                     )
                     self.logger.debug(
                         "the db_profile name is :{}".format(db_profile["name"])
                     )
                     if db_cluster["name"] == db_profile["name"]:
                         self.delete_profile_ksu(profile_id, profile_type)
                         self.db.del_one(db_collection, {"_id": profile_id})
                     else:
                         profile_ids.remove(profile_id)
                         update_dict = {profile_type: profile_ids}
                         self.db.set_one(
                             "clusters", {"_id": db_cluster["_id"]}, update_dict
                         )
         self.db.del_one("clusters", {"_id": db_cluster["_id"]})

     async def attach_profile(self, params, order_id):
         self.logger.info("profile attach Enter")

         # To get the cluster and op ids
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(cluster_id, op_id)

         # To get the cluster
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         workflow_content = {
             "cluster": self.decrypted_copy(db_cluster),
         }

         # To get the profile details
         profile_id = params["profile_id"]
         profile_type = params["profile_type"]
         profile_collection = self.profile_collection_mapping[profile_type]
         db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
         db_profile["profile_type"] = profile_type
         # content["profile"] = db_profile
         workflow_content["profile"] = db_profile

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "attach_profile_to_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["resourceState"] = "ERROR"
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "attach_profile_to_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster["operatingState"] = "IDLE"
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         profile_list = db_cluster[profile_type]
         if resource_status:
             profile_list.append(profile_id)
             db_cluster[profile_type] = profile_list
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         return

     async def detach_profile(self, params, order_id):
         self.logger.info("profile dettach Enter")

         # To get the cluster and op ids
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(cluster_id, op_id)

         # To get the cluster
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         workflow_content = {
             "cluster": self.decrypted_copy(db_cluster),
         }

         # To get the profile details
         profile_id = params["profile_id"]
         profile_type = params["profile_type"]
         profile_collection = self.profile_collection_mapping[profile_type]
         db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
         db_profile["profile_type"] = profile_type
         workflow_content["profile"] = db_profile

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "detach_profile_from_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "detach_profile_from_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster["operatingState"] = "IDLE"
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         profile_list = db_cluster[profile_type]
         self.logger.info("profile list is : {}".format(profile_list))
         if resource_status:
             profile_list.remove(profile_id)
             db_cluster[profile_type] = profile_list
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         return

     async def register(self, params, order_id):
         self.logger.info("cluster register enter")
         workflow_status = None
         resource_status = None

         # To get the cluster and op ids
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(cluster_id, op_id)

         # To get the cluster
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         db_cluster_copy = self.decrypted_copy(db_cluster)
         workflow_content = {
             "cluster": db_cluster_copy,
         }

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "register_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["state"] = "FAILED_CREATION"
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             # Clean items used in the workflow, no matter if the workflow succeeded
             clean_status, clean_msg = await self.odu.clean_items_workflow(
                 "register_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
             )
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["state"] = "CREATED"
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["state"] = "FAILED_CREATION"
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "register_cluster", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "register_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster["operatingState"] = "IDLE"
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         # Update default profile agekeys and state
         self.update_default_profile_agekeys(db_cluster_copy)
         self.update_profile_state(db_cluster, workflow_status, resource_status)

         db_register = self.db.get_one("k8sclusters", {"name": db_cluster["name"]})
         db_register["credentials"] = db_cluster["credentials"]
         self.db.set_one("k8sclusters", {"_id": db_register["_id"]}, db_register)

         if db_cluster["resourceState"] == "READY" and db_cluster["state"] == "CREATED":
             # To call the lcm.py for registering the cluster in k8scluster lcm.
             register = await self.regist.create(db_register, order_id)
             self.logger.debug(f"Register is : {register}")
         else:
             db_register["_admin"]["operationalState"] = "ERROR"
             self.db.set_one("k8sclusters", {"_id": db_register["_id"]}, db_register)

         return

     async def check_register_cluster(self, op_id, op_params, content):
         self.logger.info(
             f"check_register_cluster Operation {op_id}. Params: {op_params}."
         )
         # self.logger.debug(f"Content: {content}")
         db_cluster = content["cluster"]
         cluster_name = db_cluster["git_name"].lower()
         cluster_kustomization_name = cluster_name
         bootstrap = op_params.get("bootstrap", True)
         checkings_list = [
             {
                 "item": "kustomization",
                 "name": f"{cluster_kustomization_name}-bstrp-fluxctrl",
                 "namespace": "managed-resources",
                 "condition": {
                     "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                     "value": "True",
                 },
                 "timeout": self._checkloop_kustomization_timeout,
                 "enable": bootstrap,
                 "resourceState": "IN_PROGRESS.BOOTSTRAP_OK",
             },
         ]
         return await self.common_check_list(
             op_id, checkings_list, "clusters", db_cluster
         )

     async def deregister(self, params, order_id):
         self.logger.info("cluster deregister enter")

         # To get the cluster and op ids
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]

         # To initialize the operation states
         self.initialize_operation(cluster_id, op_id)

         # To get the cluster
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         workflow_content = {
             "cluster": self.decrypted_copy(db_cluster),
         }

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "deregister_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["state"] = "FAILED_DELETION"
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             return

         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "workflow_status is: {} and workflow_msg is: {}".format(
                 workflow_status, workflow_msg
             )
         )
         if workflow_status:
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["state"] = "FAILED_DELETION"
             db_cluster["resourceState"] = "ERROR"
         # has to call update_operation_history return content
         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "deregister_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "resource_status is :{} and resource_msg is :{}".format(
                     resource_status, resource_msg
                 )
             )
             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, resource_status
         )
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         await self.delete(params, order_id)
         # Clean items used in the workflow or in the cluster, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "deregister_cluster", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         return

     async def get_creds(self, params, order_id):
         self.logger.info("Cluster get creds Enter")
         cluster_id = params["cluster_id"]
         op_id = params["operation_id"]
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})
         result, cluster_creds = await self.odu.get_cluster_credentials(db_cluster)
         if result:
             db_cluster["credentials"] = cluster_creds
         op_len = 0
         for operations in db_cluster["operationHistory"]:
             if operations["op_id"] == op_id:
                 db_cluster["operationHistory"][op_len]["result"] = result
                 db_cluster["operationHistory"][op_len]["endDate"] = time()
             op_len += 1
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
         self.logger.info("Cluster Get Creds Exit")
         return

     async def update(self, params, order_id):
         self.logger.info("Cluster update Enter")
         # To get the cluster details
         cluster_id = params["cluster_id"]
         db_cluster = self.db.get_one("clusters", {"_id": cluster_id})

         # To get the operation params details
         op_id = params["operation_id"]
         op_params = self.get_operation_params(db_cluster, op_id)

         # To copy the cluster content and decrypting fields to use in workflows
         workflow_content = {
             "cluster": self.decrypted_copy(db_cluster),
         }

         # vim account details
         db_vim = self.db.get_one("vim_accounts", {"name": db_cluster["vim_account"]})
         workflow_content["vim_account"] = db_vim

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "update_cluster", op_id, op_params, workflow_content
         )
         if not workflow_res:
             self.logger.error(f"Failed to launch workflow: {workflow_name}")
             db_cluster["resourceState"] = "ERROR"
             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status=False, resource_status=None
             )
             self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
             # Clean items used in the workflow, no matter if the workflow succeeded
             clean_status, clean_msg = await self.odu.clean_items_workflow(
                 "update_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
             )
             return
         self.logger.info("workflow_name is: {}".format(workflow_name))
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "Workflow Status: {} Workflow Message: {}".format(
                 workflow_status, workflow_msg
             )
         )

         if workflow_status:
             db_cluster["resourceState"] = "IN_PROGRESS.GIT_SYNCED"
         else:
             db_cluster["resourceState"] = "ERROR"

         db_cluster = self.update_operation_history(
             db_cluster, op_id, workflow_status, None
         )
         # self.logger.info("Db content: {}".format(db_content))
         # self.db.set_one(self.db_collection, {"_id": _id}, db_cluster)
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "update_cluster", op_id, op_params, workflow_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "update_cluster", op_id, op_params, workflow_content
             )
             self.logger.info(
                 "Resource Status: {} Resource Message: {}".format(
                     resource_status, resource_msg
                 )
             )

             if resource_status:
                 db_cluster["resourceState"] = "READY"
             else:
                 db_cluster["resourceState"] = "ERROR"

             db_cluster = self.update_operation_history(
                 db_cluster, op_id, workflow_status, resource_status
             )

         db_cluster["operatingState"] = "IDLE"
         # self.logger.info("db_cluster: {}".format(db_cluster))
         # TODO: verify condition
         # For the moment, if the workflow completed successfully, then we update the db accordingly.
         if workflow_status:
             if "k8s_version" in op_params:
                 db_cluster["k8s_version"] = op_params["k8s_version"]
             if "node_count" in op_params:
                 db_cluster["node_count"] = op_params["node_count"]
             if "node_size" in op_params:
                 db_cluster["node_count"] = op_params["node_size"]
             # self.db.set_one(self.db_collection, {"_id": _id}, db_content)
         db_cluster["current_operation"] = None
         self.db.set_one("clusters", {"_id": db_cluster["_id"]}, db_cluster)
         return

     async def check_update_cluster(self, op_id, op_params, content):
         self.logger.info(
             f"check_update_cluster Operation {op_id}. Params: {op_params}."
         )
         self.logger.debug(f"Content: {content}")
         # return await self.check_dummy_operation(op_id, op_params, content)
         db_cluster = content["cluster"]
         cluster_name = db_cluster["git_name"].lower()
         cluster_kustomization_name = cluster_name
         db_vim_account = content["vim_account"]
         cloud_type = db_vim_account["vim_type"]
         if cloud_type == "aws":
             cluster_name = f"{cluster_name}-cluster"
         if cloud_type in ("azure", "gcp", "aws"):
             checkings_list = [
                 {
                     "item": "kustomization",
                     "name": cluster_kustomization_name,
                     "namespace": "managed-resources",
                     "condition": {
                         "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                         "value": "True",
                     },
                     "timeout": self._checkloop_kustomization_timeout,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
                 },
             ]
         else:
             return False, "Not suitable VIM account to check cluster status"
         # Scale operation
         if "node_count" in op_params:
             if cloud_type in ("azure", "gcp"):
                 checkings_list.append(
                     {
                         "item": f"cluster_{cloud_type}",
                         "name": cluster_name,
                         "namespace": "",
                         "condition": {
                             "jsonpath_filter": "status.atProvider.defaultNodePool[0].nodeCount",
                             "value": f"{op_params['node_count']}",
                         },
                         "timeout": self._checkloop_resource_timeout * 3,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.RESOURCE_READY.NODE_COUNT.CLUSTER",
                     }
                 )
             elif cloud_type == "aws":
                 checkings_list.append(
                     {
                         "item": f"nodegroup_{cloud_type}",
                         "name": f"{cluster_name}-nodegroup",
                         "namespace": "",
                         "condition": {
                             "jsonpath_filter": "status.atProvider.scalingConfig[0].desiredSize",
                             "value": f"{op_params['node_count']}",
                         },
                         "timeout": self._checkloop_resource_timeout * 3,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.RESOURCE_READY.NODE_COUNT.CLUSTER",
                     }
                 )

         # Upgrade operation
         if "k8s_version" in op_params:
             checkings_list.append(
                 {
                     "item": f"cluster_{cloud_type}",
                     "name": cluster_name,
                     "namespace": "",
                     "condition": {
                         "jsonpath_filter": "status.atProvider.defaultNodePool[0].orchestratorVersion",
                         "value": op_params["k8s_version"],
                     },
                     "timeout": self._checkloop_resource_timeout * 2,
                     "enable": True,
                     "resourceState": "IN_PROGRESS.RESOURCE_READY.K8S_VERSION.CLUSTER",
                 }
             )
         return await self.common_check_list(
             op_id, checkings_list, "clusters", db_cluster
         )


 class CloudCredentialsLcm(GitOpsLcm):
     db_collection = "vim_accounts"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def add(self, params, order_id):
         self.logger.info("Cloud Credentials create")
         vim_id = params["_id"]
         op_id = vim_id
         op_params = params
         db_content = self.db.get_one(self.db_collection, {"_id": vim_id})
         vim_config = db_content.get("config", {})
         self.db.encrypt_decrypt_fields(
             vim_config.get("credentials"),
             "decrypt",
             ["password", "secret"],
             schema_version=db_content["schema_version"],
             salt=vim_id,
         )

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_cloud_credentials", op_id, op_params, db_content
         )

         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )

         self.logger.info(
             "Workflow Status: {} Workflow Msg: {}".format(workflow_status, workflow_msg)
         )

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "create_cloud_credentials", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "create_cloud_credentials", op_id, op_params, db_content
             )
             self.logger.info(
                 "Resource Status: {} Resource Message: {}".format(
                     resource_status, resource_msg
                 )
             )

             db_content["_admin"]["operationalState"] = "ENABLED"
             for operation in db_content["_admin"]["operations"]:
                 if operation["lcmOperationType"] == "create":
                     operation["operationState"] = "ENABLED"
             self.logger.info("Content : {}".format(db_content))
         self.db.set_one("vim_accounts", {"_id": db_content["_id"]}, db_content)
         return

     async def edit(self, params, order_id):
         self.logger.info("Cloud Credentials Update")
         vim_id = params["_id"]
         op_id = vim_id
         op_params = params
         db_content = self.db.get_one("vim_accounts", {"_id": vim_id})
         vim_config = db_content.get("config", {})
         self.db.encrypt_decrypt_fields(
             vim_config.get("credentials"),
             "decrypt",
             ["password", "secret"],
             schema_version=db_content["schema_version"],
             salt=vim_id,
         )

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "update_cloud_credentials", op_id, op_params, db_content
         )
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "Workflow Status: {} Workflow Msg: {}".format(workflow_status, workflow_msg)
         )

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "update_cloud_credentials", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "update_cloud_credentials", op_id, op_params, db_content
             )
             self.logger.info(
                 "Resource Status: {} Resource Message: {}".format(
                     resource_status, resource_msg
                 )
             )
         return

     async def remove(self, params, order_id):
         self.logger.info("Cloud Credentials remove")
         vim_id = params["_id"]
         op_id = vim_id
         op_params = params
         db_content = self.db.get_one("vim_accounts", {"_id": vim_id})

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_cloud_credentials", op_id, op_params, db_content
         )
         workflow_status, workflow_msg = await self.odu.check_workflow_status(
             op_id, workflow_name
         )
         self.logger.info(
             "Workflow Status: {} Workflow Msg: {}".format(workflow_status, workflow_msg)
         )

         if workflow_status:
             resource_status, resource_msg = await self.check_resource_status(
                 "delete_cloud_credentials", op_id, op_params, db_content
             )
             self.logger.info(
                 "Resource Status: {} Resource Message: {}".format(
                     resource_status, resource_msg
                 )
             )
         self.db.del_one(self.db_collection, {"_id": db_content["_id"]})
         return


 class K8sAppLcm(GitOpsLcm):
     db_collection = "k8sapp"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("App Profile Create Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sapp", {"_id": profile_id})
         content["profile_type"] = "applications"
         op_params = self.get_operation_params(content, op_id)
         self.db.set_one("k8sapp", {"_id": content["_id"]}, content)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "create_profile", op_id, op_params, content
             )
         self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
         self.logger.info(
             f"App Profile Create Exit with resource status: {resource_status}"
         )
         return

     async def delete(self, params, order_id):
         self.logger.info("App delete Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sapp", {"_id": profile_id})
         op_params = self.get_operation_params(content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "delete_profile", op_id, op_params, content
             )

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 profile_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         self.logger.info(f"Resource status: {resource_status}")
         if resource_status:
             content["state"] = "DELETED"
             profile_type = self.profile_type_mapping[content["profile_type"]]
             self.delete_profile_ksu(profile_id, profile_type)
             self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
             self.db.del_one(self.db_collection, {"_id": content["_id"]})
         self.logger.info(
             f"App Profile Delete Exit with resource status: {resource_status}"
         )
         return


 class K8sResourceLcm(GitOpsLcm):
     db_collection = "k8sresource"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("Resource Profile Create Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sresource", {"_id": profile_id})
         content["profile_type"] = "managed-resources"
         op_params = self.get_operation_params(content, op_id)
         self.db.set_one("k8sresource", {"_id": content["_id"]}, content)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "create_profile", op_id, op_params, content
             )
         self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
         self.logger.info(
             f"Resource Create Exit with resource status: {resource_status}"
         )
         return

     async def delete(self, params, order_id):
         self.logger.info("Resource delete Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sresource", {"_id": profile_id})
         op_params = self.get_operation_params(content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "delete_profile", op_id, op_params, content
             )

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 profile_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         if resource_status:
             content["state"] = "DELETED"
             profile_type = self.profile_type_mapping[content["profile_type"]]
             self.delete_profile_ksu(profile_id, profile_type)
             self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
             self.db.del_one(self.db_collection, {"_id": content["_id"]})
         self.logger.info(
             f"Resource Delete Exit with resource status: {resource_status}"
         )
         return


 class K8sInfraControllerLcm(GitOpsLcm):
     db_collection = "k8sinfra_controller"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("Infra controller Profile Create Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sinfra_controller", {"_id": profile_id})
         content["profile_type"] = "infra-controllers"
         op_params = self.get_operation_params(content, op_id)
         self.db.set_one("k8sinfra_controller", {"_id": content["_id"]}, content)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "create_profile", op_id, op_params, content
             )
         self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
         self.logger.info(
             f"Infra Controller Create Exit with resource status: {resource_status}"
         )
         return

     async def delete(self, params, order_id):
         self.logger.info("Infra controller delete Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sinfra_controller", {"_id": profile_id})
         op_params = self.get_operation_params(content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "delete_profile", op_id, op_params, content
             )

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 profile_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         if resource_status:
             content["state"] = "DELETED"
             profile_type = self.profile_type_mapping[content["profile_type"]]
             self.delete_profile_ksu(profile_id, profile_type)
             self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
             self.db.del_one(self.db_collection, {"_id": content["_id"]})
         self.logger.info(
             f"Infra Controller Delete Exit with resource status: {resource_status}"
         )
         return


 class K8sInfraConfigLcm(GitOpsLcm):
     db_collection = "k8sinfra_config"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("Infra config Profile Create Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sinfra_config", {"_id": profile_id})
         content["profile_type"] = "infra-configs"
         op_params = self.get_operation_params(content, op_id)
         self.db.set_one("k8sinfra_config", {"_id": content["_id"]}, content)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "create_profile", op_id, op_params, content
             )
         self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
         self.logger.info(
             f"Infra Config Create Exit with resource status: {resource_status}"
         )
         return

     async def delete(self, params, order_id):
         self.logger.info("Infra config delete Enter")

         op_id = params["operation_id"]
         profile_id = params["profile_id"]

         # To initialize the operation states
         self.initialize_operation(profile_id, op_id)

         content = self.db.get_one("k8sinfra_config", {"_id": profile_id})
         op_params = self.get_operation_params(content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_profile", op_id, op_params, content
         )
         self.logger.info("workflow_name is: {}".format(workflow_name))

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, content
         )

         if workflow_status:
             resource_status, content = await self.check_resource_and_update_db(
                 "delete_profile", op_id, op_params, content
             )

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 profile_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         if resource_status:
             content["state"] = "DELETED"
             profile_type = self.profile_type_mapping[content["profile_type"]]
             self.delete_profile_ksu(profile_id, profile_type)
             self.db.set_one(self.db_collection, {"_id": content["_id"]}, content)
             self.db.del_one(self.db_collection, {"_id": content["_id"]})
         self.logger.info(
             f"Infra Config Delete Exit with resource status: {resource_status}"
         )

         return


 class OkaLcm(GitOpsLcm):
     db_collection = "okas"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)

     async def create(self, params, order_id):
         self.logger.info("OKA Create Enter")
         op_id = params["operation_id"]
         oka_id = params["oka_id"]
         self.initialize_operation(oka_id, op_id)
         db_content = self.db.get_one(self.db_collection, {"_id": oka_id})
         op_params = self.get_operation_params(db_content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_oka", op_id, op_params, db_content
         )

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_content
         )

         if workflow_status:
             resource_status, db_content = await self.check_resource_and_update_db(
                 "create_oka", op_id, op_params, db_content
             )
         self.db.set_one(self.db_collection, {"_id": db_content["_id"]}, db_content)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "create_oka", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         self.logger.info(f"OKA Create Exit with resource status: {resource_status}")
         return

     async def edit(self, params, order_id):
         self.logger.info("OKA Edit Enter")
         op_id = params["operation_id"]
         oka_id = params["oka_id"]
         self.initialize_operation(oka_id, op_id)
         db_content = self.db.get_one(self.db_collection, {"_id": oka_id})
         op_params = self.get_operation_params(db_content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "update_oka", op_id, op_params, db_content
         )
         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_content
         )

         if workflow_status:
             resource_status, db_content = await self.check_resource_and_update_db(
                 "update_oka", op_id, op_params, db_content
             )
         self.db.set_one(self.db_collection, {"_id": db_content["_id"]}, db_content)
         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "update_oka", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         self.logger.info(f"OKA Update Exit with resource status: {resource_status}")
         return

     async def delete(self, params, order_id):
         self.logger.info("OKA delete Enter")
         op_id = params["operation_id"]
         oka_id = params["oka_id"]
         self.initialize_operation(oka_id, op_id)
         db_content = self.db.get_one(self.db_collection, {"_id": oka_id})
         op_params = self.get_operation_params(db_content, op_id)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_oka", op_id, op_params, db_content
         )
         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_content
         )

         if workflow_status:
             resource_status, db_content = await self.check_resource_and_update_db(
                 "delete_oka", op_id, op_params, db_content
             )

         force = params.get("force", False)
         if force:
             force_delete_status = self.check_force_delete_and_delete_from_db(
                 oka_id, workflow_status, resource_status, force
             )
             if force_delete_status:
                 return

         if resource_status:
             db_content["state"] == "DELETED"
             self.db.set_one(self.db_collection, {"_id": db_content["_id"]}, db_content)
             self.db.del_one(self.db_collection, {"_id": db_content["_id"]})
         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "delete_oka", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         self.logger.info(f"OKA Delete Exit with resource status: {resource_status}")
         return


 class KsuLcm(GitOpsLcm):
     db_collection = "ksus"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)
         self._workflows = {
             "create_ksus": {
                 "check_resource_function": self.check_create_ksus,
             },
             "delete_ksus": {
                 "check_resource_function": self.check_delete_ksus,
             },
         }

     def get_dbclusters_from_profile(self, profile_id, profile_type):
         cluster_list = []
         db_clusters = self.db.get_list("clusters")
         self.logger.info(f"Getting list of clusters for {profile_type} {profile_id}")
         for db_cluster in db_clusters:
             if profile_id in db_cluster.get(profile_type, []):
                 self.logger.info(
                     f"Profile {profile_id} found in cluster {db_cluster['name']}"
                 )
                 cluster_list.append(db_cluster)
         return cluster_list

     async def create(self, params, order_id):
         self.logger.info("ksu Create Enter")
         db_content = []
         op_params = []
         op_id = params["operation_id"]
         for ksu_id in params["ksus_list"]:
             self.logger.info("Ksu ID: {}".format(ksu_id))
             self.initialize_operation(ksu_id, op_id)
             db_ksu = self.db.get_one(self.db_collection, {"_id": ksu_id})
             self.logger.info("Db KSU: {}".format(db_ksu))
             db_content.append(db_ksu)
             ksu_params = {}
             ksu_params = self.get_operation_params(db_ksu, op_id)
             self.logger.info("Operation Params: {}".format(ksu_params))
             # Update ksu_params["profile"] with profile name and age-pubkey
             profile_type = ksu_params["profile"]["profile_type"]
             profile_id = ksu_params["profile"]["_id"]
             profile_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
             # db_profile is decrypted inline
             # No need to use decrypted_copy because db_profile won't be updated.
             self.decrypt_age_keys(db_profile)
             ksu_params["profile"]["name"] = db_profile["name"]
             ksu_params["profile"]["age_pubkey"] = db_profile.get("age_pubkey", "")
             # Update ksu_params["oka"] with sw_catalog_path (when missing)
             # TODO: remove this in favor of doing it in ODU workflow
             for oka in ksu_params["oka"]:
                 if "sw_catalog_path" not in oka:
                     oka_id = oka["_id"]
                     db_oka = self.db.get_one("okas", {"_id": oka_id})
                     oka_type = MAP_PROFILE[
                         db_oka.get("profile_type", "infra_controller_profiles")
                     ]
                     oka[
                         "sw_catalog_path"
                     ] = f"{oka_type}/{db_oka['git_name'].lower()}/templates"
             op_params.append(ksu_params)

         # A single workflow is launched for all KSUs
         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "create_ksus", op_id, op_params, db_content
         )
         # Update workflow status in all KSUs
         wf_status_list = []
         for db_ksu, ksu_params in zip(db_content, op_params):
             workflow_status = await self.check_workflow_and_update_db(
                 op_id, workflow_name, db_ksu
             )
             wf_status_list.append(workflow_status)
         # Update resource status in all KSUs
         # TODO: Is an operation correct if n KSUs are right and 1 is not OK?
         res_status_list = []
         for db_ksu, ksu_params, wf_status in zip(db_content, op_params, wf_status_list):
             if wf_status:
                 resource_status, db_ksu = await self.check_resource_and_update_db(
                     "create_ksus", op_id, ksu_params, db_ksu
                 )
             else:
                 resource_status = False
             res_status_list.append(resource_status)
             self.db.set_one(self.db_collection, {"_id": db_ksu["_id"]}, db_ksu)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "create_ksus", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         self.logger.info(f"KSU Create EXIT with Resource Status {res_status_list}")
         return

     async def edit(self, params, order_id):
         self.logger.info("ksu edit Enter")
         db_content = []
         op_params = []
         op_id = params["operation_id"]
         for ksu_id in params["ksus_list"]:
             self.initialize_operation(ksu_id, op_id)
             db_ksu = self.db.get_one("ksus", {"_id": ksu_id})
             db_content.append(db_ksu)
             ksu_params = {}
             ksu_params = self.get_operation_params(db_ksu, op_id)
             # Update ksu_params["profile"] with profile name and age-pubkey
             profile_type = ksu_params["profile"]["profile_type"]
             profile_id = ksu_params["profile"]["_id"]
             profile_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
             # db_profile is decrypted inline
             # No need to use decrypted_copy because db_profile won't be updated.
             self.decrypt_age_keys(db_profile)
             ksu_params["profile"]["name"] = db_profile["name"]
             ksu_params["profile"]["age_pubkey"] = db_profile.get("age_pubkey", "")
             # Update ksu_params["oka"] with sw_catalog_path (when missing)
             # TODO: remove this in favor of doing it in ODU workflow
             for oka in ksu_params["oka"]:
                 if "sw_catalog_path" not in oka:
                     oka_id = oka["_id"]
                     db_oka = self.db.get_one("okas", {"_id": oka_id})
                     oka_type = MAP_PROFILE[
                         db_oka.get("profile_type", "infra_controller_profiles")
                     ]
                     oka[
                         "sw_catalog_path"
                     ] = f"{oka_type}/{db_oka['git_name']}/templates"
             op_params.append(ksu_params)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "update_ksus", op_id, op_params, db_content
         )

         for db_ksu, ksu_params in zip(db_content, op_params):
             workflow_status = await self.check_workflow_and_update_db(
                 op_id, workflow_name, db_ksu
             )

             if workflow_status:
                 resource_status, db_ksu = await self.check_resource_and_update_db(
                     "update_ksus", op_id, ksu_params, db_ksu
                 )
                 db_ksu["name"] = ksu_params["name"]
                 db_ksu["description"] = ksu_params["description"]
                 db_ksu["profile"]["profile_type"] = ksu_params["profile"][
                     "profile_type"
                 ]
                 db_ksu["profile"]["_id"] = ksu_params["profile"]["_id"]
                 db_ksu["oka"] = ksu_params["oka"]
             self.db.set_one(self.db_collection, {"_id": db_ksu["_id"]}, db_ksu)

         # Clean items used in the workflow, no matter if the workflow succeeded
         clean_status, clean_msg = await self.odu.clean_items_workflow(
             "create_ksus", op_id, op_params, db_content
         )
         self.logger.info(
             f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
         )
         self.logger.info(f"KSU Update EXIT with Resource Status {resource_status}")
         return

     async def delete(self, params, order_id):
         self.logger.info("ksu delete Enter")
         db_content = []
         op_params = []
         op_id = params["operation_id"]
         for ksu_id in params["ksus_list"]:
             self.initialize_operation(ksu_id, op_id)
             db_ksu = self.db.get_one("ksus", {"_id": ksu_id})
             db_content.append(db_ksu)
             ksu_params = {}
             ksu_params["profile"] = {}
             ksu_params["profile"]["profile_type"] = db_ksu["profile"]["profile_type"]
             ksu_params["profile"]["_id"] = db_ksu["profile"]["_id"]
             # Update ksu_params["profile"] with profile name
             profile_type = ksu_params["profile"]["profile_type"]
             profile_id = ksu_params["profile"]["_id"]
             profile_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
             ksu_params["profile"]["name"] = db_profile["name"]
             op_params.append(ksu_params)

         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "delete_ksus", op_id, op_params, db_content
         )

         for db_ksu, ksu_params in zip(db_content, op_params):
             workflow_status = await self.check_workflow_and_update_db(
                 op_id, workflow_name, db_ksu
             )

             if workflow_status:
                 resource_status, db_ksu = await self.check_resource_and_update_db(
                     "delete_ksus", op_id, ksu_params, db_ksu
                 )

             force = params.get("force", False)
             if force:
                 force_delete_status = self.check_force_delete_and_delete_from_db(
                     db_ksu["_id"], workflow_status, resource_status, force
                 )
                 if force_delete_status:
                     return

             if resource_status:
                 db_ksu["state"] == "DELETED"
                 self.delete_ksu_dependency(db_ksu["_id"], db_ksu)
                 self.db.set_one(self.db_collection, {"_id": db_ksu["_id"]}, db_ksu)
                 self.db.del_one(self.db_collection, {"_id": db_ksu["_id"]})

         self.logger.info(f"KSU Delete Exit with resource status: {resource_status}")
         return

     async def clone(self, params, order_id):
         self.logger.info("ksu clone Enter")
         op_id = params["operation_id"]
         ksus_id = params["ksus_list"][0]
         self.initialize_operation(ksus_id, op_id)
         db_content = self.db.get_one(self.db_collection, {"_id": ksus_id})
         op_params = self.get_operation_params(db_content, op_id)
         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "clone_ksus", op_id, op_params, db_content
         )

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_content
         )

         if workflow_status:
             resource_status, db_content = await self.check_resource_and_update_db(
                 "clone_ksus", op_id, op_params, db_content
             )
         self.db.set_one(self.db_collection, {"_id": db_content["_id"]}, db_content)

         self.logger.info(f"KSU Clone Exit with resource status: {resource_status}")
         return

     async def move(self, params, order_id):
         self.logger.info("ksu move Enter")
         op_id = params["operation_id"]
         ksus_id = params["ksus_list"][0]
         self.initialize_operation(ksus_id, op_id)
         db_content = self.db.get_one(self.db_collection, {"_id": ksus_id})
         op_params = self.get_operation_params(db_content, op_id)
         workflow_res, workflow_name, _ = await self.odu.launch_workflow(
             "move_ksus", op_id, op_params, db_content
         )

         workflow_status = await self.check_workflow_and_update_db(
             op_id, workflow_name, db_content
         )

         if workflow_status:
             resource_status, db_content = await self.check_resource_and_update_db(
                 "move_ksus", op_id, op_params, db_content
             )
         self.db.set_one(self.db_collection, {"_id": db_content["_id"]}, db_content)

         self.logger.info(f"KSU Move Exit with resource status: {resource_status}")
         return

     async def check_create_ksus(self, op_id, op_params, content):
         self.logger.info(f"check_create_ksus Operation {op_id}. Params: {op_params}.")
         self.logger.debug(f"Content: {content}")
         db_ksu = content
         kustomization_name = db_ksu["git_name"].lower()
         oka_list = op_params["oka"]
         oka_item = oka_list[0]
         oka_params = oka_item.get("transformation", {})
         kustomization_ns = oka_params.get("kustomization_namespace", "flux-system")
         profile_id = op_params.get("profile", {}).get("_id")
         profile_type = op_params.get("profile", {}).get("profile_type")
         self.logger.info(
             f"Checking status of KSU {db_ksu['name']} for profile {profile_id}."
         )
         dbcluster_list = self.get_dbclusters_from_profile(profile_id, profile_type)
         if not dbcluster_list:
             self.logger.info(f"No clusters found for profile {profile_id}.")
         for db_cluster in dbcluster_list:
             try:
                 self.logger.info(
                     f"Checking status of KSU {db_ksu['name']} in cluster {db_cluster['name']}."
                 )
                 cluster_kubectl = self.cluster_kubectl(db_cluster)
                 checkings_list = [
                     {
                         "item": "kustomization",
                         "name": kustomization_name,
                         "namespace": kustomization_ns,
                         "condition": {
                             "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                             "value": "True",
                         },
                         "timeout": self._checkloop_kustomization_timeout,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
                     },
                 ]
                 self.logger.info(
                     f"Checking status of KSU {db_ksu['name']} for profile {profile_id}."
                 )
                 result, message = await self.common_check_list(
                     op_id, checkings_list, "ksus", db_ksu, kubectl_obj=cluster_kubectl
                 )
                 if not result:
                     return False, message
             except Exception as e:
                 self.logger.error(
                     f"Error checking KSU in cluster {db_cluster['name']}."
                 )
                 self.logger.error(e)
                 return False, f"Error checking KSU in cluster {db_cluster['name']}."
         return True, "OK"

     async def check_delete_ksus(self, op_id, op_params, content):
         self.logger.info(f"check_delete_ksus Operation {op_id}. Params: {op_params}.")
         self.logger.debug(f"Content: {content}")
         db_ksu = content
         kustomization_name = db_ksu["git_name"].lower()
         oka_list = db_ksu["oka"]
         oka_item = oka_list[0]
         oka_params = oka_item.get("transformation", {})
         kustomization_ns = oka_params.get("kustomization_namespace", "flux-system")
         profile_id = op_params.get("profile", {}).get("_id")
         profile_type = op_params.get("profile", {}).get("profile_type")
         self.logger.info(
             f"Checking status of KSU {db_ksu['name']} for profile {profile_id}."
         )
         dbcluster_list = self.get_dbclusters_from_profile(profile_id, profile_type)
         if not dbcluster_list:
             self.logger.info(f"No clusters found for profile {profile_id}.")
         for db_cluster in dbcluster_list:
             try:
                 self.logger.info(
                     f"Checking status of KSU in cluster {db_cluster['name']}."
                 )
                 cluster_kubectl = self.cluster_kubectl(db_cluster)
                 checkings_list = [
                     {
                         "item": "kustomization",
                         "name": kustomization_name,
                         "namespace": kustomization_ns,
                         "deleted": True,
                         "timeout": self._checkloop_kustomization_timeout,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.KUSTOMIZATION_DELETED",
                     },
                 ]
                 self.logger.info(
                     f"Checking status of KSU {db_ksu['name']} for profile {profile_id}."
                 )
                 result, message = await self.common_check_list(
                     op_id, checkings_list, "ksus", db_ksu, kubectl_obj=cluster_kubectl
                 )
                 if not result:
                     return False, message
             except Exception as e:
                 self.logger.error(
                     f"Error checking KSU in cluster {db_cluster['name']}."
                 )
                 self.logger.error(e)
                 return False, f"Error checking KSU in cluster {db_cluster['name']}."
         return True, "OK"


 class AppInstanceLcm(GitOpsLcm):
     db_collection = "appinstances"

     def __init__(self, msg, lcm_tasks, config):
         """
         Init, Connect to database, filesystem storage, and messaging
         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
         :return: None
         """
         super().__init__(msg, lcm_tasks, config)
         self._workflows = {
             "create_app": {
                 "check_resource_function": self.check_create_app,
             },
             "update_app": {
                 "check_resource_function": self.check_update_app,
             },
             "delete_app": {
                 "check_resource_function": self.check_delete_app,
             },
         }

     def get_dbclusters_from_profile(self, profile_id, profile_type):
         cluster_list = []
         db_clusters = self.db.get_list("clusters")
         self.logger.info(f"Getting list of clusters for {profile_type} {profile_id}")
         for db_cluster in db_clusters:
             if profile_id in db_cluster.get(profile_type, []):
                 self.logger.info(
                     f"Profile {profile_id} found in cluster {db_cluster['name']}"
                 )
                 cluster_list.append(db_cluster)
         return cluster_list

     def update_app_dependency(self, app_id, db_app):
         self.logger.info(f"Updating AppInstance dependencies for AppInstance {app_id}")
         oka_id = db_app.get("oka")
         if not oka_id:
             self.logger.info(f"No OKA associated with AppInstance {app_id}")
             return

         used_oka = []
         all_apps = self.db.get_list(self.db_collection, {})
         for app in all_apps:
             if app["_id"] != app_id:
                 app_oka_id = app["oka"]
                 if app_oka_id not in used_oka:
                     used_oka.append(app_oka_id)
         self.logger.info(f"Used OKA: {used_oka}")

         if oka_id not in used_oka:
             self.db.set_one(
                 "okas", {"_id": oka_id}, {"_admin.usageState": "NOT_IN_USE"}
             )
         return

     async def generic_operation(self, params, order_id, operation_name):
         self.logger.info(f"Generic operation. Operation name: {operation_name}")
         # self.logger.debug(f"Params: {params}")
         try:
             op_id = params["operation_id"]
             app_id = params["appinstance"]
             self.initialize_operation(app_id, op_id)
             db_app = self.db.get_one(self.db_collection, {"_id": app_id})
             # self.logger.debug("Db App: {}".format(db_app))

             # Initialize workflow_content with a copy of the db_app, decrypting fields to use in workflows
             db_app_copy = self.decrypted_copy(db_app)
             workflow_content = {
                 "app": db_app_copy,
             }

             # Update workflow_content with profile info
             profile_type = db_app["profile_type"]
             profile_id = db_app["profile"]
             profile_collection = self.profile_collection_mapping[profile_type]
             db_profile = self.db.get_one(profile_collection, {"_id": profile_id})
             # db_profile is decrypted inline
             # No need to use decrypted_copy because db_profile won't be updated.
             self.decrypt_age_keys(db_profile)
             workflow_content["profile"] = db_profile

             op_params = self.get_operation_params(db_app, op_id)
             if not op_params:
                 op_params = {}
             self.logger.debug("Operation Params: {}".format(op_params))

             # Get SW catalog path from op_params or from DB
             aux_dict = {}
             if operation_name == "create_app":
                 aux_dict = op_params
             else:
                 aux_dict = db_app
             sw_catalog_path = ""
             if "sw_catalog_path" in aux_dict:
                 sw_catalog_path = aux_dict.get("sw_catalog_path", "")
             elif "oka" in aux_dict:
                 oka_id = aux_dict["oka"]
                 db_oka = self.db.get_one("okas", {"_id": oka_id})
                 oka_type = MAP_PROFILE[
                     db_oka.get("profile_type", "infra_controller_profiles")
                 ]
                 sw_catalog_path = f"{oka_type}/{db_oka['git_name'].lower()}"
             else:
                 self.logger.error("SW Catalog path could not be determined.")
                 raise LcmException("SW Catalog path could not be determined.")
             self.logger.debug(f"SW Catalog path: {sw_catalog_path}")

             # Get model from Git repo
             # Clone the SW catalog repo
             repodir = self.cloneGitRepo(
                 repo_url=self._full_repo_sw_catalogs_url, branch="main"
             )
             model_file_path = os.path.join(repodir, sw_catalog_path, "model.yaml")
             if not os.path.exists(model_file_path):
                 self.logger.error(f"Model file not found at path: {model_file_path}")
                 raise LcmException(f"Model file not found at path: {model_file_path}")
             # Store the model content in workflow_content
             with open(model_file_path) as model_file:
                 workflow_content["model"] = yaml.safe_load(model_file.read())

             # A single workflow is launched for the App operation
             self.logger.debug("Launching workflow {}".format(operation_name))
             (
                 workflow_res,
                 workflow_name,
                 workflow_resources,
             ) = await self.odu.launch_workflow(
                 operation_name, op_id, op_params, workflow_content
             )

             if not workflow_res:
                 self.logger.error(f"Failed to launch workflow: {workflow_name}")
                 if operation_name == "create_app":
                     db_app["state"] = "FAILED_CREATION"
                 elif operation_name == "delete_app":
                     db_app["state"] = "FAILED_DELETION"
                 db_app["resourceState"] = "ERROR"
                 db_app = self.update_operation_history(
                     db_app, op_id, workflow_status=False, resource_status=None
                 )
                 self.db.set_one(self.db_collection, {"_id": db_app["_id"]}, db_app)
                 # Clean items used in the workflow, no matter if the workflow succeeded
                 clean_status, clean_msg = await self.odu.clean_items_workflow(
                     operation_name, op_id, op_params, workflow_content
                 )
                 self.logger.info(
                     f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
                 )
                 return

             # Update resources created in workflow
             db_app["app_model"] = workflow_resources.get("app_model", {})

             # Update workflow status in App
             workflow_status = await self.check_workflow_and_update_db(
                 op_id, workflow_name, db_app
             )
             # Update resource status in DB
             if workflow_status:
                 resource_status, db_app = await self.check_resource_and_update_db(
                     operation_name, op_id, op_params, db_app
                 )
             else:
                 resource_status = False
             self.db.set_one(self.db_collection, {"_id": db_app["_id"]}, db_app)

             # Clean items used in the workflow, no matter if the workflow succeeded
             clean_status, clean_msg = await self.odu.clean_items_workflow(
                 operation_name, op_id, op_params, workflow_content
             )
             self.logger.info(
                 f"clean_status is :{clean_status} and clean_msg is :{clean_msg}"
             )

             if operation_name == "delete_app":
                 force = params.get("force", False)
                 if force:
                     force_delete_status = self.check_force_delete_and_delete_from_db(
                         db_app["_id"], workflow_status, resource_status, force
                     )
                     if force_delete_status:
                         return
                 if resource_status:
                     db_app["state"] == "DELETED"
                     self.update_app_dependency(db_app["_id"], db_app)
                     self.db.del_one(self.db_collection, {"_id": db_app["_id"]})

             self.logger.info(
                 f"Generic app operation Exit {operation_name} with resource Status {resource_status}"
             )
             return
         except Exception as e:
             self.logger.debug(traceback.format_exc())
             self.logger.debug(f"Exception: {e}", exc_info=True)
             return

     async def create(self, params, order_id):
         self.logger.info("App Create Enter")
         return await self.generic_operation(params, order_id, "create_app")

     async def update(self, params, order_id):
         self.logger.info("App Edit Enter")
         return await self.generic_operation(params, order_id, "update_app")

     async def delete(self, params, order_id):
         self.logger.info("App Delete Enter")
         return await self.generic_operation(params, order_id, "delete_app")

     async def check_appinstance(self, op_id, op_params, content, deleted=False):
         self.logger.info(
             f"check_app_instance Operation {op_id}. Params: {op_params}. Deleted: {deleted}"
         )
         self.logger.debug(f"Content: {content}")
         db_app = content
         profile_id = db_app["profile"]
         profile_type = db_app["profile_type"]
         app_name = db_app["name"]
         self.logger.info(
             f"Checking status of AppInstance {app_name} for profile {profile_id}."
         )

         # TODO: read app_model and get kustomization name and namespace
         # app_model = db_app.get("app_model", {})
         kustomization_list = [
             {
                 "name": f"jenkins-{app_name}",
                 "namespace": "flux-system",
             }
         ]
         checkings_list = []
         if deleted:
             for kustomization in kustomization_list:
                 checkings_list.append(
                     {
                         "item": "kustomization",
                         "name": kustomization["name"].lower(),
                         "namespace": kustomization["namespace"],
                         "deleted": True,
                         "timeout": self._checkloop_kustomization_timeout,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.KUSTOMIZATION_DELETED",
                     }
                 )
         else:
             for kustomization in kustomization_list:
                 checkings_list.append(
                     {
                         "item": "kustomization",
                         "name": kustomization["name"].lower(),
                         "namespace": kustomization["namespace"],
                         "condition": {
                             "jsonpath_filter": "status.conditions[?(@.type=='Ready')].status",
                             "value": "True",
                         },
                         "timeout": self._checkloop_kustomization_timeout,
                         "enable": True,
                         "resourceState": "IN_PROGRESS.KUSTOMIZATION_READY",
                     }
                 )

         dbcluster_list = self.get_dbclusters_from_profile(profile_id, profile_type)
         if not dbcluster_list:
             self.logger.info(f"No clusters found for profile {profile_id}.")
         for db_cluster in dbcluster_list:
             try:
                 self.logger.info(
                     f"Checking status of AppInstance {app_name} in cluster {db_cluster['name']}."
                 )
                 cluster_kubectl = self.cluster_kubectl(db_cluster)
                 result, message = await self.common_check_list(
                     op_id,
                     checkings_list,
                     self.db_collection,
                     db_app,
                     kubectl_obj=cluster_kubectl,
                 )
                 if not result:
                     return False, message
             except Exception as e:
                 self.logger.error(
                     f"Error checking AppInstance in cluster {db_cluster['name']}."
                 )
                 self.logger.error(e)
                 return (
                     False,
                     f"Error checking AppInstance in cluster {db_cluster['name']}.",
                 )
         return True, "OK"

     async def check_create_app(self, op_id, op_params, content):
         self.logger.info(f"check_update_app Operation {op_id}. Params: {op_params}.")
         # self.logger.debug(f"Content: {content}")
         return await self.check_appinstance(op_id, op_params, content)

     async def check_update_app(self, op_id, op_params, content):
         self.logger.info(f"check_update_app Operation {op_id}. Params: {op_params}.")
         # self.logger.debug(f"Content: {content}")
         return await self.check_appinstance(op_id, op_params, content)

     async def check_delete_app(self, op_id, op_params, content):
         self.logger.info(f"check_delete_app Operation {op_id}. Params: {op_params}.")
         # self.logger.debug(f"Content: {content}")
         return await self.check_appinstance(op_id, op_params, content, deleted=True)