Add deleting lock to K8sJujuConnector

[osm/N2VC.git] / n2vc / k8s_juju_conn.py

diff --git a/n2vc/k8s_juju_conn.py b/n2vc/k8s_juju_conn.py
index d9dc607..737cac6 100644 (file)
--- a/n2vc/k8s_juju_conn.py
+++ b/n2vc/k8s_juju_conn.py
@@ -13,6 +13,7 @@
 #     limitations under the License.
 
 import asyncio
+from typing import Union
 import os
 import uuid
 import yaml
@@ -77,6 +78,7 @@ class K8sJujuConnector(K8sConnector):
         db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri")
         self._store = MotorStore(db_uri)
         self.loading_libjuju = asyncio.Lock(loop=self.loop)
+        self.uninstall_locks = {}
 
         self.log.debug("K8S Juju connector initialized")
         # TODO: Remove these commented lines:
@@ -122,10 +124,12 @@ class K8sJujuConnector(K8sConnector):
         # if it fails in the middle of the process
         cleanup_data = []
         try:
+            self.log.debug("Initializing K8s cluster for juju")
             kubectl.create_cluster_role(
                 name=metadata_name,
                 labels=labels,
             )
+            self.log.debug("Cluster role created")
             cleanup_data.append(
                 {
                     "delete": kubectl.delete_cluster_role,
@@ -137,6 +141,7 @@ class K8sJujuConnector(K8sConnector):
                 name=metadata_name,
                 labels=labels,
             )
+            self.log.debug("Service account created")
             cleanup_data.append(
                 {
                     "delete": kubectl.delete_service_account,
@@ -148,6 +153,7 @@ class K8sJujuConnector(K8sConnector):
                 name=metadata_name,
                 labels=labels,
             )
+            self.log.debug("Role binding created")
             cleanup_data.append(
                 {
                     "delete": kubectl.delete_service_account,
@@ -159,6 +165,7 @@ class K8sJujuConnector(K8sConnector):
             )
 
             default_storage_class = kubectl.get_default_storage_class()
+            self.log.debug("Default storage class: {}".format(default_storage_class))
             await libjuju.add_k8s(
                 name=cluster_uuid,
                 rbac_id=rbac_id,
@@ -168,9 +175,10 @@ class K8sJujuConnector(K8sConnector):
                 storage_class=default_storage_class,
                 credential_name=self._get_credential_name(cluster_uuid),
             )
+            self.log.debug("K8s cluster added to juju controller")
             return cluster_uuid, True
         except Exception as e:
-            self.log.error("Error initializing k8scluster: {}".format(e))
+            self.log.error("Error initializing k8scluster: {}".format(e), exc_info=True)
             if len(cleanup_data) > 0:
                 self.log.debug("Cleaning up created resources in k8s cluster...")
                 for item in cleanup_data:
@@ -187,6 +195,9 @@ class K8sJujuConnector(K8sConnector):
         name: str,
         url: str,
         _type: str = "charm",
+        cert: str = None,
+        user: str = None,
+        password: str = None,
     ):
         raise MethodNotImplemented()
 
@@ -497,18 +508,30 @@ class K8sJujuConnector(K8sConnector):
         """
 
         self.log.debug("[uninstall] Destroying model")
-        libjuju = await self._get_libjuju(kwargs.get("vca_id"))
 
-        await libjuju.destroy_model(kdu_instance, total_timeout=3600)
+        will_not_delete = False
+        if kdu_instance not in self.uninstall_locks:
+            self.uninstall_locks[kdu_instance] = asyncio.Lock(loop=self.loop)
+        delete_lock = self.uninstall_locks[kdu_instance]
+
+        while delete_lock.locked():
+            will_not_delete = True
+            await asyncio.sleep(0.1)
+
+        if will_not_delete:
+            self.log.info("Model {} deleted by another worker.".format(kdu_instance))
+            return True
 
-        # self.log.debug("[uninstall] Model destroyed and disconnecting")
-        # await controller.disconnect()
+        try:
+            async with delete_lock:
+                libjuju = await self._get_libjuju(kwargs.get("vca_id"))
+
+                await libjuju.destroy_model(kdu_instance, total_timeout=3600)
+        finally:
+            self.uninstall_locks.pop(kdu_instance)
 
+        self.log.debug(f"[uninstall] Model {kdu_instance} destroyed")
         return True
-        # TODO: Remove these commented lines
-        # if not self.authenticated:
-        #     self.log.debug("[uninstall] Connecting to controller")
-        #     await self.login(cluster_uuid)
 
     async def exec_primitive(
         self,
@@ -647,7 +670,7 @@ class K8sJujuConnector(K8sConnector):
         complete_status: bool = False,
         yaml_format: bool = False,
         **kwargs,
-    ) -> dict:
+    ) -> Union[str, dict]:
         """Get the status of the KDU
 
         Get the current status of the KDU instance.