Fix bug 2060: Add logic to delete models gracefully
[osm/N2VC.git] / n2vc / libjuju.py
index b897bc6..4631859 100644 (file)
@@ -33,6 +33,7 @@ from juju.controller import Controller
 from juju.client import client
 from juju import tag
 
+from n2vc.definitions import Offer, RelationEndpoint
 from n2vc.juju_watcher import JujuModelWatcher
 from n2vc.provisioner import AsyncSSHProvisioner
 from n2vc.n2vc_conn import N2VCConnector
@@ -94,7 +95,7 @@ class Libjuju:
         """
         controller = None
         try:
-            controller = Controller(loop=self.loop)
+            controller = Controller()
             await asyncio.wait_for(
                 controller.connect(
                     endpoint=self.vca_connection.data.endpoints,
@@ -121,7 +122,10 @@ class Libjuju:
             )
             if controller:
                 await self.disconnect_controller(controller)
-            raise JujuControllerFailedConnecting(e)
+
+            raise JujuControllerFailedConnecting(
+                f"Error connecting to Juju controller: {e}"
+            )
 
     async def disconnect(self):
         """Disconnect"""
@@ -791,12 +795,48 @@ class Libjuju:
                 raise JujuApplicationExists(
                     "Application {} exists".format(application_name)
                 )
+        except juju.errors.JujuError as e:
+            if "already exists" in e.message:
+                raise JujuApplicationExists(
+                    "Application {} exists".format(application_name)
+                )
+            else:
+                raise e
         finally:
             await self.disconnect_model(model)
             await self.disconnect_controller(controller)
 
         return application
 
+    async def resolve(self, model_name: str):
+
+        controller = await self.get_controller()
+        model = await self.get_model(controller, model_name)
+        all_units_active = False
+        try:
+            while not all_units_active:
+                all_units_active = True
+                for application_name, application in model.applications.items():
+                    if application.status == "error":
+                        for unit in application.units:
+                            if unit.workload_status == "error":
+                                self.log.debug(
+                                    "Model {}, Application {}, Unit {} in error state, resolving".format(
+                                        model_name, application_name, unit.entity_id
+                                    )
+                                )
+                                try:
+                                    await unit.resolved(retry=False)
+                                    all_units_active = False
+                                except Exception:
+                                    pass
+
+                if not all_units_active:
+                    await asyncio.sleep(5)
+        finally:
+            await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
+
     async def scale_application(
         self,
         model_name: str,
@@ -1121,28 +1161,71 @@ class Libjuju:
             await self.disconnect_model(model)
             await self.disconnect_controller(controller)
 
+    async def offer(self, endpoint: RelationEndpoint) -> Offer:
+        """
+        Create an offer from a RelationEndpoint
+
+        :param: endpoint: Relation endpoint
+
+        :return: Offer object
+        """
+        model_name = endpoint.model_name
+        offer_name = f"{endpoint.application_name}-{endpoint.endpoint_name}"
+        controller = await self.get_controller()
+        model = None
+        try:
+            model = await self.get_model(controller, model_name)
+            await model.create_offer(endpoint.endpoint, offer_name=offer_name)
+            offer_list = await self._list_offers(model_name, offer_name=offer_name)
+            if offer_list:
+                return Offer(offer_list[0].offer_url)
+            else:
+                raise Exception("offer was not created")
+        except juju.errors.JujuError as e:
+            if "application offer already exists" not in e.message:
+                raise e
+        finally:
+            if model:
+                self.disconnect_model(model)
+            self.disconnect_controller(controller)
+
     async def consume(
         self,
-        offer_url: str,
         model_name: str,
-    ):
+        offer: Offer,
+        provider_libjuju: "Libjuju",
+    ) -> str:
         """
-        Adds a remote offer to the model. Relations can be created later using "juju relate".
+        Consumes a remote offer in the model. Relations can be created later using "juju relate".
 
-        :param: offer_url:      Offer Url
-        :param: model_name:     Model name
+        :param: model_name:             Model name
+        :param: offer:                  Offer object to consume
+        :param: provider_libjuju:       Libjuju object of the provider endpoint
 
         :raises ParseError if there's a problem parsing the offer_url
         :raises JujuError if remote offer includes and endpoint
         :raises JujuAPIError if the operation is not successful
+
+        :returns: Saas name. It is the application name in the model that reference the remote application.
         """
+        saas_name = f'{offer.name}-{offer.model_name.replace("-", "")}'
+        if offer.vca_id:
+            saas_name = f"{saas_name}-{offer.vca_id}"
         controller = await self.get_controller()
-        model = await controller.get_model(model_name)
-
+        model = None
+        provider_controller = None
         try:
-            await model.consume(offer_url)
+            model = await controller.get_model(model_name)
+            provider_controller = await provider_libjuju.get_controller()
+            await model.consume(
+                offer.url, application_alias=saas_name, controller=provider_controller
+            )
+            return saas_name
         finally:
-            await self.disconnect_model(model)
+            if model:
+                await self.disconnect_model(model)
+            if provider_controller:
+                await provider_libjuju.disconnect_controller(provider_controller)
             await self.disconnect_controller(controller)
 
     async def destroy_model(self, model_name: str, total_timeout: float = 1800):
@@ -1167,18 +1250,23 @@ class Libjuju:
             await self._destroy_pending_machines(model, only_manual=True)
             await self.disconnect_model(model)
 
-            await self._destroy_model(
-                model_name,
-                controller,
+            await asyncio.wait_for(
+                self._destroy_model(model_name, controller),
                 timeout=total_timeout,
             )
+        except Exception as e:
+            if not await self.model_exists(model_name, controller=controller):
+                return
+            raise e
         finally:
             if model:
                 await self.disconnect_model(model)
             await self.disconnect_controller(controller)
 
     async def _destroy_model(
-        self, model_name: str, controller: Controller, timeout: float = 1800
+        self,
+        model_name: str,
+        controller: Controller,
     ):
         """
         Destroy model from controller
@@ -1188,21 +1276,37 @@ class Libjuju:
         :param: timeout: Timeout in seconds
         """
 
-        async def _destroy_model_loop(model_name: str, controller: Controller):
-            while await self.model_exists(model_name, controller=controller):
+        async def _destroy_model_gracefully(model_name: str, controller: Controller):
+            self.log.info(f"Gracefully deleting model {model_name}")
+            resolved = False
+            while model_name in await controller.list_models():
+                if not resolved:
+                    await self.resolve(model_name)
+                    resolved = True
+                await controller.destroy_model(model_name, destroy_storage=True)
+
+                await asyncio.sleep(5)
+            self.log.info(f"Model {model_name} deleted gracefully")
+
+        async def _destroy_model_forcefully(model_name: str, controller: Controller):
+            self.log.info(f"Forcefully deleting model {model_name}")
+            while model_name in await controller.list_models():
                 await controller.destroy_model(
-                    model_name, destroy_storage=True, force=True, max_wait=0
+                    model_name, destroy_storage=True, force=True, max_wait=60
                 )
                 await asyncio.sleep(5)
+            self.log.info(f"Model {model_name} deleted forcefully")
 
         try:
             await asyncio.wait_for(
-                _destroy_model_loop(model_name, controller), timeout=timeout
+                _destroy_model_gracefully(model_name, controller), timeout=120
             )
         except asyncio.TimeoutError:
-            raise Exception(
-                "Timeout waiting for model {} to be destroyed".format(model_name)
-            )
+            await _destroy_model_forcefully(model_name, controller)
+        except juju.errors.JujuError as e:
+            if any("has been removed" in error for error in e.errors):
+                return
+            raise e
 
     async def destroy_application(
         self, model_name: str, application_name: str, total_timeout: float
@@ -1339,17 +1443,29 @@ class Libjuju:
         finally:
             await self.disconnect_controller(controller)
 
-    async def list_offers(self, model_name: str) -> QueryApplicationOffersResults:
-        """List models with certain names
+    async def _list_offers(
+        self, model_name: str, offer_name: str = None
+    ) -> QueryApplicationOffersResults:
+        """
+        List offers within a model
 
         :param: model_name: Model name
+        :param: offer_name: Offer name to filter.
 
-        :return:            Returns list of offers
+        :return: Returns application offers results in the model
         """
 
         controller = await self.get_controller()
         try:
-            return await controller.list_offers(model_name)
+            offers = (await controller.list_offers(model_name)).results
+            if offer_name:
+                matching_offer = []
+                for offer in offers:
+                    if offer.offer_name == offer_name:
+                        matching_offer.append(offer)
+                        break
+                offers = matching_offer
+            return offers
         finally:
             await self.disconnect_controller(controller)