Add ModelConfig
[osm/N2VC.git] / n2vc / libjuju.py
index 9945c91..e5a8c61 100644 (file)
 
 import asyncio
 import logging
 
 import asyncio
 import logging
-from juju.controller import Controller
-from juju.client.connector import NoConnectionException
-from juju.client import client
+
 import time
 
 from juju.errors import JujuAPIError
 from juju.model import Model
 from juju.machine import Machine
 from juju.application import Application
 import time
 
 from juju.errors import JujuAPIError
 from juju.model import Model
 from juju.machine import Machine
 from juju.application import Application
-from juju.client._definitions import FullStatus
+from juju.unit import Unit
+from juju.client._definitions import (
+    FullStatus,
+    QueryApplicationOffersResults,
+    Cloud,
+    CloudCredential,
+)
+from juju.controller import Controller
+from juju.client import client
+from juju import tag
+
+from n2vc.config import ModelConfig
 from n2vc.juju_watcher import JujuModelWatcher
 from n2vc.provisioner import AsyncSSHProvisioner
 from n2vc.n2vc_conn import N2VCConnector
 from n2vc.exceptions import (
     JujuMachineNotFound,
     JujuApplicationNotFound,
 from n2vc.juju_watcher import JujuModelWatcher
 from n2vc.provisioner import AsyncSSHProvisioner
 from n2vc.n2vc_conn import N2VCConnector
 from n2vc.exceptions import (
     JujuMachineNotFound,
     JujuApplicationNotFound,
-    JujuModelAlreadyExists,
+    JujuLeaderUnitNotFound,
+    JujuActionNotFound,
     JujuControllerFailedConnecting,
     JujuApplicationExists,
     JujuControllerFailedConnecting,
     JujuApplicationExists,
+    JujuInvalidK8sConfiguration,
 )
 )
+from n2vc.utils import DB_DATA
+from osm_common.dbbase import DbException
+from kubernetes.client.configuration import Configuration
+
+RBAC_LABEL_KEY_NAME = "rbac-id"
 
 
 class Libjuju:
 
 
 class Libjuju:
@@ -48,8 +64,7 @@ class Libjuju:
         log: logging.Logger = None,
         db: dict = None,
         n2vc: N2VCConnector = None,
         log: logging.Logger = None,
         db: dict = None,
         n2vc: N2VCConnector = None,
-        apt_mirror: str = None,
-        enable_os_upgrade: bool = True,
+        model_config: ModelConfig = {},
     ):
         """
         Constructor
     ):
         """
         Constructor
@@ -67,63 +82,73 @@ class Libjuju:
         :param: enable_os_upgrade:      Enable OS Upgrade
         """
 
         :param: enable_os_upgrade:      Enable OS Upgrade
         """
 
-        self.endpoints = [endpoint]  # TODO: Store and get endpoints from DB
+        self.log = log or logging.getLogger("Libjuju")
+        self.db = db
+        db_endpoints = self._get_api_endpoints_db()
+        self.endpoints = None
+        if (db_endpoints and endpoint not in db_endpoints) or not db_endpoints:
+            self.endpoints = [endpoint]
+            self._update_api_endpoints_db(self.endpoints)
+        else:
+            self.endpoints = db_endpoints
         self.api_proxy = api_proxy
         self.username = username
         self.password = password
         self.cacert = cacert
         self.loop = loop or asyncio.get_event_loop()
         self.api_proxy = api_proxy
         self.username = username
         self.password = password
         self.cacert = cacert
         self.loop = loop or asyncio.get_event_loop()
-        self.log = log or logging.getLogger("Libjuju")
-        self.db = db
         self.n2vc = n2vc
 
         # Generate config for models
         self.n2vc = n2vc
 
         # Generate config for models
-        self.model_config = {}
-        if apt_mirror:
-            self.model_config["apt-mirror"] = apt_mirror
-        self.model_config["enable-os-refresh-update"] = enable_os_upgrade
-        self.model_config["enable-os-upgrade"] = enable_os_upgrade
+        self.model_config = model_config
 
 
-        self.reconnecting = asyncio.Lock(loop=self.loop)
+        self.loop.set_exception_handler(self.handle_exception)
         self.creating_model = asyncio.Lock(loop=self.loop)
 
         self.creating_model = asyncio.Lock(loop=self.loop)
 
-        self.models = set()
-        self.controller = Controller(loop=self.loop)
-
-        self.loop.run_until_complete(self.connect())
-
-    async def connect(self):
-        """Connect to the controller"""
-
-        self.log.debug("Connecting from controller")
-        await self.controller.connect(
-            endpoint=self.endpoints,
-            username=self.username,
-            password=self.password,
-            cacert=self.cacert,
-        )
-        e = self.controller.connection().endpoint
-        self.log.info("Connected to controller: {}".format(e))
-
-    async def disconnect(self):
-        """Disconnect from controller"""
+        self.log.debug("Libjuju initialized!")
 
 
-        self.log.debug("Disconnecting from controller")
-        await self.controller.disconnect()
-        self.log.info("Disconnected from controller")
+        self.health_check_task = self._create_health_check_task()
 
 
-    def controller_connected(self) -> bool:
-        """Check if the controller connection is open
+    def _create_health_check_task(self):
+        return self.loop.create_task(self.health_check())
 
 
-        :return: bool: True if connected, False if not connected
+    async def get_controller(self, timeout: float = 15.0) -> Controller:
         """
         """
+        Get controller
 
 
-        is_connected = False
+        :param: timeout: Time in seconds to wait for controller to connect
+        """
+        controller = None
         try:
         try:
-            is_connected = self.controller.connection().is_open
-        except NoConnectionException:
-            self.log.warning("VCA not connected")
-        return is_connected
+            controller = Controller(loop=self.loop)
+            await asyncio.wait_for(
+                controller.connect(
+                    endpoint=self.endpoints,
+                    username=self.username,
+                    password=self.password,
+                    cacert=self.cacert,
+                ),
+                timeout=timeout,
+            )
+            endpoints = await controller.api_endpoints
+            if self.endpoints != endpoints:
+                self.endpoints = endpoints
+                self._update_api_endpoints_db(self.endpoints)
+            return controller
+        except asyncio.CancelledError as e:
+            raise e
+        except Exception as e:
+            self.log.error(
+                "Failed connecting to controller: {}...".format(self.endpoints)
+            )
+            if controller:
+                await self.disconnect_controller(controller)
+            raise JujuControllerFailedConnecting(e)
+
+    async def disconnect(self):
+        """Disconnect"""
+        # Cancel health check task
+        self.health_check_task.cancel()
+        self.log.debug("Libjuju disconnected!")
 
     async def disconnect_model(self, model: Model):
         """
 
     async def disconnect_model(self, model: Model):
         """
@@ -131,114 +156,112 @@ class Libjuju:
 
         :param: model: Model that will be disconnected
         """
 
         :param: model: Model that will be disconnected
         """
-        try:
-            await model.disconnect()
-        except Exception:
-            pass
+        await model.disconnect()
 
 
-    async def _reconnect(
-        self,
-        retry: bool = False,
-        timeout: int = 5,
-        time_between_retries: int = 3,
-        maximum_retries: int = 0,
-    ):
+    async def disconnect_controller(self, controller: Controller):
         """
         """
-        Reconnect to the controller
-
-        :param: retry:                  Set it to True to retry if the connection fails
-        :param: time_between_retries:   Time in seconds between retries
-        :param: maximum_retries         Maximum retries. If not set, it will retry forever
+        Disconnect controller
 
 
-        :raises: Exception if cannot connect to the controller
+        :param: controller: Controller that will be disconnected
         """
         """
+        if controller:
+            await controller.disconnect()
 
 
-        if self.reconnecting.locked():
-            # Return if another function is trying to reconnect
-            return
-        async with self.reconnecting:
-            attempt = 0
-            while True:
-                try:
-                    await asyncio.wait_for(self.connect(), timeout=timeout)
-                    break
-                except asyncio.TimeoutError:
-                    self.log.error("Error reconnecting to controller: Timeout")
-                except Exception as e:
-                    self.log.error("Error reconnecting to controller: {}".format(e))
-
-                attempt += 1
-                maximum_retries_reached = attempt == maximum_retries
-
-                if not retry or maximum_retries_reached:
-                    raise JujuControllerFailedConnecting("Controller is not connected")
-                else:
-                    await asyncio.sleep(time_between_retries)
-
-    async def add_model(self, model_name: str, cloud_name: str):
+    async def add_model(self, model_name: str, cloud_name: str, credential_name=None):
         """
         Create model
 
         :param: model_name: Model name
         :param: cloud_name: Cloud name
         """
         Create model
 
         :param: model_name: Model name
         :param: cloud_name: Cloud name
+        :param: credential_name: Credential name to use for adding the model
+                                 If not specified, same name as the cloud will be used.
         """
 
         """
 
-        # Reconnect to the controller if not connected
-        if not self.controller_connected():
-            await self._reconnect()
-
-        # Raise exception if model already exists
-        if await self.model_exists(model_name):
-            raise JujuModelAlreadyExists("Model {} already exists.".format(model_name))
-
-        # Block until other workers have finished model creation
-        while self.creating_model.locked():
-            await asyncio.sleep(0.1)
-
-        # If the model exists, return it from the controller
-        if model_name in self.models:
-            return await self.get_model(model_name)
+        # Get controller
+        controller = await self.get_controller()
+        model = None
+        try:
+            # Block until other workers have finished model creation
+            while self.creating_model.locked():
+                await asyncio.sleep(0.1)
 
 
-        # Create the model
-        self.log.debug("Creating model {}".format(model_name))
-        async with self.creating_model:
-            model = await self.controller.add_model(
-                model_name,
-                config=self.model_config,
-                cloud_name=cloud_name,
-                credential_name=cloud_name,
-            )
-            await self.disconnect_model(model)
-            self.models.add(model_name)
+            # Create the model
+            async with self.creating_model:
+                if await self.model_exists(model_name, controller=controller):
+                    return
+                self.log.debug("Creating model {}".format(model_name))
+                model = await controller.add_model(
+                    model_name,
+                    config=self.model_config,
+                    cloud_name=cloud_name,
+                    credential_name=credential_name or cloud_name,
+                )
+        finally:
+            if model:
+                await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
 
-    async def get_model(self, model_name: str) -> Model:
+    async def get_model(
+        self, controller: Controller, model_name: str, id=None
+    ) -> Model:
         """
         Get model from controller
 
         """
         Get model from controller
 
+        :param: controller: Controller
         :param: model_name: Model name
 
         :return: Model: The created Juju model object
         """
         :param: model_name: Model name
 
         :return: Model: The created Juju model object
         """
+        return await controller.get_model(model_name)
 
 
-        # Check if controller is connected
-        if not self.controller_connected():
-            await self._reconnect()
-        return await self.controller.get_model(model_name)
-
-    async def model_exists(self, model_name: str) -> bool:
+    async def model_exists(
+        self, model_name: str, controller: Controller = None
+    ) -> bool:
         """
         Check if model exists
 
         """
         Check if model exists
 
+        :param: controller: Controller
         :param: model_name: Model name
 
         :return bool
         """
         :param: model_name: Model name
 
         :return bool
         """
+        need_to_disconnect = False
+
+        # Get controller if not passed
+        if not controller:
+            controller = await self.get_controller()
+            need_to_disconnect = True
 
 
-        # Check if controller is connected
-        if not self.controller_connected():
-            await self._reconnect()
+        # Check if model exists
+        try:
+            return model_name in await controller.list_models()
+        finally:
+            if need_to_disconnect:
+                await self.disconnect_controller(controller)
+
+    async def models_exist(self, model_names: [str]) -> (bool, list):
+        """
+        Check if models exists
 
 
-        return model_name in await self.controller.list_models()
+        :param: model_names: List of strings with model names
+
+        :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
+        """
+        if not model_names:
+            raise Exception(
+                "model_names must be a non-empty array. Given value: {}".format(
+                    model_names
+                )
+            )
+        non_existing_models = []
+        models = await self.list_models()
+        existing_models = list(set(models).intersection(model_names))
+        non_existing_models = list(set(model_names) - set(existing_models))
+
+        return (
+            len(non_existing_models) == 0,
+            non_existing_models,
+        )
 
     async def get_model_status(self, model_name: str) -> FullStatus:
         """
 
     async def get_model_status(self, model_name: str) -> FullStatus:
         """
@@ -248,10 +271,13 @@ class Libjuju:
 
         :return: Full status object
         """
 
         :return: Full status object
         """
-        model = await self.get_model(model_name)
-        status = await model.get_status()
-        await self.disconnect_model(model)
-        return status
+        controller = await self.get_controller()
+        model = await self.get_model(controller, model_name)
+        try:
+            return await model.get_status()
+        finally:
+            await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
     async def create_machine(
         self,
 
     async def create_machine(
         self,
@@ -261,6 +287,7 @@ class Libjuju:
         progress_timeout: float = None,
         total_timeout: float = None,
         series: str = "xenial",
         progress_timeout: float = None,
         total_timeout: float = None,
         series: str = "xenial",
+        wait: bool = True,
     ) -> (Machine, bool):
         """
         Create machine
     ) -> (Machine, bool):
         """
         Create machine
@@ -270,6 +297,8 @@ class Libjuju:
         :param: db_dict:            Dictionary with data of the DB to write the updates
         :param: progress_timeout:   Maximum time between two updates in the model
         :param: total_timeout:      Timeout for the entity to be active
         :param: db_dict:            Dictionary with data of the DB to write the updates
         :param: progress_timeout:   Maximum time between two updates in the model
         :param: total_timeout:      Timeout for the entity to be active
+        :param: series:             Series of the machine (xenial, bionic, focal, ...)
+        :param: wait:               Wait until machine is ready
 
         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
                                                 if the machine is new or it already existed
 
         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
                                                 if the machine is new or it already existed
@@ -281,8 +310,11 @@ class Libjuju:
             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
         )
 
             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
         )
 
+        # Get controller
+        controller = await self.get_controller()
+
         # Get model
         # Get model
-        model = await self.get_model(model_name)
+        model = await self.get_model(controller, model_name)
         try:
             if machine_id is not None:
                 self.log.debug(
         try:
             if machine_id is not None:
                 self.log.debug(
@@ -299,7 +331,7 @@ class Libjuju:
                             machine_id, model_name
                         )
                     )
                             machine_id, model_name
                         )
                     )
-                    machine = model.machines[machine_id]
+                    machine = machines[machine_id]
                 else:
                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 
                 else:
                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 
@@ -313,20 +345,29 @@ class Libjuju:
                 new = True
 
                 # Wait until the machine is ready
                 new = True
 
                 # Wait until the machine is ready
-                await JujuModelWatcher.wait_for(
-                    model=model,
-                    entity=machine,
-                    progress_timeout=progress_timeout,
-                    total_timeout=total_timeout,
-                    db_dict=db_dict,
-                    n2vc=self.n2vc,
+                self.log.debug(
+                    "Wait until machine {} is ready in model {}".format(
+                        machine.entity_id, model_name
+                    )
                 )
                 )
-        except Exception as e:
-            raise e
+                if wait:
+                    await JujuModelWatcher.wait_for(
+                        model=model,
+                        entity=machine,
+                        progress_timeout=progress_timeout,
+                        total_timeout=total_timeout,
+                        db_dict=db_dict,
+                        n2vc=self.n2vc,
+                    )
         finally:
             await self.disconnect_model(model)
         finally:
             await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
 
-        self.log.debug("Machine ready at {}".format(machine.dns_name))
+        self.log.debug(
+            "Machine {} ready at {} in model {}".format(
+                machine.entity_id, machine.dns_name, model_name
+            )
+        )
         return machine, new
 
     async def provision_machine(
         return machine, new
 
     async def provision_machine(
@@ -358,8 +399,11 @@ class Libjuju:
             )
         )
 
             )
         )
 
+        # Get controller
+        controller = await self.get_controller()
+
         # Get model
         # Get model
-        model = await self.get_model(model_name)
+        model = await self.get_model(controller, model_name)
 
         try:
             # Get provisioner
 
         try:
             # Get provisioner
@@ -395,7 +439,7 @@ class Libjuju:
                     connection=connection,
                     nonce=params.nonce,
                     machine_id=machine_id,
                     connection=connection,
                     nonce=params.nonce,
                     machine_id=machine_id,
-                    api=self.api_proxy,
+                    proxy=self.api_proxy,
                 )
             )
 
                 )
             )
 
@@ -413,6 +457,11 @@ class Libjuju:
                 self.log.error(msg=msg)
                 raise JujuMachineNotFound(msg)
 
                 self.log.error(msg=msg)
                 raise JujuMachineNotFound(msg)
 
+            self.log.debug(
+                "Wait until machine {} is ready in model {}".format(
+                    machine.entity_id, model_name
+                )
+            )
             await JujuModelWatcher.wait_for(
                 model=model,
                 entity=machine,
             await JujuModelWatcher.wait_for(
                 model=model,
                 entity=machine,
@@ -425,11 +474,36 @@ class Libjuju:
             raise e
         finally:
             await self.disconnect_model(model)
             raise e
         finally:
             await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
 
-        self.log.debug("Machine provisioned {}".format(machine_id))
+        self.log.debug(
+            "Machine provisioned {} in model {}".format(machine_id, model_name)
+        )
 
         return machine_id
 
 
         return machine_id
 
+    async def deploy(
+        self, uri: str, model_name: str, wait: bool = True, timeout: float = 3600
+    ):
+        """
+        Deploy bundle or charm: Similar to the juju CLI command `juju deploy`
+
+        :param: uri:            Path or Charm Store uri in which the charm or bundle can be found
+        :param: model_name:     Model name
+        :param: wait:           Indicates whether to wait or not until all applications are active
+        :param: timeout:        Time in seconds to wait until all applications are active
+        """
+        controller = await self.get_controller()
+        model = await self.get_model(controller, model_name)
+        try:
+            await model.deploy(uri)
+            if wait:
+                await JujuModelWatcher.wait_for_model(model, timeout=timeout)
+                self.log.debug("All units active in model {}".format(model_name))
+        finally:
+            await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
+
     async def deploy_charm(
         self,
         application_name: str,
     async def deploy_charm(
         self,
         application_name: str,
@@ -441,6 +515,7 @@ class Libjuju:
         total_timeout: float = None,
         config: dict = None,
         series: str = None,
         total_timeout: float = None,
         config: dict = None,
         series: str = None,
+        num_units: int = 1,
     ):
         """Deploy charm
 
     ):
         """Deploy charm
 
@@ -453,22 +528,27 @@ class Libjuju:
         :param: total_timeout:      Timeout for the entity to be active
         :param: config:             Config for the charm
         :param: series:             Series of the charm
         :param: total_timeout:      Timeout for the entity to be active
         :param: config:             Config for the charm
         :param: series:             Series of the charm
+        :param: num_units:          Number of units
 
         :return: (juju.application.Application): Juju application
         """
 
         :return: (juju.application.Application): Juju application
         """
+        self.log.debug(
+            "Deploying charm {} to machine {} in model ~{}".format(
+                application_name, machine_id, model_name
+            )
+        )
+        self.log.debug("charm: {}".format(path))
+
+        # Get controller
+        controller = await self.get_controller()
 
         # Get model
 
         # Get model
-        model = await self.get_model(model_name)
+        model = await self.get_model(controller, model_name)
 
         try:
             application = None
             if application_name not in model.applications:
 
         try:
             application = None
             if application_name not in model.applications:
-                self.log.debug(
-                    "Deploying charm {} to machine {} in model ~{}".format(
-                        application_name, machine_id, model_name
-                    )
-                )
-                self.log.debug("charm: {}".format(path))
+
                 if machine_id is not None:
                     if machine_id not in model.machines:
                         msg = "Machine {} not found in model".format(machine_id)
                 if machine_id is not None:
                     if machine_id not in model.machines:
                         msg = "Machine {} not found in model".format(machine_id)
@@ -487,6 +567,16 @@ class Libjuju:
                     config=config,
                 )
 
                     config=config,
                 )
 
+                self.log.debug(
+                    "Wait until application {} is ready in model {}".format(
+                        application_name, model_name
+                    )
+                )
+                if num_units > 1:
+                    for _ in range(num_units - 1):
+                        m, _ = await self.create_machine(model_name, wait=False)
+                        await application.add_unit(to=m.entity_id)
+
                 await JujuModelWatcher.wait_for(
                     model=model,
                     entity=application,
                 await JujuModelWatcher.wait_for(
                     model=model,
                     entity=application,
@@ -495,21 +585,22 @@ class Libjuju:
                     db_dict=db_dict,
                     n2vc=self.n2vc,
                 )
                     db_dict=db_dict,
                     n2vc=self.n2vc,
                 )
+                self.log.debug(
+                    "Application {} is ready in model {}".format(
+                        application_name, model_name
+                    )
+                )
             else:
             else:
-                raise JujuApplicationExists("Application {} exists".format(application_name))
-
-        except Exception as e:
-            raise e
+                raise JujuApplicationExists(
+                    "Application {} exists".format(application_name)
+                )
         finally:
             await self.disconnect_model(model)
         finally:
             await self.disconnect_model(model)
-
-        self.log.debug("application deployed")
+            await self.disconnect_controller(controller)
 
         return application
 
 
         return application
 
-    async def _get_application(
-        self, model: Model, application_name: str
-    ) -> Application:
+    def _get_application(self, model: Model, application_name: str) -> Application:
         """Get application
 
         :param: model:              Model object
         """Get application
 
         :param: model:              Model object
@@ -534,7 +625,6 @@ class Libjuju:
 
         :param: application_name:   Application name
         :param: model_name:         Model name
 
         :param: application_name:   Application name
         :param: model_name:         Model name
-        :param: cloud_name:         Cloud name
         :param: action_name:        Name of the action
         :param: db_dict:            Dictionary with data of the DB to write the updates
         :param: progress_timeout:   Maximum time between two updates in the model
         :param: action_name:        Name of the action
         :param: db_dict:            Dictionary with data of the DB to write the updates
         :param: progress_timeout:   Maximum time between two updates in the model
@@ -542,38 +632,58 @@ class Libjuju:
 
         :return: (str, str): (output and status)
         """
 
         :return: (str, str): (output and status)
         """
-        # Get model and observer
-        model = await self.get_model(model_name)
+        self.log.debug(
+            "Executing action {} using params {}".format(action_name, kwargs)
+        )
+        # Get controller
+        controller = await self.get_controller()
+
+        # Get model
+        model = await self.get_model(controller, model_name)
 
         try:
             # Get application
 
         try:
             # Get application
-            application = await self._get_application(
-                model, application_name=application_name,
+            application = self._get_application(
+                model,
+                application_name=application_name,
             )
             if application is None:
                 raise JujuApplicationNotFound("Cannot execute action")
 
             )
             if application is None:
                 raise JujuApplicationNotFound("Cannot execute action")
 
-            # Get unit
+            # Get leader unit
+            # Racing condition:
+            #   Ocassionally, self._get_leader_unit() will return None
+            #   because the leader elected hook has not been triggered yet.
+            #   Therefore, we are doing some retries. If it happens again,
+            #   re-open bug 1236
+            attempts = 3
+            time_between_retries = 10
             unit = None
             unit = None
-            for u in application.units:
-                if await u.is_leader_from_status():
-                    unit = u
+            for _ in range(attempts):
+                unit = await self._get_leader_unit(application)
+                if unit is None:
+                    await asyncio.sleep(time_between_retries)
+                else:
+                    break
             if unit is None:
             if unit is None:
-                raise Exception("Cannot execute action: leader unit not found")
+                raise JujuLeaderUnitNotFound(
+                    "Cannot execute action: leader unit not found"
+                )
 
             actions = await application.get_actions()
 
             if action_name not in actions:
 
             actions = await application.get_actions()
 
             if action_name not in actions:
-                raise Exception(
+                raise JujuActionNotFound(
                     "Action {} not in available actions".format(action_name)
                 )
 
                     "Action {} not in available actions".format(action_name)
                 )
 
-            self.log.debug(
-                "Executing action {} using params {}".format(action_name, kwargs)
-            )
             action = await unit.run_action(action_name, **kwargs)
 
             action = await unit.run_action(action_name, **kwargs)
 
-            # Register action with observer and wait for it to finish
+            self.log.debug(
+                "Wait until action {} is completed in application {} (model={})".format(
+                    action_name, application_name, model_name
+                )
+            )
             await JujuModelWatcher.wait_for(
                 model=model,
                 entity=action,
             await JujuModelWatcher.wait_for(
                 model=model,
                 entity=action,
@@ -582,17 +692,21 @@ class Libjuju:
                 db_dict=db_dict,
                 n2vc=self.n2vc,
             )
                 db_dict=db_dict,
                 n2vc=self.n2vc,
             )
+
             output = await model.get_action_output(action_uuid=action.entity_id)
             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
             status = (
                 status[action.entity_id] if action.entity_id in status else "failed"
             )
 
             output = await model.get_action_output(action_uuid=action.entity_id)
             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
             status = (
                 status[action.entity_id] if action.entity_id in status else "failed"
             )
 
-            self.log.debug("action completed with status: {}".format(action.status))
-        except Exception as e:
-            raise e
+            self.log.debug(
+                "Action {} completed with status {} in application {} (model={})".format(
+                    action_name, action.status, application_name, model_name
+                )
+            )
         finally:
             await self.disconnect_model(model)
         finally:
             await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
         return output, status
 
 
         return output, status
 
@@ -608,51 +722,76 @@ class Libjuju:
                 ...
             }
         """
                 ...
             }
         """
+        self.log.debug(
+            "Getting list of actions for application {}".format(application_name)
+        )
+
+        # Get controller
+        controller = await self.get_controller()
 
         # Get model
 
         # Get model
-        model = await self.get_model(model_name)
+        model = await self.get_model(controller, model_name)
 
 
-        # Get application
-        application = await self._get_application(
-            model, application_name=application_name,
-        )
+        try:
+            # Get application
+            application = self._get_application(
+                model,
+                application_name=application_name,
+            )
 
 
-        # Get list of actions
-        actions = await application.get_actions()
+            # Return list of actions
+            return await application.get_actions()
 
 
-        # Disconnect from model
-        await self.disconnect_model(model)
+        finally:
+            # Disconnect from model and controller
+            await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
 
-        return actions
+    async def get_metrics(self, model_name: str, application_name: str) -> dict:
+        """Get the metrics collected by the VCA.
+
+        :param model_name The name or unique id of the network service
+        :param application_name The name of the application
+        """
+        if not model_name or not application_name:
+            raise Exception("model_name and application_name must be non-empty strings")
+        metrics = {}
+        controller = await self.get_controller()
+        model = await self.get_model(controller, model_name)
+        try:
+            application = self._get_application(model, application_name)
+            if application is not None:
+                metrics = await application.get_metrics()
+        finally:
+            self.disconnect_model(model)
+            self.disconnect_controller(controller)
+        return metrics
 
     async def add_relation(
         self,
         model_name: str,
 
     async def add_relation(
         self,
         model_name: str,
-        application_name_1: str,
-        application_name_2: str,
-        relation_1: str,
-        relation_2: str,
+        endpoint_1: str,
+        endpoint_2: str,
     ):
         """Add relation
 
     ):
         """Add relation
 
-        :param: model_name:             Model name
-        :param: application_name_1      First application name
-        :param: application_name_2:     Second application name
-        :param: relation_1:             First relation name
-        :param: relation_2:             Second relation name
+        :param: model_name:     Model name
+        :param: endpoint_1      First endpoint name
+                                ("app:endpoint" format or directly the saas name)
+        :param: endpoint_2:     Second endpoint name (^ same format)
         """
 
         """
 
-        # Get model
-        model = await self.get_model(model_name)
+        self.log.debug("Adding relation: {} -> {}".format(endpoint_1, endpoint_2))
 
 
-        # Build relation strings
-        r1 = "{}:{}".format(application_name_1, relation_1)
-        r2 = "{}:{}".format(application_name_2, relation_2)
+        # Get controller
+        controller = await self.get_controller()
+
+        # Get model
+        model = await self.get_model(controller, model_name)
 
         # Add relation
 
         # Add relation
-        self.log.debug("Adding relation: {} -> {}".format(r1, r2))
         try:
         try:
-            await model.add_relation(relation1=r1, relation2=r2)
+            await model.add_relation(endpoint_1, endpoint_2)
         except JujuAPIError as e:
             if "not found" in e.message:
                 self.log.warning("Relation not found: {}".format(e.message))
         except JujuAPIError as e:
             if "not found" in e.message:
                 self.log.warning("Relation not found: {}".format(e.message))
@@ -664,76 +803,82 @@ class Libjuju:
             raise e
         finally:
             await self.disconnect_model(model)
             raise e
         finally:
             await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
 
 
-    async def destroy_model(
-        self, model_name: str, total_timeout: float,
+    async def consume(
+        self,
+        offer_url: str,
+        model_name: str,
     ):
     ):
+        """
+        Adds a remote offer to the model. Relations can be created later using "juju relate".
+
+        :param: offer_url:      Offer Url
+        :param: model_name:     Model name
+
+        :raises ParseError if there's a problem parsing the offer_url
+        :raises JujuError if remote offer includes and endpoint
+        :raises JujuAPIError if the operation is not successful
+        """
+        controller = await self.get_controller()
+        model = await controller.get_model(model_name)
+
+        try:
+            await model.consume(offer_url)
+        finally:
+            await self.disconnect_model(model)
+            await self.disconnect_controller(controller)
+
+    async def destroy_model(self, model_name: str, total_timeout: float):
         """
         Destroy model
 
         :param: model_name:     Model name
         :param: total_timeout:  Timeout
         """
         """
         Destroy model
 
         :param: model_name:     Model name
         :param: total_timeout:  Timeout
         """
-        model = await self.get_model(model_name)
-        uuid = model.info.uuid
 
 
-        # Destroy applications
-        for application_name in model.applications:
-            try:
-                await self.destroy_application(
-                    model, application_name=application_name,
-                )
-            except Exception as e:
-                self.log.error(
-                    "Error destroying application {} in model {}: {}".format(
-                        application_name, model_name, e
-                    )
-                )
+        controller = await self.get_controller()
+        model = None
+        try:
+            if not await self.model_exists(model_name, controller=controller):
+                return
 
 
-        # Destroy machines
-        machines = await model.get_machines()
-        for machine_id in machines:
-            try:
-                await self.destroy_machine(
-                    model, machine_id=machine_id, total_timeout=total_timeout,
-                )
-            except asyncio.CancelledError:
-                raise
-            except Exception:
-                pass
-
-        # Disconnect model
-        await self.disconnect_model(model)
-
-        # Destroy model
-        self.models.remove(model_name)
-        await self.controller.destroy_model(uuid)
-
-        # Wait until model is destroyed
-        self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
-        last_exception = ""
-
-        if total_timeout is None:
-            total_timeout = 3600
-        end = time.time() + total_timeout
-        while time.time() < end:
-            try:
-                models = await self.controller.list_models()
+            model = await self.get_model(controller, model_name)
+            self.log.debug("Destroying model {}".format(model_name))
+            uuid = model.info.uuid
+
+            # Destroy machines that are manually provisioned
+            # and still are in pending state
+            await self._destroy_pending_machines(model, only_manual=True)
+
+            # Disconnect model
+            await self.disconnect_model(model)
+
+            await controller.destroy_model(uuid, force=True, max_wait=0)
+
+            # Wait until model is destroyed
+            self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
+
+            if total_timeout is None:
+                total_timeout = 3600
+            end = time.time() + total_timeout
+            while time.time() < end:
+                models = await controller.list_models()
                 if model_name not in models:
                     self.log.debug(
                         "The model {} ({}) was destroyed".format(model_name, uuid)
                     )
                     return
                 if model_name not in models:
                     self.log.debug(
                         "The model {} ({}) was destroyed".format(model_name, uuid)
                     )
                     return
-            except asyncio.CancelledError:
-                raise
-            except Exception as e:
-                last_exception = e
-            await asyncio.sleep(5)
-        raise Exception(
-            "Timeout waiting for model {} to be destroyed {}".format(
-                model_name, last_exception
+                await asyncio.sleep(5)
+            raise Exception(
+                "Timeout waiting for model {} to be destroyed".format(model_name)
             )
             )
-        )
+        except Exception as e:
+            if model:
+                await self.disconnect_model(model)
+            raise e
+        finally:
+            await self.disconnect_controller(controller)
 
     async def destroy_application(self, model: Model, application_name: str):
         """
 
     async def destroy_application(self, model: Model, application_name: str):
         """
@@ -753,40 +898,49 @@ class Libjuju:
         else:
             self.log.warning("Application not found: {}".format(application_name))
 
         else:
             self.log.warning("Application not found: {}".format(application_name))
 
-    async def destroy_machine(
-        self, model: Model, machine_id: str, total_timeout: float = 3600
-    ):
+    async def _destroy_pending_machines(self, model: Model, only_manual: bool = False):
         """
         """
-        Destroy machine
+        Destroy pending machines in a given model
 
 
-        :param: model:          Model object
-        :param: machine_id:     Machine id
-        :param: total_timeout:  Timeout in seconds
+        :param: only_manual:    Bool that indicates only manually provisioned
+                                machines should be destroyed (if True), or that
+                                all pending machines should be destroyed
         """
         """
-        machines = await model.get_machines()
-        if machine_id in machines:
-            machine = model.machines[machine_id]
-            # TODO: change this by machine.is_manual when this is upstreamed:
-            # https://github.com/juju/python-libjuju/pull/396
-            if "instance-id" in machine.safe_data and machine.safe_data[
-                "instance-id"
-            ].startswith("manual:"):
+        status = await model.get_status()
+        for machine_id in status.machines:
+            machine_status = status.machines[machine_id]
+            if machine_status.agent_status.status == "pending":
+                if only_manual and not machine_status.instance_id.startswith("manual:"):
+                    break
+                machine = model.machines[machine_id]
                 await machine.destroy(force=True)
 
                 await machine.destroy(force=True)
 
-                # max timeout
-                end = time.time() + total_timeout
-
-                # wait for machine removal
-                machines = await model.get_machines()
-                while machine_id in machines and time.time() < end:
-                    self.log.debug(
-                        "Waiting for machine {} is destroyed".format(machine_id)
-                    )
-                    await asyncio.sleep(0.5)
-                    machines = await model.get_machines()
-                self.log.debug("Machine destroyed: {}".format(machine_id))
-        else:
-            self.log.debug("Machine not found: {}".format(machine_id))
+    # async def destroy_machine(
+    #     self, model: Model, machine_id: str, total_timeout: float = 3600
+    # ):
+    #     """
+    #     Destroy machine
+
+    #     :param: model:          Model object
+    #     :param: machine_id:     Machine id
+    #     :param: total_timeout:  Timeout in seconds
+    #     """
+    #     machines = await model.get_machines()
+    #     if machine_id in machines:
+    #         machine = machines[machine_id]
+    #         await machine.destroy(force=True)
+    #         # max timeout
+    #         end = time.time() + total_timeout
+
+    #         # wait for machine removal
+    #         machines = await model.get_machines()
+    #         while machine_id in machines and time.time() < end:
+    #             self.log.debug("Waiting for machine {} is destroyed".format(machine_id))
+    #             await asyncio.sleep(0.5)
+    #             machines = await model.get_machines()
+    #         self.log.debug("Machine destroyed: {}".format(machine_id))
+    #     else:
+    #         self.log.debug("Machine not found: {}".format(machine_id))
 
     async def configure_application(
         self, model_name: str, application_name: str, config: dict = None
 
     async def configure_application(
         self, model_name: str, application_name: str, config: dict = None
@@ -797,10 +951,279 @@ class Libjuju:
         :param: application_name:   Application name
         :param: config:             Config to apply to the charm
         """
         :param: application_name:   Application name
         :param: config:             Config to apply to the charm
         """
+        self.log.debug("Configuring application {}".format(application_name))
+
         if config:
         if config:
-            model = await self.get_model(model_name)
-            application = await self._get_application(
-                model, application_name=application_name,
-            )
-            await application.set_config(config)
-            await self.disconnect_model(model)
+            controller = await self.get_controller()
+            model = None
+            try:
+                model = await self.get_model(controller, model_name)
+                application = self._get_application(
+                    model,
+                    application_name=application_name,
+                )
+                await application.set_config(config)
+            finally:
+                if model:
+                    await self.disconnect_model(model)
+                await self.disconnect_controller(controller)
+
+    def _get_api_endpoints_db(self) -> [str]:
+        """
+        Get API Endpoints from DB
+
+        :return: List of API endpoints
+        """
+        self.log.debug("Getting endpoints from database")
+
+        juju_info = self.db.get_one(
+            DB_DATA.api_endpoints.table,
+            q_filter=DB_DATA.api_endpoints.filter,
+            fail_on_empty=False,
+        )
+        if juju_info and DB_DATA.api_endpoints.key in juju_info:
+            return juju_info[DB_DATA.api_endpoints.key]
+
+    def _update_api_endpoints_db(self, endpoints: [str]):
+        """
+        Update API endpoints in Database
+
+        :param: List of endpoints
+        """
+        self.log.debug("Saving endpoints {} in database".format(endpoints))
+
+        juju_info = self.db.get_one(
+            DB_DATA.api_endpoints.table,
+            q_filter=DB_DATA.api_endpoints.filter,
+            fail_on_empty=False,
+        )
+        # If it doesn't, then create it
+        if not juju_info:
+            try:
+                self.db.create(
+                    DB_DATA.api_endpoints.table,
+                    DB_DATA.api_endpoints.filter,
+                )
+            except DbException as e:
+                # Racing condition: check if another N2VC worker has created it
+                juju_info = self.db.get_one(
+                    DB_DATA.api_endpoints.table,
+                    q_filter=DB_DATA.api_endpoints.filter,
+                    fail_on_empty=False,
+                )
+                if not juju_info:
+                    raise e
+        self.db.set_one(
+            DB_DATA.api_endpoints.table,
+            DB_DATA.api_endpoints.filter,
+            {DB_DATA.api_endpoints.key: endpoints},
+        )
+
+    def handle_exception(self, loop, context):
+        # All unhandled exceptions by libjuju are handled here.
+        pass
+
+    async def health_check(self, interval: float = 300.0):
+        """
+        Health check to make sure controller and controller_model connections are OK
+
+        :param: interval: Time in seconds between checks
+        """
+        controller = None
+        while True:
+            try:
+                controller = await self.get_controller()
+                # self.log.debug("VCA is alive")
+            except Exception as e:
+                self.log.error("Health check to VCA failed: {}".format(e))
+            finally:
+                await self.disconnect_controller(controller)
+            await asyncio.sleep(interval)
+
+    async def list_models(self, contains: str = None) -> [str]:
+        """List models with certain names
+
+        :param: contains:   String that is contained in model name
+
+        :retur: [models] Returns list of model names
+        """
+
+        controller = await self.get_controller()
+        try:
+            models = await controller.list_models()
+            if contains:
+                models = [model for model in models if contains in model]
+            return models
+        finally:
+            await self.disconnect_controller(controller)
+
+    async def list_offers(self, model_name: str) -> QueryApplicationOffersResults:
+        """List models with certain names
+
+        :param: model_name: Model name
+
+        :return:            Returns list of offers
+        """
+
+        controller = await self.get_controller()
+        try:
+            return await controller.list_offers(model_name)
+        finally:
+            await self.disconnect_controller(controller)
+
+    async def add_k8s(
+        self,
+        name: str,
+        rbac_id: str,
+        token: str,
+        client_cert_data: str,
+        configuration: Configuration,
+        storage_class: str,
+        credential_name: str = None,
+    ):
+        """
+        Add a Kubernetes cloud to the controller
+
+        Similar to the `juju add-k8s` command in the CLI
+
+        :param: name:               Name for the K8s cloud
+        :param: configuration:      Kubernetes configuration object
+        :param: storage_class:      Storage Class to use in the cloud
+        :param: credential_name:    Storage Class to use in the cloud
+        """
+
+        if not storage_class:
+            raise Exception("storage_class must be a non-empty string")
+        if not name:
+            raise Exception("name must be a non-empty string")
+        if not configuration:
+            raise Exception("configuration must be provided")
+
+        endpoint = configuration.host
+        credential = self.get_k8s_cloud_credential(
+            configuration,
+            client_cert_data,
+            token,
+        )
+        credential.attrs[RBAC_LABEL_KEY_NAME] = rbac_id
+        cloud = client.Cloud(
+            type_="kubernetes",
+            auth_types=[credential.auth_type],
+            endpoint=endpoint,
+            ca_certificates=[client_cert_data],
+            config={
+                "operator-storage": storage_class,
+                "workload-storage": storage_class,
+            },
+        )
+
+        return await self.add_cloud(
+            name, cloud, credential, credential_name=credential_name
+        )
+
+    def get_k8s_cloud_credential(
+        self,
+        configuration: Configuration,
+        client_cert_data: str,
+        token: str = None,
+    ) -> client.CloudCredential:
+        attrs = {}
+        # TODO: Test with AKS
+        key = None  # open(configuration.key_file, "r").read()
+        username = configuration.username
+        password = configuration.password
+
+        if client_cert_data:
+            attrs["ClientCertificateData"] = client_cert_data
+        if key:
+            attrs["ClientKeyData"] = key
+        if token:
+            if username or password:
+                raise JujuInvalidK8sConfiguration("Cannot set both token and user/pass")
+            attrs["Token"] = token
+
+        auth_type = None
+        if key:
+            auth_type = "oauth2"
+            if client_cert_data:
+                auth_type = "oauth2withcert"
+            if not token:
+                raise JujuInvalidK8sConfiguration(
+                    "missing token for auth type {}".format(auth_type)
+                )
+        elif username:
+            if not password:
+                self.log.debug(
+                    "credential for user {} has empty password".format(username)
+                )
+            attrs["username"] = username
+            attrs["password"] = password
+            if client_cert_data:
+                auth_type = "userpasswithcert"
+            else:
+                auth_type = "userpass"
+        elif client_cert_data and token:
+            auth_type = "certificate"
+        else:
+            raise JujuInvalidK8sConfiguration("authentication method not supported")
+        return client.CloudCredential(auth_type=auth_type, attrs=attrs)
+
+    async def add_cloud(
+        self,
+        name: str,
+        cloud: Cloud,
+        credential: CloudCredential = None,
+        credential_name: str = None,
+    ) -> Cloud:
+        """
+        Add cloud to the controller
+
+        :param: name:               Name of the cloud to be added
+        :param: cloud:              Cloud object
+        :param: credential:         CloudCredentials object for the cloud
+        :param: credential_name:    Credential name.
+                                    If not defined, cloud of the name will be used.
+        """
+        controller = await self.get_controller()
+        try:
+            _ = await controller.add_cloud(name, cloud)
+            if credential:
+                await controller.add_credential(
+                    credential_name or name, credential=credential, cloud=name
+                )
+            # Need to return the object returned by the controller.add_cloud() function
+            # I'm returning the original value now until this bug is fixed:
+            #   https://github.com/juju/python-libjuju/issues/443
+            return cloud
+        finally:
+            await self.disconnect_controller(controller)
+
+    async def remove_cloud(self, name: str):
+        """
+        Remove cloud
+
+        :param: name:   Name of the cloud to be removed
+        """
+        controller = await self.get_controller()
+        try:
+            await controller.remove_cloud(name)
+        finally:
+            await self.disconnect_controller(controller)
+
+    async def _get_leader_unit(self, application: Application) -> Unit:
+        unit = None
+        for u in application.units:
+            if await u.is_leader_from_status():
+                unit = u
+                break
+        return unit
+
+    async def get_cloud_credentials(self, cloud_name: str, credential_name: str):
+        controller = await self.get_controller()
+        try:
+            facade = client.CloudFacade.from_connection(controller.connection())
+            cloud_cred_tag = tag.credential(cloud_name, self.username, credential_name)
+            params = [client.Entity(cloud_cred_tag)]
+            return (await facade.Credential(params)).results
+        finally:
+            await self.disconnect_controller(controller)