n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuLeaderUnitNotFound,
  33     JujuActionNotFound,
  34     JujuModelAlreadyExists,
  35     JujuControllerFailedConnecting,
  36     JujuApplicationExists,
  37     JujuError
  38 )
  39 from n2vc.utils import DB_DATA
  40 from osm_common.dbbase import DbException
  41
  42
  43 class Libjuju:
  44     def __init__(
  45         self,
  46         endpoint: str,
  47         api_proxy: str,
  48         username: str,
  49         password: str,
  50         cacert: str,
  51         loop: asyncio.AbstractEventLoop = None,
  52         log: logging.Logger = None,
  53         db: dict = None,
  54         n2vc: N2VCConnector = None,
  55         apt_mirror: str = None,
  56         enable_os_upgrade: bool = True,
  57     ):
  58         """
  59         Constructor
  60
  61         :param: endpoint:               Endpoint of the juju controller (host:port)
  62         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  63         :param: username:               Juju username
  64         :param: password:               Juju password
  65         :param: cacert:                 Juju CA Certificate
  66         :param: loop:                   Asyncio loop
  67         :param: log:                    Logger
  68         :param: db:                     DB object
  69         :param: n2vc:                   N2VC object
  70         :param: apt_mirror:             APT Mirror
  71         :param: enable_os_upgrade:      Enable OS Upgrade
  72         """
  73
  74         self.log = log or logging.getLogger("Libjuju")
  75         self.db = db
  76         db_endpoints = self._get_api_endpoints_db()
  77         self.endpoints = db_endpoints or [endpoint]
  78         if db_endpoints is None:
  79             self._update_api_endpoints_db(self.endpoints)
  80         self.api_proxy = api_proxy
  81         self.username = username
  82         self.password = password
  83         self.cacert = cacert
  84         self.loop = loop or asyncio.get_event_loop()
  85         self.n2vc = n2vc
  86
  87         # Generate config for models
  88         self.model_config = {}
  89         if apt_mirror:
  90             self.model_config["apt-mirror"] = apt_mirror
  91         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  92         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  93
  94         self.loop.set_exception_handler(self.handle_exception)
  95         self.creating_model = asyncio.Lock(loop=self.loop)
  96
  97         self.models = set()
  98         self.log.debug("Libjuju initialized!")
  99
 100         self.health_check_task = self.loop.create_task(self.health_check())
 101
 102     async def get_controller(self, timeout: float = 5.0) -> Controller:
 103         """
 104         Get controller
 105
 106         :param: timeout: Time in seconds to wait for controller to connect
 107         """
 108         controller = None
 109         try:
 110             controller = Controller(loop=self.loop)
 111             await asyncio.wait_for(
 112                 controller.connect(
 113                     endpoint=self.endpoints,
 114                     username=self.username,
 115                     password=self.password,
 116                     cacert=self.cacert,
 117                 ),
 118                 timeout=timeout,
 119             )
 120             endpoints = await controller.api_endpoints
 121             if self.endpoints != endpoints:
 122                 self.endpoints = endpoints
 123                 self._update_api_endpoints_db(self.endpoints)
 124             return controller
 125         except asyncio.CancelledError as e:
 126             raise e
 127         except Exception as e:
 128             self.log.error(
 129                 "Failed connecting to controller: {}...".format(self.endpoints)
 130             )
 131             if controller:
 132                 await self.disconnect_controller(controller)
 133             raise JujuControllerFailedConnecting(e)
 134
 135     async def disconnect(self):
 136         """Disconnect"""
 137         # Cancel health check task
 138         self.health_check_task.cancel()
 139         self.log.debug("Libjuju disconnected!")
 140
 141     async def disconnect_model(self, model: Model):
 142         """
 143         Disconnect model
 144
 145         :param: model: Model that will be disconnected
 146         """
 147         await model.disconnect()
 148
 149     async def disconnect_controller(self, controller: Controller):
 150         """
 151         Disconnect controller
 152
 153         :param: controller: Controller that will be disconnected
 154         """
 155         await controller.disconnect()
 156
 157     async def add_model(self, model_name: str, cloud_name: str):
 158         """
 159         Create model
 160
 161         :param: model_name: Model name
 162         :param: cloud_name: Cloud name
 163         """
 164
 165         # Get controller
 166         controller = await self.get_controller()
 167         model = None
 168         try:
 169             # Raise exception if model already exists
 170             if await self.model_exists(model_name, controller=controller):
 171                 raise JujuModelAlreadyExists(
 172                     "Model {} already exists.".format(model_name)
 173                 )
 174
 175             # Block until other workers have finished model creation
 176             while self.creating_model.locked():
 177                 await asyncio.sleep(0.1)
 178
 179             # If the model exists, return it from the controller
 180             if model_name in self.models:
 181                 return
 182
 183             # Create the model
 184             async with self.creating_model:
 185                 self.log.debug("Creating model {}".format(model_name))
 186                 model = await controller.add_model(
 187                     model_name,
 188                     config=self.model_config,
 189                     cloud_name=cloud_name,
 190                     credential_name=cloud_name,
 191                 )
 192                 self.models.add(model_name)
 193         finally:
 194             if model:
 195                 await self.disconnect_model(model)
 196             await self.disconnect_controller(controller)
 197
 198     async def get_executed_actions(self, model_name: str) -> list:
 199         """
 200         Get executed/history of actions for a model.
 201
 202         :param: model_name: Model name, str.
 203         :return: List of executed actions for a model.
 204         """
 205         model = None
 206         executed_actions = []
 207         controller = await self.get_controller()
 208         try:
 209             model = await self.get_model(controller, model_name)
 210             # Get all unique action names
 211             actions = {}
 212             for application in model.applications:
 213                 application_actions = await self.get_actions(application, model_name)
 214                 actions.update(application_actions)
 215             # Get status of all actions
 216             for application_action in actions:
 217                 app_action_status_list = await model.get_action_status(name=application_action)
 218                 for action_id, action_status in app_action_status_list.items():
 219                     executed_action = {"id": action_id, "action": application_action,
 220                                        "status": action_status}
 221                     # Get action output by id
 222                     action_status = await model.get_action_output(executed_action["id"])
 223                     for k, v in action_status.items():
 224                         executed_action[k] = v
 225                     executed_actions.append(executed_action)
 226         except Exception as e:
 227             raise JujuError("Error in getting executed actions for model: {}. Error: {}"
 228                             .format(model_name, str(e)))
 229         finally:
 230             if model:
 231                 await self.disconnect_model(model)
 232             await self.disconnect_controller(controller)
 233         return executed_actions
 234
 235     async def get_application_configs(self, model_name: str, application_name: str) -> dict:
 236         """
 237         Get available configs for an application.
 238
 239         :param: model_name: Model name, str.
 240         :param: application_name: Application name, str.
 241
 242         :return: A dict which has key - action name, value - action description
 243         """
 244         model = None
 245         application_configs = {}
 246         controller = await self.get_controller()
 247         try:
 248             model = await self.get_model(controller, model_name)
 249             application = self._get_application(model, application_name=application_name)
 250             application_configs = await application.get_config()
 251         except Exception as e:
 252             raise JujuError("Error in getting configs for application: {} in model: {}. Error: {}"
 253                             .format(application_name, model_name, str(e)))
 254         finally:
 255             if model:
 256                 await self.disconnect_model(model)
 257             await self.disconnect_controller(controller)
 258         return application_configs
 259
 260     async def get_model(
 261         self, controller: Controller, model_name: str, id=None
 262     ) -> Model:
 263         """
 264         Get model from controller
 265
 266         :param: controller: Controller
 267         :param: model_name: Model name
 268
 269         :return: Model: The created Juju model object
 270         """
 271         return await controller.get_model(model_name)
 272
 273     async def model_exists(
 274         self, model_name: str, controller: Controller = None
 275     ) -> bool:
 276         """
 277         Check if model exists
 278
 279         :param: controller: Controller
 280         :param: model_name: Model name
 281
 282         :return bool
 283         """
 284         need_to_disconnect = False
 285
 286         # Get controller if not passed
 287         if not controller:
 288             controller = await self.get_controller()
 289             need_to_disconnect = True
 290
 291         # Check if model exists
 292         try:
 293             return model_name in await controller.list_models()
 294         finally:
 295             if need_to_disconnect:
 296                 await self.disconnect_controller(controller)
 297
 298     async def models_exist(self, model_names: [str]) -> (bool, list):
 299         """
 300         Check if models exists
 301
 302         :param: model_names: List of strings with model names
 303
 304         :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
 305         """
 306         if not model_names:
 307             raise Exception(
 308                 "model_names must be a non-empty array. Given value: {}".format(
 309                     model_names
 310                 )
 311             )
 312         non_existing_models = []
 313         models = await self.list_models()
 314         existing_models = list(set(models).intersection(model_names))
 315         non_existing_models = list(set(model_names) - set(existing_models))
 316
 317         return (
 318             len(non_existing_models) == 0,
 319             non_existing_models,
 320         )
 321
 322     async def get_model_status(self, model_name: str) -> FullStatus:
 323         """
 324         Get model status
 325
 326         :param: model_name: Model name
 327
 328         :return: Full status object
 329         """
 330         controller = await self.get_controller()
 331         model = await self.get_model(controller, model_name)
 332         try:
 333             return await model.get_status()
 334         finally:
 335             await self.disconnect_model(model)
 336             await self.disconnect_controller(controller)
 337
 338     async def create_machine(
 339         self,
 340         model_name: str,
 341         machine_id: str = None,
 342         db_dict: dict = None,
 343         progress_timeout: float = None,
 344         total_timeout: float = None,
 345         series: str = "xenial",
 346         wait: bool = True,
 347     ) -> (Machine, bool):
 348         """
 349         Create machine
 350
 351         :param: model_name:         Model name
 352         :param: machine_id:         Machine id
 353         :param: db_dict:            Dictionary with data of the DB to write the updates
 354         :param: progress_timeout:   Maximum time between two updates in the model
 355         :param: total_timeout:      Timeout for the entity to be active
 356         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 357         :param: wait:               Wait until machine is ready
 358
 359         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 360                                                 if the machine is new or it already existed
 361         """
 362         new = False
 363         machine = None
 364
 365         self.log.debug(
 366             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 367         )
 368
 369         # Get controller
 370         controller = await self.get_controller()
 371
 372         # Get model
 373         model = await self.get_model(controller, model_name)
 374         try:
 375             if machine_id is not None:
 376                 self.log.debug(
 377                     "Searching machine (id={}) in model {}".format(
 378                         machine_id, model_name
 379                     )
 380                 )
 381
 382                 # Get machines from model and get the machine with machine_id if exists
 383                 machines = await model.get_machines()
 384                 if machine_id in machines:
 385                     self.log.debug(
 386                         "Machine (id={}) found in model {}".format(
 387                             machine_id, model_name
 388                         )
 389                     )
 390                     machine = machines[machine_id]
 391                 else:
 392                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 393
 394             if machine is None:
 395                 self.log.debug("Creating a new machine in model {}".format(model_name))
 396
 397                 # Create machine
 398                 machine = await model.add_machine(
 399                     spec=None, constraints=None, disks=None, series=series
 400                 )
 401                 new = True
 402
 403                 # Wait until the machine is ready
 404                 self.log.debug(
 405                     "Wait until machine {} is ready in model {}".format(
 406                         machine.entity_id, model_name
 407                     )
 408                 )
 409                 if wait:
 410                     await JujuModelWatcher.wait_for(
 411                         model=model,
 412                         entity=machine,
 413                         progress_timeout=progress_timeout,
 414                         total_timeout=total_timeout,
 415                         db_dict=db_dict,
 416                         n2vc=self.n2vc,
 417                     )
 418         finally:
 419             await self.disconnect_model(model)
 420             await self.disconnect_controller(controller)
 421
 422         self.log.debug(
 423             "Machine {} ready at {} in model {}".format(
 424                 machine.entity_id, machine.dns_name, model_name
 425             )
 426         )
 427         return machine, new
 428
 429     async def provision_machine(
 430         self,
 431         model_name: str,
 432         hostname: str,
 433         username: str,
 434         private_key_path: str,
 435         db_dict: dict = None,
 436         progress_timeout: float = None,
 437         total_timeout: float = None,
 438     ) -> str:
 439         """
 440         Manually provisioning of a machine
 441
 442         :param: model_name:         Model name
 443         :param: hostname:           IP to access the machine
 444         :param: username:           Username to login to the machine
 445         :param: private_key_path:   Local path for the private key
 446         :param: db_dict:            Dictionary with data of the DB to write the updates
 447         :param: progress_timeout:   Maximum time between two updates in the model
 448         :param: total_timeout:      Timeout for the entity to be active
 449
 450         :return: (Entity): Machine id
 451         """
 452         self.log.debug(
 453             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 454                 model_name, hostname, username
 455             )
 456         )
 457
 458         # Get controller
 459         controller = await self.get_controller()
 460
 461         # Get model
 462         model = await self.get_model(controller, model_name)
 463
 464         try:
 465             # Get provisioner
 466             provisioner = AsyncSSHProvisioner(
 467                 host=hostname,
 468                 user=username,
 469                 private_key_path=private_key_path,
 470                 log=self.log,
 471             )
 472
 473             # Provision machine
 474             params = await provisioner.provision_machine()
 475
 476             params.jobs = ["JobHostUnits"]
 477
 478             self.log.debug("Adding machine to model")
 479             connection = model.connection()
 480             client_facade = client.ClientFacade.from_connection(connection)
 481
 482             results = await client_facade.AddMachines(params=[params])
 483             error = results.machines[0].error
 484
 485             if error:
 486                 msg = "Error adding machine: {}".format(error.message)
 487                 self.log.error(msg=msg)
 488                 raise ValueError(msg)
 489
 490             machine_id = results.machines[0].machine
 491
 492             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 493             asyncio.ensure_future(
 494                 provisioner.install_agent(
 495                     connection=connection,
 496                     nonce=params.nonce,
 497                     machine_id=machine_id,
 498                     proxy=self.api_proxy,
 499                 )
 500             )
 501
 502             machine = None
 503             for _ in range(10):
 504                 machine_list = await model.get_machines()
 505                 if machine_id in machine_list:
 506                     self.log.debug("Machine {} found in model!".format(machine_id))
 507                     machine = model.machines.get(machine_id)
 508                     break
 509                 await asyncio.sleep(2)
 510
 511             if machine is None:
 512                 msg = "Machine {} not found in model".format(machine_id)
 513                 self.log.error(msg=msg)
 514                 raise JujuMachineNotFound(msg)
 515
 516             self.log.debug(
 517                 "Wait until machine {} is ready in model {}".format(
 518                     machine.entity_id, model_name
 519                 )
 520             )
 521             await JujuModelWatcher.wait_for(
 522                 model=model,
 523                 entity=machine,
 524                 progress_timeout=progress_timeout,
 525                 total_timeout=total_timeout,
 526                 db_dict=db_dict,
 527                 n2vc=self.n2vc,
 528             )
 529         except Exception as e:
 530             raise e
 531         finally:
 532             await self.disconnect_model(model)
 533             await self.disconnect_controller(controller)
 534
 535         self.log.debug(
 536             "Machine provisioned {} in model {}".format(machine_id, model_name)
 537         )
 538
 539         return machine_id
 540
 541     async def deploy_charm(
 542         self,
 543         application_name: str,
 544         path: str,
 545         model_name: str,
 546         machine_id: str,
 547         db_dict: dict = None,
 548         progress_timeout: float = None,
 549         total_timeout: float = None,
 550         config: dict = None,
 551         series: str = None,
 552         num_units: int = 1,
 553     ):
 554         """Deploy charm
 555
 556         :param: application_name:   Application name
 557         :param: path:               Local path to the charm
 558         :param: model_name:         Model name
 559         :param: machine_id          ID of the machine
 560         :param: db_dict:            Dictionary with data of the DB to write the updates
 561         :param: progress_timeout:   Maximum time between two updates in the model
 562         :param: total_timeout:      Timeout for the entity to be active
 563         :param: config:             Config for the charm
 564         :param: series:             Series of the charm
 565         :param: num_units:          Number of units
 566
 567         :return: (juju.application.Application): Juju application
 568         """
 569         self.log.debug(
 570             "Deploying charm {} to machine {} in model ~{}".format(
 571                 application_name, machine_id, model_name
 572             )
 573         )
 574         self.log.debug("charm: {}".format(path))
 575
 576         # Get controller
 577         controller = await self.get_controller()
 578
 579         # Get model
 580         model = await self.get_model(controller, model_name)
 581
 582         try:
 583             application = None
 584             if application_name not in model.applications:
 585
 586                 if machine_id is not None:
 587                     if machine_id not in model.machines:
 588                         msg = "Machine {} not found in model".format(machine_id)
 589                         self.log.error(msg=msg)
 590                         raise JujuMachineNotFound(msg)
 591                     machine = model.machines[machine_id]
 592                     series = machine.series
 593
 594                 application = await model.deploy(
 595                     entity_url=path,
 596                     application_name=application_name,
 597                     channel="stable",
 598                     num_units=1,
 599                     series=series,
 600                     to=machine_id,
 601                     config=config,
 602                 )
 603
 604                 self.log.debug(
 605                     "Wait until application {} is ready in model {}".format(
 606                         application_name, model_name
 607                     )
 608                 )
 609                 if num_units > 1:
 610                     for _ in range(num_units - 1):
 611                         m, _ = await self.create_machine(model_name, wait=False)
 612                         await application.add_unit(to=m.entity_id)
 613
 614                 await JujuModelWatcher.wait_for(
 615                     model=model,
 616                     entity=application,
 617                     progress_timeout=progress_timeout,
 618                     total_timeout=total_timeout,
 619                     db_dict=db_dict,
 620                     n2vc=self.n2vc,
 621                 )
 622                 self.log.debug(
 623                     "Application {} is ready in model {}".format(
 624                         application_name, model_name
 625                     )
 626                 )
 627             else:
 628                 raise JujuApplicationExists(
 629                     "Application {} exists".format(application_name)
 630                 )
 631         finally:
 632             await self.disconnect_model(model)
 633             await self.disconnect_controller(controller)
 634
 635         return application
 636
 637     def _get_application(self, model: Model, application_name: str) -> Application:
 638         """Get application
 639
 640         :param: model:              Model object
 641         :param: application_name:   Application name
 642
 643         :return: juju.application.Application (or None if it doesn't exist)
 644         """
 645         if model.applications and application_name in model.applications:
 646             return model.applications[application_name]
 647
 648     async def execute_action(
 649         self,
 650         application_name: str,
 651         model_name: str,
 652         action_name: str,
 653         db_dict: dict = None,
 654         progress_timeout: float = None,
 655         total_timeout: float = None,
 656         **kwargs
 657     ):
 658         """Execute action
 659
 660         :param: application_name:   Application name
 661         :param: model_name:         Model name
 662         :param: action_name:        Name of the action
 663         :param: db_dict:            Dictionary with data of the DB to write the updates
 664         :param: progress_timeout:   Maximum time between two updates in the model
 665         :param: total_timeout:      Timeout for the entity to be active
 666
 667         :return: (str, str): (output and status)
 668         """
 669         self.log.debug(
 670             "Executing action {} using params {}".format(action_name, kwargs)
 671         )
 672         # Get controller
 673         controller = await self.get_controller()
 674
 675         # Get model
 676         model = await self.get_model(controller, model_name)
 677
 678         try:
 679             # Get application
 680             application = self._get_application(
 681                 model, application_name=application_name,
 682             )
 683             if application is None:
 684                 raise JujuApplicationNotFound("Cannot execute action")
 685
 686             # Get unit
 687             unit = None
 688             for u in application.units:
 689                 if await u.is_leader_from_status():
 690                     unit = u
 691             if unit is None:
 692                 raise JujuLeaderUnitNotFound(
 693                     "Cannot execute action: leader unit not found"
 694                 )
 695
 696             actions = await application.get_actions()
 697
 698             if action_name not in actions:
 699                 raise JujuActionNotFound(
 700                     "Action {} not in available actions".format(action_name)
 701                 )
 702
 703             action = await unit.run_action(action_name, **kwargs)
 704
 705             self.log.debug(
 706                 "Wait until action {} is completed in application {} (model={})".format(
 707                     action_name, application_name, model_name
 708                 )
 709             )
 710             await JujuModelWatcher.wait_for(
 711                 model=model,
 712                 entity=action,
 713                 progress_timeout=progress_timeout,
 714                 total_timeout=total_timeout,
 715                 db_dict=db_dict,
 716                 n2vc=self.n2vc,
 717             )
 718
 719             output = await model.get_action_output(action_uuid=action.entity_id)
 720             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 721             status = (
 722                 status[action.entity_id] if action.entity_id in status else "failed"
 723             )
 724
 725             self.log.debug(
 726                 "Action {} completed with status {} in application {} (model={})".format(
 727                     action_name, action.status, application_name, model_name
 728                 )
 729             )
 730         finally:
 731             await self.disconnect_model(model)
 732             await self.disconnect_controller(controller)
 733
 734         return output, status
 735
 736     async def get_actions(self, application_name: str, model_name: str) -> dict:
 737         """Get list of actions
 738
 739         :param: application_name: Application name
 740         :param: model_name: Model name
 741
 742         :return: Dict with this format
 743             {
 744                 "action_name": "Description of the action",
 745                 ...
 746             }
 747         """
 748         self.log.debug(
 749             "Getting list of actions for application {}".format(application_name)
 750         )
 751
 752         # Get controller
 753         controller = await self.get_controller()
 754
 755         # Get model
 756         model = await self.get_model(controller, model_name)
 757
 758         try:
 759             # Get application
 760             application = self._get_application(
 761                 model, application_name=application_name,
 762             )
 763
 764             # Return list of actions
 765             return await application.get_actions()
 766
 767         finally:
 768             # Disconnect from model and controller
 769             await self.disconnect_model(model)
 770             await self.disconnect_controller(controller)
 771
 772     async def add_relation(
 773         self,
 774         model_name: str,
 775         application_name_1: str,
 776         application_name_2: str,
 777         relation_1: str,
 778         relation_2: str,
 779     ):
 780         """Add relation
 781
 782         :param: model_name:             Model name
 783         :param: application_name_1      First application name
 784         :param: application_name_2:     Second application name
 785         :param: relation_1:             First relation name
 786         :param: relation_2:             Second relation name
 787         """
 788
 789         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 790
 791         # Get controller
 792         controller = await self.get_controller()
 793
 794         # Get model
 795         model = await self.get_model(controller, model_name)
 796
 797         # Build relation strings
 798         r1 = "{}:{}".format(application_name_1, relation_1)
 799         r2 = "{}:{}".format(application_name_2, relation_2)
 800
 801         # Add relation
 802         try:
 803             await model.add_relation(relation1=r1, relation2=r2)
 804         except JujuAPIError as e:
 805             if "not found" in e.message:
 806                 self.log.warning("Relation not found: {}".format(e.message))
 807                 return
 808             if "already exists" in e.message:
 809                 self.log.warning("Relation already exists: {}".format(e.message))
 810                 return
 811             # another exception, raise it
 812             raise e
 813         finally:
 814             await self.disconnect_model(model)
 815             await self.disconnect_controller(controller)
 816
 817     async def destroy_model(self, model_name: str, total_timeout: float):
 818         """
 819         Destroy model
 820
 821         :param: model_name:     Model name
 822         :param: total_timeout:  Timeout
 823         """
 824
 825         controller = await self.get_controller()
 826         model = await self.get_model(controller, model_name)
 827         try:
 828             self.log.debug("Destroying model {}".format(model_name))
 829             uuid = model.info.uuid
 830
 831             # Disconnect model
 832             await self.disconnect_model(model)
 833
 834             # Destroy model
 835             if model_name in self.models:
 836                 self.models.remove(model_name)
 837
 838             await controller.destroy_model(uuid, force=True, max_wait=0)
 839
 840             # Wait until model is destroyed
 841             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 842
 843             if total_timeout is None:
 844                 total_timeout = 3600
 845             end = time.time() + total_timeout
 846             while time.time() < end:
 847                 models = await controller.list_models()
 848                 if model_name not in models:
 849                     self.log.debug(
 850                         "The model {} ({}) was destroyed".format(model_name, uuid)
 851                     )
 852                     return
 853                 await asyncio.sleep(5)
 854             raise Exception(
 855                 "Timeout waiting for model {} to be destroyed".format(model_name)
 856             )
 857         finally:
 858             await self.disconnect_controller(controller)
 859
 860     async def destroy_application(self, model: Model, application_name: str):
 861         """
 862         Destroy application
 863
 864         :param: model:              Model object
 865         :param: application_name:   Application name
 866         """
 867         self.log.debug(
 868             "Destroying application {} in model {}".format(
 869                 application_name, model.info.name
 870             )
 871         )
 872         application = model.applications.get(application_name)
 873         if application:
 874             await application.destroy()
 875         else:
 876             self.log.warning("Application not found: {}".format(application_name))
 877
 878     # async def destroy_machine(
 879     #     self, model: Model, machine_id: str, total_timeout: float = 3600
 880     # ):
 881     #     """
 882     #     Destroy machine
 883
 884     #     :param: model:          Model object
 885     #     :param: machine_id:     Machine id
 886     #     :param: total_timeout:  Timeout in seconds
 887     #     """
 888     #     machines = await model.get_machines()
 889     #     if machine_id in machines:
 890     #         machine = machines[machine_id]
 891     #         await machine.destroy(force=True)
 892     #         # max timeout
 893     #         end = time.time() + total_timeout
 894
 895     #         # wait for machine removal
 896     #         machines = await model.get_machines()
 897     #         while machine_id in machines and time.time() < end:
 898     #             self.log.debug("Waiting for machine {} is destroyed".format(machine_id))
 899     #             await asyncio.sleep(0.5)
 900     #             machines = await model.get_machines()
 901     #         self.log.debug("Machine destroyed: {}".format(machine_id))
 902     #     else:
 903     #         self.log.debug("Machine not found: {}".format(machine_id))
 904
 905     async def configure_application(
 906         self, model_name: str, application_name: str, config: dict = None
 907     ):
 908         """Configure application
 909
 910         :param: model_name:         Model name
 911         :param: application_name:   Application name
 912         :param: config:             Config to apply to the charm
 913         """
 914         self.log.debug("Configuring application {}".format(application_name))
 915
 916         if config:
 917             try:
 918                 controller = await self.get_controller()
 919                 model = await self.get_model(controller, model_name)
 920                 application = self._get_application(
 921                     model, application_name=application_name,
 922                 )
 923                 await application.set_config(config)
 924             finally:
 925                 await self.disconnect_model(model)
 926                 await self.disconnect_controller(controller)
 927
 928     def _get_api_endpoints_db(self) -> [str]:
 929         """
 930         Get API Endpoints from DB
 931
 932         :return: List of API endpoints
 933         """
 934         self.log.debug("Getting endpoints from database")
 935
 936         juju_info = self.db.get_one(
 937             DB_DATA.api_endpoints.table,
 938             q_filter=DB_DATA.api_endpoints.filter,
 939             fail_on_empty=False,
 940         )
 941         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 942             return juju_info[DB_DATA.api_endpoints.key]
 943
 944     def _update_api_endpoints_db(self, endpoints: [str]):
 945         """
 946         Update API endpoints in Database
 947
 948         :param: List of endpoints
 949         """
 950         self.log.debug("Saving endpoints {} in database".format(endpoints))
 951
 952         juju_info = self.db.get_one(
 953             DB_DATA.api_endpoints.table,
 954             q_filter=DB_DATA.api_endpoints.filter,
 955             fail_on_empty=False,
 956         )
 957         # If it doesn't, then create it
 958         if not juju_info:
 959             try:
 960                 self.db.create(
 961                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 962                 )
 963             except DbException as e:
 964                 # Racing condition: check if another N2VC worker has created it
 965                 juju_info = self.db.get_one(
 966                     DB_DATA.api_endpoints.table,
 967                     q_filter=DB_DATA.api_endpoints.filter,
 968                     fail_on_empty=False,
 969                 )
 970                 if not juju_info:
 971                     raise e
 972         self.db.set_one(
 973             DB_DATA.api_endpoints.table,
 974             DB_DATA.api_endpoints.filter,
 975             {DB_DATA.api_endpoints.key: endpoints},
 976         )
 977
 978     def handle_exception(self, loop, context):
 979         # All unhandled exceptions by libjuju are handled here.
 980         pass
 981
 982     async def health_check(self, interval: float = 300.0):
 983         """
 984         Health check to make sure controller and controller_model connections are OK
 985
 986         :param: interval: Time in seconds between checks
 987         """
 988         while True:
 989             try:
 990                 controller = await self.get_controller()
 991                 # self.log.debug("VCA is alive")
 992             except Exception as e:
 993                 self.log.error("Health check to VCA failed: {}".format(e))
 994             finally:
 995                 await self.disconnect_controller(controller)
 996             await asyncio.sleep(interval)
 997
 998     async def list_models(self, contains: str = None) -> [str]:
 999         """List models with certain names
1000
1001         :param: contains:   String that is contained in model name
1002
1003         :retur: [models] Returns list of model names
1004         """
1005
1006         controller = await self.get_controller()
1007         try:
1008             models = await controller.list_models()
1009             if contains:
1010                 models = [model for model in models if contains in model]
1011             return models
1012         finally:
1013             await self.disconnect_controller(controller)