n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuLeaderUnitNotFound,
  33     JujuActionNotFound,
  34     JujuModelAlreadyExists,
  35     JujuControllerFailedConnecting,
  36     JujuApplicationExists,
  37 )
  38 from n2vc.utils import DB_DATA
  39 from osm_common.dbbase import DbException
  40
  41
  42 class Libjuju:
  43     def __init__(
  44         self,
  45         endpoint: str,
  46         api_proxy: str,
  47         username: str,
  48         password: str,
  49         cacert: str,
  50         loop: asyncio.AbstractEventLoop = None,
  51         log: logging.Logger = None,
  52         db: dict = None,
  53         n2vc: N2VCConnector = None,
  54         apt_mirror: str = None,
  55         enable_os_upgrade: bool = True,
  56     ):
  57         """
  58         Constructor
  59
  60         :param: endpoint:               Endpoint of the juju controller (host:port)
  61         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  62         :param: username:               Juju username
  63         :param: password:               Juju password
  64         :param: cacert:                 Juju CA Certificate
  65         :param: loop:                   Asyncio loop
  66         :param: log:                    Logger
  67         :param: db:                     DB object
  68         :param: n2vc:                   N2VC object
  69         :param: apt_mirror:             APT Mirror
  70         :param: enable_os_upgrade:      Enable OS Upgrade
  71         """
  72
  73         self.log = log or logging.getLogger("Libjuju")
  74         self.db = db
  75         db_endpoints = self._get_api_endpoints_db()
  76         self.endpoints = db_endpoints or [endpoint]
  77         if db_endpoints is None:
  78             self._update_api_endpoints_db(self.endpoints)
  79         self.api_proxy = api_proxy
  80         self.username = username
  81         self.password = password
  82         self.cacert = cacert
  83         self.loop = loop or asyncio.get_event_loop()
  84         self.n2vc = n2vc
  85
  86         # Generate config for models
  87         self.model_config = {}
  88         if apt_mirror:
  89             self.model_config["apt-mirror"] = apt_mirror
  90         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  91         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  92
  93         self.loop.set_exception_handler(self.handle_exception)
  94         self.creating_model = asyncio.Lock(loop=self.loop)
  95
  96         self.models = set()
  97         self.log.debug("Libjuju initialized!")
  98
  99         self.health_check_task = self.loop.create_task(self.health_check())
 100
 101     async def get_controller(self, timeout: float = 5.0) -> Controller:
 102         """
 103         Get controller
 104
 105         :param: timeout: Time in seconds to wait for controller to connect
 106         """
 107         controller = None
 108         try:
 109             controller = Controller(loop=self.loop)
 110             await asyncio.wait_for(
 111                 controller.connect(
 112                     endpoint=self.endpoints,
 113                     username=self.username,
 114                     password=self.password,
 115                     cacert=self.cacert,
 116                 ),
 117                 timeout=timeout,
 118             )
 119             endpoints = await controller.api_endpoints
 120             if self.endpoints != endpoints:
 121                 self.endpoints = endpoints
 122                 self._update_api_endpoints_db(self.endpoints)
 123             return controller
 124         except asyncio.CancelledError as e:
 125             raise e
 126         except Exception as e:
 127             self.log.error(
 128                 "Failed connecting to controller: {}...".format(self.endpoints)
 129             )
 130             if controller:
 131                 await self.disconnect_controller(controller)
 132             raise JujuControllerFailedConnecting(e)
 133
 134     async def disconnect(self):
 135         """Disconnect"""
 136         # Cancel health check task
 137         self.health_check_task.cancel()
 138         self.log.debug("Libjuju disconnected!")
 139
 140     async def disconnect_model(self, model: Model):
 141         """
 142         Disconnect model
 143
 144         :param: model: Model that will be disconnected
 145         """
 146         await model.disconnect()
 147
 148     async def disconnect_controller(self, controller: Controller):
 149         """
 150         Disconnect controller
 151
 152         :param: controller: Controller that will be disconnected
 153         """
 154         await controller.disconnect()
 155
 156     async def add_model(self, model_name: str, cloud_name: str):
 157         """
 158         Create model
 159
 160         :param: model_name: Model name
 161         :param: cloud_name: Cloud name
 162         """
 163
 164         # Get controller
 165         controller = await self.get_controller()
 166         model = None
 167         try:
 168             # Raise exception if model already exists
 169             if await self.model_exists(model_name, controller=controller):
 170                 raise JujuModelAlreadyExists(
 171                     "Model {} already exists.".format(model_name)
 172                 )
 173
 174             # Block until other workers have finished model creation
 175             while self.creating_model.locked():
 176                 await asyncio.sleep(0.1)
 177
 178             # If the model exists, return it from the controller
 179             if model_name in self.models:
 180                 return
 181
 182             # Create the model
 183             async with self.creating_model:
 184                 self.log.debug("Creating model {}".format(model_name))
 185                 model = await controller.add_model(
 186                     model_name,
 187                     config=self.model_config,
 188                     cloud_name=cloud_name,
 189                     credential_name=cloud_name,
 190                 )
 191                 self.models.add(model_name)
 192         finally:
 193             if model:
 194                 await self.disconnect_model(model)
 195             await self.disconnect_controller(controller)
 196
 197     async def get_model(
 198         self, controller: Controller, model_name: str, id=None
 199     ) -> Model:
 200         """
 201         Get model from controller
 202
 203         :param: controller: Controller
 204         :param: model_name: Model name
 205
 206         :return: Model: The created Juju model object
 207         """
 208         return await controller.get_model(model_name)
 209
 210     async def model_exists(
 211         self, model_name: str, controller: Controller = None
 212     ) -> bool:
 213         """
 214         Check if model exists
 215
 216         :param: controller: Controller
 217         :param: model_name: Model name
 218
 219         :return bool
 220         """
 221         need_to_disconnect = False
 222
 223         # Get controller if not passed
 224         if not controller:
 225             controller = await self.get_controller()
 226             need_to_disconnect = True
 227
 228         # Check if model exists
 229         try:
 230             return model_name in await controller.list_models()
 231         finally:
 232             if need_to_disconnect:
 233                 await self.disconnect_controller(controller)
 234
 235     async def models_exist(self, model_names: [str]) -> (bool, list):
 236         """
 237         Check if models exists
 238
 239         :param: model_names: List of strings with model names
 240
 241         :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
 242         """
 243         if not model_names:
 244             raise Exception(
 245                 "model_names must be a non-empty array. Given value: {}".format(
 246                     model_names
 247                 )
 248             )
 249         non_existing_models = []
 250         models = await self.list_models()
 251         existing_models = list(set(models).intersection(model_names))
 252         non_existing_models = list(set(model_names) - set(existing_models))
 253
 254         return (
 255             len(non_existing_models) == 0,
 256             non_existing_models,
 257         )
 258
 259     async def get_model_status(self, model_name: str) -> FullStatus:
 260         """
 261         Get model status
 262
 263         :param: model_name: Model name
 264
 265         :return: Full status object
 266         """
 267         controller = await self.get_controller()
 268         model = await self.get_model(controller, model_name)
 269         try:
 270             return await model.get_status()
 271         finally:
 272             await self.disconnect_model(model)
 273             await self.disconnect_controller(controller)
 274
 275     async def create_machine(
 276         self,
 277         model_name: str,
 278         machine_id: str = None,
 279         db_dict: dict = None,
 280         progress_timeout: float = None,
 281         total_timeout: float = None,
 282         series: str = "xenial",
 283         wait: bool = True,
 284     ) -> (Machine, bool):
 285         """
 286         Create machine
 287
 288         :param: model_name:         Model name
 289         :param: machine_id:         Machine id
 290         :param: db_dict:            Dictionary with data of the DB to write the updates
 291         :param: progress_timeout:   Maximum time between two updates in the model
 292         :param: total_timeout:      Timeout for the entity to be active
 293         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 294         :param: wait:               Wait until machine is ready
 295
 296         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 297                                                 if the machine is new or it already existed
 298         """
 299         new = False
 300         machine = None
 301
 302         self.log.debug(
 303             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 304         )
 305
 306         # Get controller
 307         controller = await self.get_controller()
 308
 309         # Get model
 310         model = await self.get_model(controller, model_name)
 311         try:
 312             if machine_id is not None:
 313                 self.log.debug(
 314                     "Searching machine (id={}) in model {}".format(
 315                         machine_id, model_name
 316                     )
 317                 )
 318
 319                 # Get machines from model and get the machine with machine_id if exists
 320                 machines = await model.get_machines()
 321                 if machine_id in machines:
 322                     self.log.debug(
 323                         "Machine (id={}) found in model {}".format(
 324                             machine_id, model_name
 325                         )
 326                     )
 327                     machine = machines[machine_id]
 328                 else:
 329                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 330
 331             if machine is None:
 332                 self.log.debug("Creating a new machine in model {}".format(model_name))
 333
 334                 # Create machine
 335                 machine = await model.add_machine(
 336                     spec=None, constraints=None, disks=None, series=series
 337                 )
 338                 new = True
 339
 340                 # Wait until the machine is ready
 341                 self.log.debug(
 342                     "Wait until machine {} is ready in model {}".format(
 343                         machine.entity_id, model_name
 344                     )
 345                 )
 346                 if wait:
 347                     await JujuModelWatcher.wait_for(
 348                         model=model,
 349                         entity=machine,
 350                         progress_timeout=progress_timeout,
 351                         total_timeout=total_timeout,
 352                         db_dict=db_dict,
 353                         n2vc=self.n2vc,
 354                     )
 355         finally:
 356             await self.disconnect_model(model)
 357             await self.disconnect_controller(controller)
 358
 359         self.log.debug(
 360             "Machine {} ready at {} in model {}".format(
 361                 machine.entity_id, machine.dns_name, model_name
 362             )
 363         )
 364         return machine, new
 365
 366     async def provision_machine(
 367         self,
 368         model_name: str,
 369         hostname: str,
 370         username: str,
 371         private_key_path: str,
 372         db_dict: dict = None,
 373         progress_timeout: float = None,
 374         total_timeout: float = None,
 375     ) -> str:
 376         """
 377         Manually provisioning of a machine
 378
 379         :param: model_name:         Model name
 380         :param: hostname:           IP to access the machine
 381         :param: username:           Username to login to the machine
 382         :param: private_key_path:   Local path for the private key
 383         :param: db_dict:            Dictionary with data of the DB to write the updates
 384         :param: progress_timeout:   Maximum time between two updates in the model
 385         :param: total_timeout:      Timeout for the entity to be active
 386
 387         :return: (Entity): Machine id
 388         """
 389         self.log.debug(
 390             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 391                 model_name, hostname, username
 392             )
 393         )
 394
 395         # Get controller
 396         controller = await self.get_controller()
 397
 398         # Get model
 399         model = await self.get_model(controller, model_name)
 400
 401         try:
 402             # Get provisioner
 403             provisioner = AsyncSSHProvisioner(
 404                 host=hostname,
 405                 user=username,
 406                 private_key_path=private_key_path,
 407                 log=self.log,
 408             )
 409
 410             # Provision machine
 411             params = await provisioner.provision_machine()
 412
 413             params.jobs = ["JobHostUnits"]
 414
 415             self.log.debug("Adding machine to model")
 416             connection = model.connection()
 417             client_facade = client.ClientFacade.from_connection(connection)
 418
 419             results = await client_facade.AddMachines(params=[params])
 420             error = results.machines[0].error
 421
 422             if error:
 423                 msg = "Error adding machine: {}".format(error.message)
 424                 self.log.error(msg=msg)
 425                 raise ValueError(msg)
 426
 427             machine_id = results.machines[0].machine
 428
 429             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 430             asyncio.ensure_future(
 431                 provisioner.install_agent(
 432                     connection=connection,
 433                     nonce=params.nonce,
 434                     machine_id=machine_id,
 435                     proxy=self.api_proxy,
 436                 )
 437             )
 438
 439             machine = None
 440             for _ in range(10):
 441                 machine_list = await model.get_machines()
 442                 if machine_id in machine_list:
 443                     self.log.debug("Machine {} found in model!".format(machine_id))
 444                     machine = model.machines.get(machine_id)
 445                     break
 446                 await asyncio.sleep(2)
 447
 448             if machine is None:
 449                 msg = "Machine {} not found in model".format(machine_id)
 450                 self.log.error(msg=msg)
 451                 raise JujuMachineNotFound(msg)
 452
 453             self.log.debug(
 454                 "Wait until machine {} is ready in model {}".format(
 455                     machine.entity_id, model_name
 456                 )
 457             )
 458             await JujuModelWatcher.wait_for(
 459                 model=model,
 460                 entity=machine,
 461                 progress_timeout=progress_timeout,
 462                 total_timeout=total_timeout,
 463                 db_dict=db_dict,
 464                 n2vc=self.n2vc,
 465             )
 466         except Exception as e:
 467             raise e
 468         finally:
 469             await self.disconnect_model(model)
 470             await self.disconnect_controller(controller)
 471
 472         self.log.debug(
 473             "Machine provisioned {} in model {}".format(machine_id, model_name)
 474         )
 475
 476         return machine_id
 477
 478     async def deploy_charm(
 479         self,
 480         application_name: str,
 481         path: str,
 482         model_name: str,
 483         machine_id: str,
 484         db_dict: dict = None,
 485         progress_timeout: float = None,
 486         total_timeout: float = None,
 487         config: dict = None,
 488         series: str = None,
 489         num_units: int = 1,
 490     ):
 491         """Deploy charm
 492
 493         :param: application_name:   Application name
 494         :param: path:               Local path to the charm
 495         :param: model_name:         Model name
 496         :param: machine_id          ID of the machine
 497         :param: db_dict:            Dictionary with data of the DB to write the updates
 498         :param: progress_timeout:   Maximum time between two updates in the model
 499         :param: total_timeout:      Timeout for the entity to be active
 500         :param: config:             Config for the charm
 501         :param: series:             Series of the charm
 502         :param: num_units:          Number of units
 503
 504         :return: (juju.application.Application): Juju application
 505         """
 506         self.log.debug(
 507             "Deploying charm {} to machine {} in model ~{}".format(
 508                 application_name, machine_id, model_name
 509             )
 510         )
 511         self.log.debug("charm: {}".format(path))
 512
 513         # Get controller
 514         controller = await self.get_controller()
 515
 516         # Get model
 517         model = await self.get_model(controller, model_name)
 518
 519         try:
 520             application = None
 521             if application_name not in model.applications:
 522
 523                 if machine_id is not None:
 524                     if machine_id not in model.machines:
 525                         msg = "Machine {} not found in model".format(machine_id)
 526                         self.log.error(msg=msg)
 527                         raise JujuMachineNotFound(msg)
 528                     machine = model.machines[machine_id]
 529                     series = machine.series
 530
 531                 application = await model.deploy(
 532                     entity_url=path,
 533                     application_name=application_name,
 534                     channel="stable",
 535                     num_units=1,
 536                     series=series,
 537                     to=machine_id,
 538                     config=config,
 539                 )
 540
 541                 self.log.debug(
 542                     "Wait until application {} is ready in model {}".format(
 543                         application_name, model_name
 544                     )
 545                 )
 546                 if num_units > 1:
 547                     for _ in range(num_units - 1):
 548                         m, _ = await self.create_machine(model_name, wait=False)
 549                         await application.add_unit(to=m.entity_id)
 550
 551                 await JujuModelWatcher.wait_for(
 552                     model=model,
 553                     entity=application,
 554                     progress_timeout=progress_timeout,
 555                     total_timeout=total_timeout,
 556                     db_dict=db_dict,
 557                     n2vc=self.n2vc,
 558                 )
 559                 self.log.debug(
 560                     "Application {} is ready in model {}".format(
 561                         application_name, model_name
 562                     )
 563                 )
 564             else:
 565                 raise JujuApplicationExists(
 566                     "Application {} exists".format(application_name)
 567                 )
 568         finally:
 569             await self.disconnect_model(model)
 570             await self.disconnect_controller(controller)
 571
 572         return application
 573
 574     def _get_application(self, model: Model, application_name: str) -> Application:
 575         """Get application
 576
 577         :param: model:              Model object
 578         :param: application_name:   Application name
 579
 580         :return: juju.application.Application (or None if it doesn't exist)
 581         """
 582         if model.applications and application_name in model.applications:
 583             return model.applications[application_name]
 584
 585     async def execute_action(
 586         self,
 587         application_name: str,
 588         model_name: str,
 589         action_name: str,
 590         db_dict: dict = None,
 591         progress_timeout: float = None,
 592         total_timeout: float = None,
 593         **kwargs
 594     ):
 595         """Execute action
 596
 597         :param: application_name:   Application name
 598         :param: model_name:         Model name
 599         :param: action_name:        Name of the action
 600         :param: db_dict:            Dictionary with data of the DB to write the updates
 601         :param: progress_timeout:   Maximum time between two updates in the model
 602         :param: total_timeout:      Timeout for the entity to be active
 603
 604         :return: (str, str): (output and status)
 605         """
 606         self.log.debug(
 607             "Executing action {} using params {}".format(action_name, kwargs)
 608         )
 609         # Get controller
 610         controller = await self.get_controller()
 611
 612         # Get model
 613         model = await self.get_model(controller, model_name)
 614
 615         try:
 616             # Get application
 617             application = self._get_application(
 618                 model, application_name=application_name,
 619             )
 620             if application is None:
 621                 raise JujuApplicationNotFound("Cannot execute action")
 622
 623             # Get unit
 624             unit = None
 625             for u in application.units:
 626                 if await u.is_leader_from_status():
 627                     unit = u
 628             if unit is None:
 629                 raise JujuLeaderUnitNotFound(
 630                     "Cannot execute action: leader unit not found"
 631                 )
 632
 633             actions = await application.get_actions()
 634
 635             if action_name not in actions:
 636                 raise JujuActionNotFound(
 637                     "Action {} not in available actions".format(action_name)
 638                 )
 639
 640             action = await unit.run_action(action_name, **kwargs)
 641
 642             self.log.debug(
 643                 "Wait until action {} is completed in application {} (model={})".format(
 644                     action_name, application_name, model_name
 645                 )
 646             )
 647             await JujuModelWatcher.wait_for(
 648                 model=model,
 649                 entity=action,
 650                 progress_timeout=progress_timeout,
 651                 total_timeout=total_timeout,
 652                 db_dict=db_dict,
 653                 n2vc=self.n2vc,
 654             )
 655
 656             output = await model.get_action_output(action_uuid=action.entity_id)
 657             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 658             status = (
 659                 status[action.entity_id] if action.entity_id in status else "failed"
 660             )
 661
 662             self.log.debug(
 663                 "Action {} completed with status {} in application {} (model={})".format(
 664                     action_name, action.status, application_name, model_name
 665                 )
 666             )
 667         finally:
 668             await self.disconnect_model(model)
 669             await self.disconnect_controller(controller)
 670
 671         return output, status
 672
 673     async def get_actions(self, application_name: str, model_name: str) -> dict:
 674         """Get list of actions
 675
 676         :param: application_name: Application name
 677         :param: model_name: Model name
 678
 679         :return: Dict with this format
 680             {
 681                 "action_name": "Description of the action",
 682                 ...
 683             }
 684         """
 685         self.log.debug(
 686             "Getting list of actions for application {}".format(application_name)
 687         )
 688
 689         # Get controller
 690         controller = await self.get_controller()
 691
 692         # Get model
 693         model = await self.get_model(controller, model_name)
 694
 695         try:
 696             # Get application
 697             application = self._get_application(
 698                 model, application_name=application_name,
 699             )
 700
 701             # Return list of actions
 702             return await application.get_actions()
 703
 704         finally:
 705             # Disconnect from model and controller
 706             await self.disconnect_model(model)
 707             await self.disconnect_controller(controller)
 708
 709     async def add_relation(
 710         self,
 711         model_name: str,
 712         application_name_1: str,
 713         application_name_2: str,
 714         relation_1: str,
 715         relation_2: str,
 716     ):
 717         """Add relation
 718
 719         :param: model_name:             Model name
 720         :param: application_name_1      First application name
 721         :param: application_name_2:     Second application name
 722         :param: relation_1:             First relation name
 723         :param: relation_2:             Second relation name
 724         """
 725
 726         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 727
 728         # Get controller
 729         controller = await self.get_controller()
 730
 731         # Get model
 732         model = await self.get_model(controller, model_name)
 733
 734         # Build relation strings
 735         r1 = "{}:{}".format(application_name_1, relation_1)
 736         r2 = "{}:{}".format(application_name_2, relation_2)
 737
 738         # Add relation
 739         try:
 740             await model.add_relation(relation1=r1, relation2=r2)
 741         except JujuAPIError as e:
 742             if "not found" in e.message:
 743                 self.log.warning("Relation not found: {}".format(e.message))
 744                 return
 745             if "already exists" in e.message:
 746                 self.log.warning("Relation already exists: {}".format(e.message))
 747                 return
 748             # another exception, raise it
 749             raise e
 750         finally:
 751             await self.disconnect_model(model)
 752             await self.disconnect_controller(controller)
 753
 754     async def destroy_model(self, model_name: str, total_timeout: float):
 755         """
 756         Destroy model
 757
 758         :param: model_name:     Model name
 759         :param: total_timeout:  Timeout
 760         """
 761
 762         controller = await self.get_controller()
 763         model = await self.get_model(controller, model_name)
 764         try:
 765             self.log.debug("Destroying model {}".format(model_name))
 766             uuid = model.info.uuid
 767
 768             # Destroy machines
 769             machines = await model.get_machines()
 770             for machine_id in machines:
 771                 try:
 772                     await self.destroy_machine(
 773                         model, machine_id=machine_id, total_timeout=total_timeout,
 774                     )
 775                 except asyncio.CancelledError:
 776                     raise
 777                 except Exception:
 778                     pass
 779
 780             # Disconnect model
 781             await self.disconnect_model(model)
 782
 783             # Destroy model
 784             if model_name in self.models:
 785                 self.models.remove(model_name)
 786
 787             await controller.destroy_model(uuid)
 788
 789             # Wait until model is destroyed
 790             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 791             last_exception = ""
 792
 793             if total_timeout is None:
 794                 total_timeout = 3600
 795             end = time.time() + total_timeout
 796             while time.time() < end:
 797                 try:
 798                     models = await controller.list_models()
 799                     if model_name not in models:
 800                         self.log.debug(
 801                             "The model {} ({}) was destroyed".format(model_name, uuid)
 802                         )
 803                         return
 804                 except asyncio.CancelledError:
 805                     raise
 806                 except Exception as e:
 807                     last_exception = e
 808                 await asyncio.sleep(5)
 809             raise Exception(
 810                 "Timeout waiting for model {} to be destroyed {}".format(
 811                     model_name, last_exception
 812                 )
 813             )
 814         finally:
 815             await self.disconnect_controller(controller)
 816
 817     async def destroy_application(self, model: Model, application_name: str):
 818         """
 819         Destroy application
 820
 821         :param: model:              Model object
 822         :param: application_name:   Application name
 823         """
 824         self.log.debug(
 825             "Destroying application {} in model {}".format(
 826                 application_name, model.info.name
 827             )
 828         )
 829         application = model.applications.get(application_name)
 830         if application:
 831             await application.destroy()
 832         else:
 833             self.log.warning("Application not found: {}".format(application_name))
 834
 835     async def destroy_machine(
 836         self, model: Model, machine_id: str, total_timeout: float = 3600
 837     ):
 838         """
 839         Destroy machine
 840
 841         :param: model:          Model object
 842         :param: machine_id:     Machine id
 843         :param: total_timeout:  Timeout in seconds
 844         """
 845         machines = await model.get_machines()
 846         if machine_id in machines:
 847             machine = machines[machine_id]
 848             await machine.destroy(force=True)
 849             # max timeout
 850             end = time.time() + total_timeout
 851
 852             # wait for machine removal
 853             machines = await model.get_machines()
 854             while machine_id in machines and time.time() < end:
 855                 self.log.debug("Waiting for machine {} is destroyed".format(machine_id))
 856                 await asyncio.sleep(0.5)
 857                 machines = await model.get_machines()
 858             self.log.debug("Machine destroyed: {}".format(machine_id))
 859         else:
 860             self.log.debug("Machine not found: {}".format(machine_id))
 861
 862     async def configure_application(
 863         self, model_name: str, application_name: str, config: dict = None
 864     ):
 865         """Configure application
 866
 867         :param: model_name:         Model name
 868         :param: application_name:   Application name
 869         :param: config:             Config to apply to the charm
 870         """
 871         self.log.debug("Configuring application {}".format(application_name))
 872
 873         if config:
 874             try:
 875                 controller = await self.get_controller()
 876                 model = await self.get_model(controller, model_name)
 877                 application = self._get_application(
 878                     model, application_name=application_name,
 879                 )
 880                 await application.set_config(config)
 881             finally:
 882                 await self.disconnect_model(model)
 883                 await self.disconnect_controller(controller)
 884
 885     def _get_api_endpoints_db(self) -> [str]:
 886         """
 887         Get API Endpoints from DB
 888
 889         :return: List of API endpoints
 890         """
 891         self.log.debug("Getting endpoints from database")
 892
 893         juju_info = self.db.get_one(
 894             DB_DATA.api_endpoints.table,
 895             q_filter=DB_DATA.api_endpoints.filter,
 896             fail_on_empty=False,
 897         )
 898         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 899             return juju_info[DB_DATA.api_endpoints.key]
 900
 901     def _update_api_endpoints_db(self, endpoints: [str]):
 902         """
 903         Update API endpoints in Database
 904
 905         :param: List of endpoints
 906         """
 907         self.log.debug("Saving endpoints {} in database".format(endpoints))
 908
 909         juju_info = self.db.get_one(
 910             DB_DATA.api_endpoints.table,
 911             q_filter=DB_DATA.api_endpoints.filter,
 912             fail_on_empty=False,
 913         )
 914         # If it doesn't, then create it
 915         if not juju_info:
 916             try:
 917                 self.db.create(
 918                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 919                 )
 920             except DbException as e:
 921                 # Racing condition: check if another N2VC worker has created it
 922                 juju_info = self.db.get_one(
 923                     DB_DATA.api_endpoints.table,
 924                     q_filter=DB_DATA.api_endpoints.filter,
 925                     fail_on_empty=False,
 926                 )
 927                 if not juju_info:
 928                     raise e
 929         self.db.set_one(
 930             DB_DATA.api_endpoints.table,
 931             DB_DATA.api_endpoints.filter,
 932             {DB_DATA.api_endpoints.key: endpoints},
 933         )
 934
 935     def handle_exception(self, loop, context):
 936         # All unhandled exceptions by libjuju are handled here.
 937         pass
 938
 939     async def health_check(self, interval: float = 300.0):
 940         """
 941         Health check to make sure controller and controller_model connections are OK
 942
 943         :param: interval: Time in seconds between checks
 944         """
 945         while True:
 946             try:
 947                 controller = await self.get_controller()
 948                 # self.log.debug("VCA is alive")
 949             except Exception as e:
 950                 self.log.error("Health check to VCA failed: {}".format(e))
 951             finally:
 952                 await self.disconnect_controller(controller)
 953             await asyncio.sleep(interval)
 954
 955     async def list_models(self, contains: str = None) -> [str]:
 956         """List models with certain names
 957
 958         :param: contains:   String that is contained in model name
 959
 960         :retur: [models] Returns list of model names
 961         """
 962
 963         controller = await self.get_controller()
 964         try:
 965             models = await controller.list_models()
 966             if contains:
 967                 models = [model for model in models if contains in model]
 968             return models
 969         finally:
 970             await self.disconnect_controller(controller)