n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuLeaderUnitNotFound,
  33     JujuActionNotFound,
  34     JujuModelAlreadyExists,
  35     JujuControllerFailedConnecting,
  36     JujuApplicationExists,
  37 )
  38 from n2vc.utils import DB_DATA
  39 from osm_common.dbbase import DbException
  40
  41
  42 class Libjuju:
  43     def __init__(
  44         self,
  45         endpoint: str,
  46         api_proxy: str,
  47         username: str,
  48         password: str,
  49         cacert: str,
  50         loop: asyncio.AbstractEventLoop = None,
  51         log: logging.Logger = None,
  52         db: dict = None,
  53         n2vc: N2VCConnector = None,
  54         apt_mirror: str = None,
  55         enable_os_upgrade: bool = True,
  56     ):
  57         """
  58         Constructor
  59
  60         :param: endpoint:               Endpoint of the juju controller (host:port)
  61         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  62         :param: username:               Juju username
  63         :param: password:               Juju password
  64         :param: cacert:                 Juju CA Certificate
  65         :param: loop:                   Asyncio loop
  66         :param: log:                    Logger
  67         :param: db:                     DB object
  68         :param: n2vc:                   N2VC object
  69         :param: apt_mirror:             APT Mirror
  70         :param: enable_os_upgrade:      Enable OS Upgrade
  71         """
  72
  73         self.log = log or logging.getLogger("Libjuju")
  74         self.db = db
  75         db_endpoints = self._get_api_endpoints_db()
  76         self.endpoints = db_endpoints or [endpoint]
  77         if db_endpoints is None:
  78             self._update_api_endpoints_db(self.endpoints)
  79         self.api_proxy = api_proxy
  80         self.username = username
  81         self.password = password
  82         self.cacert = cacert
  83         self.loop = loop or asyncio.get_event_loop()
  84         self.n2vc = n2vc
  85
  86         # Generate config for models
  87         self.model_config = {}
  88         if apt_mirror:
  89             self.model_config["apt-mirror"] = apt_mirror
  90         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  91         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  92
  93         self.loop.set_exception_handler(self.handle_exception)
  94         self.creating_model = asyncio.Lock(loop=self.loop)
  95
  96         self.models = set()
  97         self.log.debug("Libjuju initialized!")
  98
  99         self.health_check_task = self.loop.create_task(self.health_check())
 100
 101     async def get_controller(self, timeout: float = 5.0) -> Controller:
 102         """
 103         Get controller
 104
 105         :param: timeout: Time in seconds to wait for controller to connect
 106         """
 107         controller = None
 108         try:
 109             controller = Controller(loop=self.loop)
 110             await asyncio.wait_for(
 111                 controller.connect(
 112                     endpoint=self.endpoints,
 113                     username=self.username,
 114                     password=self.password,
 115                     cacert=self.cacert,
 116                 ),
 117                 timeout=timeout,
 118             )
 119             endpoints = await controller.api_endpoints
 120             if self.endpoints != endpoints:
 121                 self.endpoints = endpoints
 122                 self._update_api_endpoints_db(self.endpoints)
 123             return controller
 124         except asyncio.CancelledError as e:
 125             raise e
 126         except Exception as e:
 127             self.log.error(
 128                 "Failed connecting to controller: {}...".format(self.endpoints)
 129             )
 130             if controller:
 131                 await self.disconnect_controller(controller)
 132             raise JujuControllerFailedConnecting(e)
 133
 134     async def disconnect(self):
 135         """Disconnect"""
 136         # Cancel health check task
 137         self.health_check_task.cancel()
 138         self.log.debug("Libjuju disconnected!")
 139
 140     async def disconnect_model(self, model: Model):
 141         """
 142         Disconnect model
 143
 144         :param: model: Model that will be disconnected
 145         """
 146         await model.disconnect()
 147
 148     async def disconnect_controller(self, controller: Controller):
 149         """
 150         Disconnect controller
 151
 152         :param: controller: Controller that will be disconnected
 153         """
 154         await controller.disconnect()
 155
 156     async def add_model(self, model_name: str, cloud_name: str):
 157         """
 158         Create model
 159
 160         :param: model_name: Model name
 161         :param: cloud_name: Cloud name
 162         """
 163
 164         # Get controller
 165         controller = await self.get_controller()
 166         model = None
 167         try:
 168             # Raise exception if model already exists
 169             if await self.model_exists(model_name, controller=controller):
 170                 raise JujuModelAlreadyExists(
 171                     "Model {} already exists.".format(model_name)
 172                 )
 173
 174             # Block until other workers have finished model creation
 175             while self.creating_model.locked():
 176                 await asyncio.sleep(0.1)
 177
 178             # If the model exists, return it from the controller
 179             if model_name in self.models:
 180                 return
 181
 182             # Create the model
 183             async with self.creating_model:
 184                 self.log.debug("Creating model {}".format(model_name))
 185                 model = await controller.add_model(
 186                     model_name,
 187                     config=self.model_config,
 188                     cloud_name=cloud_name,
 189                     credential_name=cloud_name,
 190                 )
 191                 self.models.add(model_name)
 192         finally:
 193             if model:
 194                 await self.disconnect_model(model)
 195             await self.disconnect_controller(controller)
 196
 197     async def get_model(
 198         self, controller: Controller, model_name: str, id=None
 199     ) -> Model:
 200         """
 201         Get model from controller
 202
 203         :param: controller: Controller
 204         :param: model_name: Model name
 205
 206         :return: Model: The created Juju model object
 207         """
 208         return await controller.get_model(model_name)
 209
 210     async def model_exists(
 211         self, model_name: str, controller: Controller = None
 212     ) -> bool:
 213         """
 214         Check if model exists
 215
 216         :param: controller: Controller
 217         :param: model_name: Model name
 218
 219         :return bool
 220         """
 221         need_to_disconnect = False
 222
 223         # Get controller if not passed
 224         if not controller:
 225             controller = await self.get_controller()
 226             need_to_disconnect = True
 227
 228         # Check if model exists
 229         try:
 230             return model_name in await controller.list_models()
 231         finally:
 232             if need_to_disconnect:
 233                 await self.disconnect_controller(controller)
 234
 235     async def models_exist(self, model_names: [str]) -> (bool, list):
 236         """
 237         Check if models exists
 238
 239         :param: model_names: List of strings with model names
 240
 241         :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
 242         """
 243         if not model_names:
 244             raise Exception(
 245                 "model_names must be a non-empty array. Given value: {}".format(model_names)
 246             )
 247         non_existing_models = []
 248         models = await self.list_models()
 249         existing_models = list(set(models).intersection(model_names))
 250         non_existing_models = list(set(model_names) - set(existing_models))
 251
 252         return (
 253             len(non_existing_models) == 0,
 254             non_existing_models,
 255         )
 256
 257     async def get_model_status(self, model_name: str) -> FullStatus:
 258         """
 259         Get model status
 260
 261         :param: model_name: Model name
 262
 263         :return: Full status object
 264         """
 265         controller = await self.get_controller()
 266         model = await self.get_model(controller, model_name)
 267         try:
 268             return await model.get_status()
 269         finally:
 270             await self.disconnect_model(model)
 271             await self.disconnect_controller(controller)
 272
 273     async def create_machine(
 274         self,
 275         model_name: str,
 276         machine_id: str = None,
 277         db_dict: dict = None,
 278         progress_timeout: float = None,
 279         total_timeout: float = None,
 280         series: str = "xenial",
 281         wait: bool = True,
 282     ) -> (Machine, bool):
 283         """
 284         Create machine
 285
 286         :param: model_name:         Model name
 287         :param: machine_id:         Machine id
 288         :param: db_dict:            Dictionary with data of the DB to write the updates
 289         :param: progress_timeout:   Maximum time between two updates in the model
 290         :param: total_timeout:      Timeout for the entity to be active
 291         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 292         :param: wait:               Wait until machine is ready
 293
 294         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 295                                                 if the machine is new or it already existed
 296         """
 297         new = False
 298         machine = None
 299
 300         self.log.debug(
 301             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 302         )
 303
 304         # Get controller
 305         controller = await self.get_controller()
 306
 307         # Get model
 308         model = await self.get_model(controller, model_name)
 309         try:
 310             if machine_id is not None:
 311                 self.log.debug(
 312                     "Searching machine (id={}) in model {}".format(
 313                         machine_id, model_name
 314                     )
 315                 )
 316
 317                 # Get machines from model and get the machine with machine_id if exists
 318                 machines = await model.get_machines()
 319                 if machine_id in machines:
 320                     self.log.debug(
 321                         "Machine (id={}) found in model {}".format(
 322                             machine_id, model_name
 323                         )
 324                     )
 325                     machine = machines[machine_id]
 326                 else:
 327                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 328
 329             if machine is None:
 330                 self.log.debug("Creating a new machine in model {}".format(model_name))
 331
 332                 # Create machine
 333                 machine = await model.add_machine(
 334                     spec=None, constraints=None, disks=None, series=series
 335                 )
 336                 new = True
 337
 338                 # Wait until the machine is ready
 339                 self.log.debug(
 340                     "Wait until machine {} is ready in model {}".format(
 341                         machine.entity_id, model_name
 342                     )
 343                 )
 344                 if wait:
 345                     await JujuModelWatcher.wait_for(
 346                         model=model,
 347                         entity=machine,
 348                         progress_timeout=progress_timeout,
 349                         total_timeout=total_timeout,
 350                         db_dict=db_dict,
 351                         n2vc=self.n2vc,
 352                     )
 353         finally:
 354             await self.disconnect_model(model)
 355             await self.disconnect_controller(controller)
 356
 357         self.log.debug(
 358             "Machine {} ready at {} in model {}".format(
 359                 machine.entity_id, machine.dns_name, model_name
 360             )
 361         )
 362         return machine, new
 363
 364     async def provision_machine(
 365         self,
 366         model_name: str,
 367         hostname: str,
 368         username: str,
 369         private_key_path: str,
 370         db_dict: dict = None,
 371         progress_timeout: float = None,
 372         total_timeout: float = None,
 373     ) -> str:
 374         """
 375         Manually provisioning of a machine
 376
 377         :param: model_name:         Model name
 378         :param: hostname:           IP to access the machine
 379         :param: username:           Username to login to the machine
 380         :param: private_key_path:   Local path for the private key
 381         :param: db_dict:            Dictionary with data of the DB to write the updates
 382         :param: progress_timeout:   Maximum time between two updates in the model
 383         :param: total_timeout:      Timeout for the entity to be active
 384
 385         :return: (Entity): Machine id
 386         """
 387         self.log.debug(
 388             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 389                 model_name, hostname, username
 390             )
 391         )
 392
 393         # Get controller
 394         controller = await self.get_controller()
 395
 396         # Get model
 397         model = await self.get_model(controller, model_name)
 398
 399         try:
 400             # Get provisioner
 401             provisioner = AsyncSSHProvisioner(
 402                 host=hostname,
 403                 user=username,
 404                 private_key_path=private_key_path,
 405                 log=self.log,
 406             )
 407
 408             # Provision machine
 409             params = await provisioner.provision_machine()
 410
 411             params.jobs = ["JobHostUnits"]
 412
 413             self.log.debug("Adding machine to model")
 414             connection = model.connection()
 415             client_facade = client.ClientFacade.from_connection(connection)
 416
 417             results = await client_facade.AddMachines(params=[params])
 418             error = results.machines[0].error
 419
 420             if error:
 421                 msg = "Error adding machine: {}".format(error.message)
 422                 self.log.error(msg=msg)
 423                 raise ValueError(msg)
 424
 425             machine_id = results.machines[0].machine
 426
 427             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 428             asyncio.ensure_future(
 429                 provisioner.install_agent(
 430                     connection=connection,
 431                     nonce=params.nonce,
 432                     machine_id=machine_id,
 433                     proxy=self.api_proxy,
 434                 )
 435             )
 436
 437             machine = None
 438             for _ in range(10):
 439                 machine_list = await model.get_machines()
 440                 if machine_id in machine_list:
 441                     self.log.debug("Machine {} found in model!".format(machine_id))
 442                     machine = model.machines.get(machine_id)
 443                     break
 444                 await asyncio.sleep(2)
 445
 446             if machine is None:
 447                 msg = "Machine {} not found in model".format(machine_id)
 448                 self.log.error(msg=msg)
 449                 raise JujuMachineNotFound(msg)
 450
 451             self.log.debug(
 452                 "Wait until machine {} is ready in model {}".format(
 453                     machine.entity_id, model_name
 454                 )
 455             )
 456             await JujuModelWatcher.wait_for(
 457                 model=model,
 458                 entity=machine,
 459                 progress_timeout=progress_timeout,
 460                 total_timeout=total_timeout,
 461                 db_dict=db_dict,
 462                 n2vc=self.n2vc,
 463             )
 464         except Exception as e:
 465             raise e
 466         finally:
 467             await self.disconnect_model(model)
 468             await self.disconnect_controller(controller)
 469
 470         self.log.debug(
 471             "Machine provisioned {} in model {}".format(machine_id, model_name)
 472         )
 473
 474         return machine_id
 475
 476     async def deploy_charm(
 477         self,
 478         application_name: str,
 479         path: str,
 480         model_name: str,
 481         machine_id: str,
 482         db_dict: dict = None,
 483         progress_timeout: float = None,
 484         total_timeout: float = None,
 485         config: dict = None,
 486         series: str = None,
 487         num_units: int = 1,
 488     ):
 489         """Deploy charm
 490
 491         :param: application_name:   Application name
 492         :param: path:               Local path to the charm
 493         :param: model_name:         Model name
 494         :param: machine_id          ID of the machine
 495         :param: db_dict:            Dictionary with data of the DB to write the updates
 496         :param: progress_timeout:   Maximum time between two updates in the model
 497         :param: total_timeout:      Timeout for the entity to be active
 498         :param: config:             Config for the charm
 499         :param: series:             Series of the charm
 500         :param: num_units:          Number of units
 501
 502         :return: (juju.application.Application): Juju application
 503         """
 504         self.log.debug(
 505             "Deploying charm {} to machine {} in model ~{}".format(
 506                 application_name, machine_id, model_name
 507             )
 508         )
 509         self.log.debug("charm: {}".format(path))
 510
 511         # Get controller
 512         controller = await self.get_controller()
 513
 514         # Get model
 515         model = await self.get_model(controller, model_name)
 516
 517         try:
 518             application = None
 519             if application_name not in model.applications:
 520
 521                 if machine_id is not None:
 522                     if machine_id not in model.machines:
 523                         msg = "Machine {} not found in model".format(machine_id)
 524                         self.log.error(msg=msg)
 525                         raise JujuMachineNotFound(msg)
 526                     machine = model.machines[machine_id]
 527                     series = machine.series
 528
 529                 application = await model.deploy(
 530                     entity_url=path,
 531                     application_name=application_name,
 532                     channel="stable",
 533                     num_units=1,
 534                     series=series,
 535                     to=machine_id,
 536                     config=config,
 537                 )
 538
 539                 self.log.debug(
 540                     "Wait until application {} is ready in model {}".format(
 541                         application_name, model_name
 542                     )
 543                 )
 544                 if num_units > 1:
 545                     for _ in range(num_units - 1):
 546                         m, _ = await self.create_machine(model_name, wait=False)
 547                         await application.add_unit(to=m.entity_id)
 548
 549                 await JujuModelWatcher.wait_for(
 550                     model=model,
 551                     entity=application,
 552                     progress_timeout=progress_timeout,
 553                     total_timeout=total_timeout,
 554                     db_dict=db_dict,
 555                     n2vc=self.n2vc,
 556                 )
 557                 self.log.debug(
 558                     "Application {} is ready in model {}".format(
 559                         application_name, model_name
 560                     )
 561                 )
 562             else:
 563                 raise JujuApplicationExists(
 564                     "Application {} exists".format(application_name)
 565                 )
 566         finally:
 567             await self.disconnect_model(model)
 568             await self.disconnect_controller(controller)
 569
 570         return application
 571
 572     def _get_application(self, model: Model, application_name: str) -> Application:
 573         """Get application
 574
 575         :param: model:              Model object
 576         :param: application_name:   Application name
 577
 578         :return: juju.application.Application (or None if it doesn't exist)
 579         """
 580         if model.applications and application_name in model.applications:
 581             return model.applications[application_name]
 582
 583     async def execute_action(
 584         self,
 585         application_name: str,
 586         model_name: str,
 587         action_name: str,
 588         db_dict: dict = None,
 589         progress_timeout: float = None,
 590         total_timeout: float = None,
 591         **kwargs
 592     ):
 593         """Execute action
 594
 595         :param: application_name:   Application name
 596         :param: model_name:         Model name
 597         :param: action_name:        Name of the action
 598         :param: db_dict:            Dictionary with data of the DB to write the updates
 599         :param: progress_timeout:   Maximum time between two updates in the model
 600         :param: total_timeout:      Timeout for the entity to be active
 601
 602         :return: (str, str): (output and status)
 603         """
 604         self.log.debug(
 605             "Executing action {} using params {}".format(action_name, kwargs)
 606         )
 607         # Get controller
 608         controller = await self.get_controller()
 609
 610         # Get model
 611         model = await self.get_model(controller, model_name)
 612
 613         try:
 614             # Get application
 615             application = self._get_application(
 616                 model, application_name=application_name,
 617             )
 618             if application is None:
 619                 raise JujuApplicationNotFound("Cannot execute action")
 620
 621             # Get unit
 622             unit = None
 623             for u in application.units:
 624                 if await u.is_leader_from_status():
 625                     unit = u
 626             if unit is None:
 627                 raise JujuLeaderUnitNotFound("Cannot execute action: leader unit not found")
 628
 629             actions = await application.get_actions()
 630
 631             if action_name not in actions:
 632                 raise JujuActionNotFound(
 633                     "Action {} not in available actions".format(action_name)
 634                 )
 635
 636             action = await unit.run_action(action_name, **kwargs)
 637
 638             self.log.debug(
 639                 "Wait until action {} is completed in application {} (model={})".format(
 640                     action_name, application_name, model_name
 641                 )
 642             )
 643             await JujuModelWatcher.wait_for(
 644                 model=model,
 645                 entity=action,
 646                 progress_timeout=progress_timeout,
 647                 total_timeout=total_timeout,
 648                 db_dict=db_dict,
 649                 n2vc=self.n2vc,
 650             )
 651
 652             output = await model.get_action_output(action_uuid=action.entity_id)
 653             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 654             status = (
 655                 status[action.entity_id] if action.entity_id in status else "failed"
 656             )
 657
 658             self.log.debug(
 659                 "Action {} completed with status {} in application {} (model={})".format(
 660                     action_name, action.status, application_name, model_name
 661                 )
 662             )
 663         finally:
 664             await self.disconnect_model(model)
 665             await self.disconnect_controller(controller)
 666
 667         return output, status
 668
 669     async def get_actions(self, application_name: str, model_name: str) -> dict:
 670         """Get list of actions
 671
 672         :param: application_name: Application name
 673         :param: model_name: Model name
 674
 675         :return: Dict with this format
 676             {
 677                 "action_name": "Description of the action",
 678                 ...
 679             }
 680         """
 681         self.log.debug(
 682             "Getting list of actions for application {}".format(application_name)
 683         )
 684
 685         # Get controller
 686         controller = await self.get_controller()
 687
 688         # Get model
 689         model = await self.get_model(controller, model_name)
 690
 691         try:
 692             # Get application
 693             application = self._get_application(
 694                 model, application_name=application_name,
 695             )
 696
 697             # Return list of actions
 698             return await application.get_actions()
 699
 700         finally:
 701             # Disconnect from model and controller
 702             await self.disconnect_model(model)
 703             await self.disconnect_controller(controller)
 704
 705     async def add_relation(
 706         self,
 707         model_name: str,
 708         application_name_1: str,
 709         application_name_2: str,
 710         relation_1: str,
 711         relation_2: str,
 712     ):
 713         """Add relation
 714
 715         :param: model_name:             Model name
 716         :param: application_name_1      First application name
 717         :param: application_name_2:     Second application name
 718         :param: relation_1:             First relation name
 719         :param: relation_2:             Second relation name
 720         """
 721
 722         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 723
 724         # Get controller
 725         controller = await self.get_controller()
 726
 727         # Get model
 728         model = await self.get_model(controller, model_name)
 729
 730         # Build relation strings
 731         r1 = "{}:{}".format(application_name_1, relation_1)
 732         r2 = "{}:{}".format(application_name_2, relation_2)
 733
 734         # Add relation
 735         try:
 736             await model.add_relation(relation1=r1, relation2=r2)
 737         except JujuAPIError as e:
 738             if "not found" in e.message:
 739                 self.log.warning("Relation not found: {}".format(e.message))
 740                 return
 741             if "already exists" in e.message:
 742                 self.log.warning("Relation already exists: {}".format(e.message))
 743                 return
 744             # another exception, raise it
 745             raise e
 746         finally:
 747             await self.disconnect_model(model)
 748             await self.disconnect_controller(controller)
 749
 750     async def destroy_model(self, model_name: str, total_timeout: float):
 751         """
 752         Destroy model
 753
 754         :param: model_name:     Model name
 755         :param: total_timeout:  Timeout
 756         """
 757
 758         controller = await self.get_controller()
 759         model = await self.get_model(controller, model_name)
 760         try:
 761             self.log.debug("Destroying model {}".format(model_name))
 762             uuid = model.info.uuid
 763
 764             # Destroy machines
 765             machines = await model.get_machines()
 766             for machine_id in machines:
 767                 try:
 768                     await self.destroy_machine(
 769                         model, machine_id=machine_id, total_timeout=total_timeout,
 770                     )
 771                 except asyncio.CancelledError:
 772                     raise
 773                 except Exception:
 774                     pass
 775
 776             # Disconnect model
 777             await self.disconnect_model(model)
 778
 779             # Destroy model
 780             if model_name in self.models:
 781                 self.models.remove(model_name)
 782
 783             await controller.destroy_model(uuid)
 784
 785             # Wait until model is destroyed
 786             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 787             last_exception = ""
 788
 789             if total_timeout is None:
 790                 total_timeout = 3600
 791             end = time.time() + total_timeout
 792             while time.time() < end:
 793                 try:
 794                     models = await controller.list_models()
 795                     if model_name not in models:
 796                         self.log.debug(
 797                             "The model {} ({}) was destroyed".format(model_name, uuid)
 798                         )
 799                         return
 800                 except asyncio.CancelledError:
 801                     raise
 802                 except Exception as e:
 803                     last_exception = e
 804                 await asyncio.sleep(5)
 805             raise Exception(
 806                 "Timeout waiting for model {} to be destroyed {}".format(
 807                     model_name, last_exception
 808                 )
 809             )
 810         finally:
 811             await self.disconnect_controller(controller)
 812
 813     async def destroy_application(self, model: Model, application_name: str):
 814         """
 815         Destroy application
 816
 817         :param: model:              Model object
 818         :param: application_name:   Application name
 819         """
 820         self.log.debug(
 821             "Destroying application {} in model {}".format(
 822                 application_name, model.info.name
 823             )
 824         )
 825         application = model.applications.get(application_name)
 826         if application:
 827             await application.destroy()
 828         else:
 829             self.log.warning("Application not found: {}".format(application_name))
 830
 831     async def destroy_machine(
 832         self, model: Model, machine_id: str, total_timeout: float = 3600
 833     ):
 834         """
 835         Destroy machine
 836
 837         :param: model:          Model object
 838         :param: machine_id:     Machine id
 839         :param: total_timeout:  Timeout in seconds
 840         """
 841         machines = await model.get_machines()
 842         if machine_id in machines:
 843             machine = machines[machine_id]
 844             await machine.destroy(force=True)
 845             # max timeout
 846             end = time.time() + total_timeout
 847
 848             # wait for machine removal
 849             machines = await model.get_machines()
 850             while machine_id in machines and time.time() < end:
 851                 self.log.debug(
 852                     "Waiting for machine {} is destroyed".format(machine_id)
 853                 )
 854                 await asyncio.sleep(0.5)
 855                 machines = await model.get_machines()
 856             self.log.debug("Machine destroyed: {}".format(machine_id))
 857         else:
 858             self.log.debug("Machine not found: {}".format(machine_id))
 859
 860     async def configure_application(
 861         self, model_name: str, application_name: str, config: dict = None
 862     ):
 863         """Configure application
 864
 865         :param: model_name:         Model name
 866         :param: application_name:   Application name
 867         :param: config:             Config to apply to the charm
 868         """
 869         self.log.debug("Configuring application {}".format(application_name))
 870
 871         if config:
 872             try:
 873                 controller = await self.get_controller()
 874                 model = await self.get_model(controller, model_name)
 875                 application = self._get_application(
 876                     model, application_name=application_name,
 877                 )
 878                 await application.set_config(config)
 879             finally:
 880                 await self.disconnect_model(model)
 881                 await self.disconnect_controller(controller)
 882
 883     def _get_api_endpoints_db(self) -> [str]:
 884         """
 885         Get API Endpoints from DB
 886
 887         :return: List of API endpoints
 888         """
 889         self.log.debug("Getting endpoints from database")
 890
 891         juju_info = self.db.get_one(
 892             DB_DATA.api_endpoints.table,
 893             q_filter=DB_DATA.api_endpoints.filter,
 894             fail_on_empty=False,
 895         )
 896         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 897             return juju_info[DB_DATA.api_endpoints.key]
 898
 899     def _update_api_endpoints_db(self, endpoints: [str]):
 900         """
 901         Update API endpoints in Database
 902
 903         :param: List of endpoints
 904         """
 905         self.log.debug("Saving endpoints {} in database".format(endpoints))
 906
 907         juju_info = self.db.get_one(
 908             DB_DATA.api_endpoints.table,
 909             q_filter=DB_DATA.api_endpoints.filter,
 910             fail_on_empty=False,
 911         )
 912         # If it doesn't, then create it
 913         if not juju_info:
 914             try:
 915                 self.db.create(
 916                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 917                 )
 918             except DbException as e:
 919                 # Racing condition: check if another N2VC worker has created it
 920                 juju_info = self.db.get_one(
 921                     DB_DATA.api_endpoints.table,
 922                     q_filter=DB_DATA.api_endpoints.filter,
 923                     fail_on_empty=False,
 924                 )
 925                 if not juju_info:
 926                     raise e
 927         self.db.set_one(
 928             DB_DATA.api_endpoints.table,
 929             DB_DATA.api_endpoints.filter,
 930             {DB_DATA.api_endpoints.key: endpoints},
 931         )
 932
 933     def handle_exception(self, loop, context):
 934         # All unhandled exceptions by libjuju are handled here.
 935         pass
 936
 937     async def health_check(self, interval: float = 300.0):
 938         """
 939         Health check to make sure controller and controller_model connections are OK
 940
 941         :param: interval: Time in seconds between checks
 942         """
 943         while True:
 944             try:
 945                 controller = await self.get_controller()
 946                 # self.log.debug("VCA is alive")
 947             except Exception as e:
 948                 self.log.error("Health check to VCA failed: {}".format(e))
 949             finally:
 950                 await self.disconnect_controller(controller)
 951             await asyncio.sleep(interval)
 952
 953     async def list_models(self, contains: str = None) -> [str]:
 954         """List models with certain names
 955
 956         :param: contains:   String that is contained in model name
 957
 958         :retur: [models] Returns list of model names
 959         """
 960
 961         controller = await self.get_controller()
 962         try:
 963             models = await controller.list_models()
 964             if contains:
 965                 models = [model for model in models if contains in model]
 966             return models
 967         finally:
 968             await self.disconnect_controller(controller)