n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus, QueryApplicationOffersResults
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuLeaderUnitNotFound,
  33     JujuActionNotFound,
  34     JujuModelAlreadyExists,
  35     JujuControllerFailedConnecting,
  36     JujuApplicationExists,
  37 )
  38 from n2vc.utils import DB_DATA
  39 from osm_common.dbbase import DbException
  40
  41
  42 class Libjuju:
  43     def __init__(
  44         self,
  45         endpoint: str,
  46         api_proxy: str,
  47         username: str,
  48         password: str,
  49         cacert: str,
  50         loop: asyncio.AbstractEventLoop = None,
  51         log: logging.Logger = None,
  52         db: dict = None,
  53         n2vc: N2VCConnector = None,
  54         apt_mirror: str = None,
  55         enable_os_upgrade: bool = True,
  56     ):
  57         """
  58         Constructor
  59
  60         :param: endpoint:               Endpoint of the juju controller (host:port)
  61         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  62         :param: username:               Juju username
  63         :param: password:               Juju password
  64         :param: cacert:                 Juju CA Certificate
  65         :param: loop:                   Asyncio loop
  66         :param: log:                    Logger
  67         :param: db:                     DB object
  68         :param: n2vc:                   N2VC object
  69         :param: apt_mirror:             APT Mirror
  70         :param: enable_os_upgrade:      Enable OS Upgrade
  71         """
  72
  73         self.log = log or logging.getLogger("Libjuju")
  74         self.db = db
  75         db_endpoints = self._get_api_endpoints_db()
  76         self.endpoints = db_endpoints or [endpoint]
  77         if db_endpoints is None:
  78             self._update_api_endpoints_db(self.endpoints)
  79         self.api_proxy = api_proxy
  80         self.username = username
  81         self.password = password
  82         self.cacert = cacert
  83         self.loop = loop or asyncio.get_event_loop()
  84         self.n2vc = n2vc
  85
  86         # Generate config for models
  87         self.model_config = {}
  88         if apt_mirror:
  89             self.model_config["apt-mirror"] = apt_mirror
  90         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  91         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  92
  93         self.loop.set_exception_handler(self.handle_exception)
  94         self.creating_model = asyncio.Lock(loop=self.loop)
  95
  96         self.models = set()
  97         self.log.debug("Libjuju initialized!")
  98
  99         self.health_check_task = self.loop.create_task(self.health_check())
 100
 101     async def get_controller(self, timeout: float = 5.0) -> Controller:
 102         """
 103         Get controller
 104
 105         :param: timeout: Time in seconds to wait for controller to connect
 106         """
 107         controller = None
 108         try:
 109             controller = Controller(loop=self.loop)
 110             await asyncio.wait_for(
 111                 controller.connect(
 112                     endpoint=self.endpoints,
 113                     username=self.username,
 114                     password=self.password,
 115                     cacert=self.cacert,
 116                 ),
 117                 timeout=timeout,
 118             )
 119             endpoints = await controller.api_endpoints
 120             if self.endpoints != endpoints:
 121                 self.endpoints = endpoints
 122                 self._update_api_endpoints_db(self.endpoints)
 123             return controller
 124         except asyncio.CancelledError as e:
 125             raise e
 126         except Exception as e:
 127             self.log.error(
 128                 "Failed connecting to controller: {}...".format(self.endpoints)
 129             )
 130             if controller:
 131                 await self.disconnect_controller(controller)
 132             raise JujuControllerFailedConnecting(e)
 133
 134     async def disconnect(self):
 135         """Disconnect"""
 136         # Cancel health check task
 137         self.health_check_task.cancel()
 138         self.log.debug("Libjuju disconnected!")
 139
 140     async def disconnect_model(self, model: Model):
 141         """
 142         Disconnect model
 143
 144         :param: model: Model that will be disconnected
 145         """
 146         await model.disconnect()
 147
 148     async def disconnect_controller(self, controller: Controller):
 149         """
 150         Disconnect controller
 151
 152         :param: controller: Controller that will be disconnected
 153         """
 154         await controller.disconnect()
 155
 156     async def add_model(self, model_name: str, cloud_name: str):
 157         """
 158         Create model
 159
 160         :param: model_name: Model name
 161         :param: cloud_name: Cloud name
 162         """
 163
 164         # Get controller
 165         controller = await self.get_controller()
 166         model = None
 167         try:
 168             # Raise exception if model already exists
 169             if await self.model_exists(model_name, controller=controller):
 170                 raise JujuModelAlreadyExists(
 171                     "Model {} already exists.".format(model_name)
 172                 )
 173
 174             # Block until other workers have finished model creation
 175             while self.creating_model.locked():
 176                 await asyncio.sleep(0.1)
 177
 178             # If the model exists, return it from the controller
 179             if model_name in self.models:
 180                 return
 181
 182             # Create the model
 183             async with self.creating_model:
 184                 self.log.debug("Creating model {}".format(model_name))
 185                 model = await controller.add_model(
 186                     model_name,
 187                     config=self.model_config,
 188                     cloud_name=cloud_name,
 189                     credential_name=cloud_name,
 190                 )
 191                 self.models.add(model_name)
 192         finally:
 193             if model:
 194                 await self.disconnect_model(model)
 195             await self.disconnect_controller(controller)
 196
 197     async def get_model(
 198         self, controller: Controller, model_name: str, id=None
 199     ) -> Model:
 200         """
 201         Get model from controller
 202
 203         :param: controller: Controller
 204         :param: model_name: Model name
 205
 206         :return: Model: The created Juju model object
 207         """
 208         return await controller.get_model(model_name)
 209
 210     async def model_exists(
 211         self, model_name: str, controller: Controller = None
 212     ) -> bool:
 213         """
 214         Check if model exists
 215
 216         :param: controller: Controller
 217         :param: model_name: Model name
 218
 219         :return bool
 220         """
 221         need_to_disconnect = False
 222
 223         # Get controller if not passed
 224         if not controller:
 225             controller = await self.get_controller()
 226             need_to_disconnect = True
 227
 228         # Check if model exists
 229         try:
 230             return model_name in await controller.list_models()
 231         finally:
 232             if need_to_disconnect:
 233                 await self.disconnect_controller(controller)
 234
 235     async def models_exist(self, model_names: [str]) -> (bool, list):
 236         """
 237         Check if models exists
 238
 239         :param: model_names: List of strings with model names
 240
 241         :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
 242         """
 243         if not model_names:
 244             raise Exception(
 245                 "model_names must be a non-empty array. Given value: {}".format(
 246                     model_names
 247                 )
 248             )
 249         non_existing_models = []
 250         models = await self.list_models()
 251         existing_models = list(set(models).intersection(model_names))
 252         non_existing_models = list(set(model_names) - set(existing_models))
 253
 254         return (
 255             len(non_existing_models) == 0,
 256             non_existing_models,
 257         )
 258
 259     async def get_model_status(self, model_name: str) -> FullStatus:
 260         """
 261         Get model status
 262
 263         :param: model_name: Model name
 264
 265         :return: Full status object
 266         """
 267         controller = await self.get_controller()
 268         model = await self.get_model(controller, model_name)
 269         try:
 270             return await model.get_status()
 271         finally:
 272             await self.disconnect_model(model)
 273             await self.disconnect_controller(controller)
 274
 275     async def create_machine(
 276         self,
 277         model_name: str,
 278         machine_id: str = None,
 279         db_dict: dict = None,
 280         progress_timeout: float = None,
 281         total_timeout: float = None,
 282         series: str = "xenial",
 283         wait: bool = True,
 284     ) -> (Machine, bool):
 285         """
 286         Create machine
 287
 288         :param: model_name:         Model name
 289         :param: machine_id:         Machine id
 290         :param: db_dict:            Dictionary with data of the DB to write the updates
 291         :param: progress_timeout:   Maximum time between two updates in the model
 292         :param: total_timeout:      Timeout for the entity to be active
 293         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 294         :param: wait:               Wait until machine is ready
 295
 296         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 297                                                 if the machine is new or it already existed
 298         """
 299         new = False
 300         machine = None
 301
 302         self.log.debug(
 303             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 304         )
 305
 306         # Get controller
 307         controller = await self.get_controller()
 308
 309         # Get model
 310         model = await self.get_model(controller, model_name)
 311         try:
 312             if machine_id is not None:
 313                 self.log.debug(
 314                     "Searching machine (id={}) in model {}".format(
 315                         machine_id, model_name
 316                     )
 317                 )
 318
 319                 # Get machines from model and get the machine with machine_id if exists
 320                 machines = await model.get_machines()
 321                 if machine_id in machines:
 322                     self.log.debug(
 323                         "Machine (id={}) found in model {}".format(
 324                             machine_id, model_name
 325                         )
 326                     )
 327                     machine = machines[machine_id]
 328                 else:
 329                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 330
 331             if machine is None:
 332                 self.log.debug("Creating a new machine in model {}".format(model_name))
 333
 334                 # Create machine
 335                 machine = await model.add_machine(
 336                     spec=None, constraints=None, disks=None, series=series
 337                 )
 338                 new = True
 339
 340                 # Wait until the machine is ready
 341                 self.log.debug(
 342                     "Wait until machine {} is ready in model {}".format(
 343                         machine.entity_id, model_name
 344                     )
 345                 )
 346                 if wait:
 347                     await JujuModelWatcher.wait_for(
 348                         model=model,
 349                         entity=machine,
 350                         progress_timeout=progress_timeout,
 351                         total_timeout=total_timeout,
 352                         db_dict=db_dict,
 353                         n2vc=self.n2vc,
 354                     )
 355         finally:
 356             await self.disconnect_model(model)
 357             await self.disconnect_controller(controller)
 358
 359         self.log.debug(
 360             "Machine {} ready at {} in model {}".format(
 361                 machine.entity_id, machine.dns_name, model_name
 362             )
 363         )
 364         return machine, new
 365
 366     async def provision_machine(
 367         self,
 368         model_name: str,
 369         hostname: str,
 370         username: str,
 371         private_key_path: str,
 372         db_dict: dict = None,
 373         progress_timeout: float = None,
 374         total_timeout: float = None,
 375     ) -> str:
 376         """
 377         Manually provisioning of a machine
 378
 379         :param: model_name:         Model name
 380         :param: hostname:           IP to access the machine
 381         :param: username:           Username to login to the machine
 382         :param: private_key_path:   Local path for the private key
 383         :param: db_dict:            Dictionary with data of the DB to write the updates
 384         :param: progress_timeout:   Maximum time between two updates in the model
 385         :param: total_timeout:      Timeout for the entity to be active
 386
 387         :return: (Entity): Machine id
 388         """
 389         self.log.debug(
 390             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 391                 model_name, hostname, username
 392             )
 393         )
 394
 395         # Get controller
 396         controller = await self.get_controller()
 397
 398         # Get model
 399         model = await self.get_model(controller, model_name)
 400
 401         try:
 402             # Get provisioner
 403             provisioner = AsyncSSHProvisioner(
 404                 host=hostname,
 405                 user=username,
 406                 private_key_path=private_key_path,
 407                 log=self.log,
 408             )
 409
 410             # Provision machine
 411             params = await provisioner.provision_machine()
 412
 413             params.jobs = ["JobHostUnits"]
 414
 415             self.log.debug("Adding machine to model")
 416             connection = model.connection()
 417             client_facade = client.ClientFacade.from_connection(connection)
 418
 419             results = await client_facade.AddMachines(params=[params])
 420             error = results.machines[0].error
 421
 422             if error:
 423                 msg = "Error adding machine: {}".format(error.message)
 424                 self.log.error(msg=msg)
 425                 raise ValueError(msg)
 426
 427             machine_id = results.machines[0].machine
 428
 429             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 430             asyncio.ensure_future(
 431                 provisioner.install_agent(
 432                     connection=connection,
 433                     nonce=params.nonce,
 434                     machine_id=machine_id,
 435                     proxy=self.api_proxy,
 436                 )
 437             )
 438
 439             machine = None
 440             for _ in range(10):
 441                 machine_list = await model.get_machines()
 442                 if machine_id in machine_list:
 443                     self.log.debug("Machine {} found in model!".format(machine_id))
 444                     machine = model.machines.get(machine_id)
 445                     break
 446                 await asyncio.sleep(2)
 447
 448             if machine is None:
 449                 msg = "Machine {} not found in model".format(machine_id)
 450                 self.log.error(msg=msg)
 451                 raise JujuMachineNotFound(msg)
 452
 453             self.log.debug(
 454                 "Wait until machine {} is ready in model {}".format(
 455                     machine.entity_id, model_name
 456                 )
 457             )
 458             await JujuModelWatcher.wait_for(
 459                 model=model,
 460                 entity=machine,
 461                 progress_timeout=progress_timeout,
 462                 total_timeout=total_timeout,
 463                 db_dict=db_dict,
 464                 n2vc=self.n2vc,
 465             )
 466         except Exception as e:
 467             raise e
 468         finally:
 469             await self.disconnect_model(model)
 470             await self.disconnect_controller(controller)
 471
 472         self.log.debug(
 473             "Machine provisioned {} in model {}".format(machine_id, model_name)
 474         )
 475
 476         return machine_id
 477
 478     async def deploy_charm(
 479         self,
 480         application_name: str,
 481         path: str,
 482         model_name: str,
 483         machine_id: str,
 484         db_dict: dict = None,
 485         progress_timeout: float = None,
 486         total_timeout: float = None,
 487         config: dict = None,
 488         series: str = None,
 489         num_units: int = 1,
 490     ):
 491         """Deploy charm
 492
 493         :param: application_name:   Application name
 494         :param: path:               Local path to the charm
 495         :param: model_name:         Model name
 496         :param: machine_id          ID of the machine
 497         :param: db_dict:            Dictionary with data of the DB to write the updates
 498         :param: progress_timeout:   Maximum time between two updates in the model
 499         :param: total_timeout:      Timeout for the entity to be active
 500         :param: config:             Config for the charm
 501         :param: series:             Series of the charm
 502         :param: num_units:          Number of units
 503
 504         :return: (juju.application.Application): Juju application
 505         """
 506         self.log.debug(
 507             "Deploying charm {} to machine {} in model ~{}".format(
 508                 application_name, machine_id, model_name
 509             )
 510         )
 511         self.log.debug("charm: {}".format(path))
 512
 513         # Get controller
 514         controller = await self.get_controller()
 515
 516         # Get model
 517         model = await self.get_model(controller, model_name)
 518
 519         try:
 520             application = None
 521             if application_name not in model.applications:
 522
 523                 if machine_id is not None:
 524                     if machine_id not in model.machines:
 525                         msg = "Machine {} not found in model".format(machine_id)
 526                         self.log.error(msg=msg)
 527                         raise JujuMachineNotFound(msg)
 528                     machine = model.machines[machine_id]
 529                     series = machine.series
 530
 531                 application = await model.deploy(
 532                     entity_url=path,
 533                     application_name=application_name,
 534                     channel="stable",
 535                     num_units=1,
 536                     series=series,
 537                     to=machine_id,
 538                     config=config,
 539                 )
 540
 541                 self.log.debug(
 542                     "Wait until application {} is ready in model {}".format(
 543                         application_name, model_name
 544                     )
 545                 )
 546                 if num_units > 1:
 547                     for _ in range(num_units - 1):
 548                         m, _ = await self.create_machine(model_name, wait=False)
 549                         await application.add_unit(to=m.entity_id)
 550
 551                 await JujuModelWatcher.wait_for(
 552                     model=model,
 553                     entity=application,
 554                     progress_timeout=progress_timeout,
 555                     total_timeout=total_timeout,
 556                     db_dict=db_dict,
 557                     n2vc=self.n2vc,
 558                 )
 559                 self.log.debug(
 560                     "Application {} is ready in model {}".format(
 561                         application_name, model_name
 562                     )
 563                 )
 564             else:
 565                 raise JujuApplicationExists(
 566                     "Application {} exists".format(application_name)
 567                 )
 568         finally:
 569             await self.disconnect_model(model)
 570             await self.disconnect_controller(controller)
 571
 572         return application
 573
 574     def _get_application(self, model: Model, application_name: str) -> Application:
 575         """Get application
 576
 577         :param: model:              Model object
 578         :param: application_name:   Application name
 579
 580         :return: juju.application.Application (or None if it doesn't exist)
 581         """
 582         if model.applications and application_name in model.applications:
 583             return model.applications[application_name]
 584
 585     async def execute_action(
 586         self,
 587         application_name: str,
 588         model_name: str,
 589         action_name: str,
 590         db_dict: dict = None,
 591         progress_timeout: float = None,
 592         total_timeout: float = None,
 593         **kwargs
 594     ):
 595         """Execute action
 596
 597         :param: application_name:   Application name
 598         :param: model_name:         Model name
 599         :param: action_name:        Name of the action
 600         :param: db_dict:            Dictionary with data of the DB to write the updates
 601         :param: progress_timeout:   Maximum time between two updates in the model
 602         :param: total_timeout:      Timeout for the entity to be active
 603
 604         :return: (str, str): (output and status)
 605         """
 606         self.log.debug(
 607             "Executing action {} using params {}".format(action_name, kwargs)
 608         )
 609         # Get controller
 610         controller = await self.get_controller()
 611
 612         # Get model
 613         model = await self.get_model(controller, model_name)
 614
 615         try:
 616             # Get application
 617             application = self._get_application(
 618                 model, application_name=application_name,
 619             )
 620             if application is None:
 621                 raise JujuApplicationNotFound("Cannot execute action")
 622
 623             # Get unit
 624             unit = None
 625             for u in application.units:
 626                 if await u.is_leader_from_status():
 627                     unit = u
 628             if unit is None:
 629                 raise JujuLeaderUnitNotFound(
 630                     "Cannot execute action: leader unit not found"
 631                 )
 632
 633             actions = await application.get_actions()
 634
 635             if action_name not in actions:
 636                 raise JujuActionNotFound(
 637                     "Action {} not in available actions".format(action_name)
 638                 )
 639
 640             action = await unit.run_action(action_name, **kwargs)
 641
 642             self.log.debug(
 643                 "Wait until action {} is completed in application {} (model={})".format(
 644                     action_name, application_name, model_name
 645                 )
 646             )
 647             await JujuModelWatcher.wait_for(
 648                 model=model,
 649                 entity=action,
 650                 progress_timeout=progress_timeout,
 651                 total_timeout=total_timeout,
 652                 db_dict=db_dict,
 653                 n2vc=self.n2vc,
 654             )
 655
 656             output = await model.get_action_output(action_uuid=action.entity_id)
 657             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 658             status = (
 659                 status[action.entity_id] if action.entity_id in status else "failed"
 660             )
 661
 662             self.log.debug(
 663                 "Action {} completed with status {} in application {} (model={})".format(
 664                     action_name, action.status, application_name, model_name
 665                 )
 666             )
 667         finally:
 668             await self.disconnect_model(model)
 669             await self.disconnect_controller(controller)
 670
 671         return output, status
 672
 673     async def get_actions(self, application_name: str, model_name: str) -> dict:
 674         """Get list of actions
 675
 676         :param: application_name: Application name
 677         :param: model_name: Model name
 678
 679         :return: Dict with this format
 680             {
 681                 "action_name": "Description of the action",
 682                 ...
 683             }
 684         """
 685         self.log.debug(
 686             "Getting list of actions for application {}".format(application_name)
 687         )
 688
 689         # Get controller
 690         controller = await self.get_controller()
 691
 692         # Get model
 693         model = await self.get_model(controller, model_name)
 694
 695         try:
 696             # Get application
 697             application = self._get_application(
 698                 model, application_name=application_name,
 699             )
 700
 701             # Return list of actions
 702             return await application.get_actions()
 703
 704         finally:
 705             # Disconnect from model and controller
 706             await self.disconnect_model(model)
 707             await self.disconnect_controller(controller)
 708
 709     async def add_relation(
 710         self, model_name: str, endpoint_1: str, endpoint_2: str,
 711     ):
 712         """Add relation
 713
 714         :param: model_name:     Model name
 715         :param: endpoint_1      First endpoint name
 716                                 ("app:endpoint" format or directly the saas name)
 717         :param: endpoint_2:     Second endpoint name (^ same format)
 718         """
 719
 720         self.log.debug("Adding relation: {} -> {}".format(endpoint_1, endpoint_2))
 721
 722         # Get controller
 723         controller = await self.get_controller()
 724
 725         # Get model
 726         model = await self.get_model(controller, model_name)
 727
 728         # Add relation
 729         try:
 730             await model.add_relation(endpoint_1, endpoint_2)
 731         except JujuAPIError as e:
 732             if "not found" in e.message:
 733                 self.log.warning("Relation not found: {}".format(e.message))
 734                 return
 735             if "already exists" in e.message:
 736                 self.log.warning("Relation already exists: {}".format(e.message))
 737                 return
 738             # another exception, raise it
 739             raise e
 740         finally:
 741             await self.disconnect_model(model)
 742             await self.disconnect_controller(controller)
 743
 744     async def consume(
 745         self, offer_url: str, model_name: str,
 746     ):
 747         """
 748         Adds a remote offer to the model. Relations can be created later using "juju relate".
 749
 750         :param: offer_url:      Offer Url
 751         :param: model_name:     Model name
 752
 753         :raises ParseError if there's a problem parsing the offer_url
 754         :raises JujuError if remote offer includes and endpoint
 755         :raises JujuAPIError if the operation is not successful
 756         """
 757         controller = await self.get_controller()
 758         model = await controller.get_model(model_name)
 759
 760         try:
 761             await model.consume(offer_url)
 762         finally:
 763             await self.disconnect_model(model)
 764             await self.disconnect_controller(controller)
 765
 766     async def destroy_model(self, model_name: str, total_timeout: float):
 767         """
 768         Destroy model
 769
 770         :param: model_name:     Model name
 771         :param: total_timeout:  Timeout
 772         """
 773
 774         controller = await self.get_controller()
 775         model = await self.get_model(controller, model_name)
 776         try:
 777             self.log.debug("Destroying model {}".format(model_name))
 778             uuid = model.info.uuid
 779
 780             # Destroy machines
 781             machines = await model.get_machines()
 782             for machine_id in machines:
 783                 try:
 784                     await self.destroy_machine(
 785                         model, machine_id=machine_id, total_timeout=total_timeout,
 786                     )
 787                 except asyncio.CancelledError:
 788                     raise
 789                 except Exception:
 790                     pass
 791
 792             # Disconnect model
 793             await self.disconnect_model(model)
 794
 795             # Destroy model
 796             if model_name in self.models:
 797                 self.models.remove(model_name)
 798
 799             await controller.destroy_model(uuid)
 800
 801             # Wait until model is destroyed
 802             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 803             last_exception = ""
 804
 805             if total_timeout is None:
 806                 total_timeout = 3600
 807             end = time.time() + total_timeout
 808             while time.time() < end:
 809                 try:
 810                     models = await controller.list_models()
 811                     if model_name not in models:
 812                         self.log.debug(
 813                             "The model {} ({}) was destroyed".format(model_name, uuid)
 814                         )
 815                         return
 816                 except asyncio.CancelledError:
 817                     raise
 818                 except Exception as e:
 819                     last_exception = e
 820                 await asyncio.sleep(5)
 821             raise Exception(
 822                 "Timeout waiting for model {} to be destroyed {}".format(
 823                     model_name, last_exception
 824                 )
 825             )
 826         finally:
 827             await self.disconnect_controller(controller)
 828
 829     async def destroy_application(self, model: Model, application_name: str):
 830         """
 831         Destroy application
 832
 833         :param: model:              Model object
 834         :param: application_name:   Application name
 835         """
 836         self.log.debug(
 837             "Destroying application {} in model {}".format(
 838                 application_name, model.info.name
 839             )
 840         )
 841         application = model.applications.get(application_name)
 842         if application:
 843             await application.destroy()
 844         else:
 845             self.log.warning("Application not found: {}".format(application_name))
 846
 847     async def destroy_machine(
 848         self, model: Model, machine_id: str, total_timeout: float = 3600
 849     ):
 850         """
 851         Destroy machine
 852
 853         :param: model:          Model object
 854         :param: machine_id:     Machine id
 855         :param: total_timeout:  Timeout in seconds
 856         """
 857         machines = await model.get_machines()
 858         if machine_id in machines:
 859             machine = machines[machine_id]
 860             await machine.destroy(force=True)
 861             # max timeout
 862             end = time.time() + total_timeout
 863
 864             # wait for machine removal
 865             machines = await model.get_machines()
 866             while machine_id in machines and time.time() < end:
 867                 self.log.debug("Waiting for machine {} is destroyed".format(machine_id))
 868                 await asyncio.sleep(0.5)
 869                 machines = await model.get_machines()
 870             self.log.debug("Machine destroyed: {}".format(machine_id))
 871         else:
 872             self.log.debug("Machine not found: {}".format(machine_id))
 873
 874     async def configure_application(
 875         self, model_name: str, application_name: str, config: dict = None
 876     ):
 877         """Configure application
 878
 879         :param: model_name:         Model name
 880         :param: application_name:   Application name
 881         :param: config:             Config to apply to the charm
 882         """
 883         self.log.debug("Configuring application {}".format(application_name))
 884
 885         if config:
 886             try:
 887                 controller = await self.get_controller()
 888                 model = await self.get_model(controller, model_name)
 889                 application = self._get_application(
 890                     model, application_name=application_name,
 891                 )
 892                 await application.set_config(config)
 893             finally:
 894                 await self.disconnect_model(model)
 895                 await self.disconnect_controller(controller)
 896
 897     def _get_api_endpoints_db(self) -> [str]:
 898         """
 899         Get API Endpoints from DB
 900
 901         :return: List of API endpoints
 902         """
 903         self.log.debug("Getting endpoints from database")
 904
 905         juju_info = self.db.get_one(
 906             DB_DATA.api_endpoints.table,
 907             q_filter=DB_DATA.api_endpoints.filter,
 908             fail_on_empty=False,
 909         )
 910         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 911             return juju_info[DB_DATA.api_endpoints.key]
 912
 913     def _update_api_endpoints_db(self, endpoints: [str]):
 914         """
 915         Update API endpoints in Database
 916
 917         :param: List of endpoints
 918         """
 919         self.log.debug("Saving endpoints {} in database".format(endpoints))
 920
 921         juju_info = self.db.get_one(
 922             DB_DATA.api_endpoints.table,
 923             q_filter=DB_DATA.api_endpoints.filter,
 924             fail_on_empty=False,
 925         )
 926         # If it doesn't, then create it
 927         if not juju_info:
 928             try:
 929                 self.db.create(
 930                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 931                 )
 932             except DbException as e:
 933                 # Racing condition: check if another N2VC worker has created it
 934                 juju_info = self.db.get_one(
 935                     DB_DATA.api_endpoints.table,
 936                     q_filter=DB_DATA.api_endpoints.filter,
 937                     fail_on_empty=False,
 938                 )
 939                 if not juju_info:
 940                     raise e
 941         self.db.set_one(
 942             DB_DATA.api_endpoints.table,
 943             DB_DATA.api_endpoints.filter,
 944             {DB_DATA.api_endpoints.key: endpoints},
 945         )
 946
 947     def handle_exception(self, loop, context):
 948         # All unhandled exceptions by libjuju are handled here.
 949         pass
 950
 951     async def health_check(self, interval: float = 300.0):
 952         """
 953         Health check to make sure controller and controller_model connections are OK
 954
 955         :param: interval: Time in seconds between checks
 956         """
 957         while True:
 958             try:
 959                 controller = await self.get_controller()
 960                 # self.log.debug("VCA is alive")
 961             except Exception as e:
 962                 self.log.error("Health check to VCA failed: {}".format(e))
 963             finally:
 964                 await self.disconnect_controller(controller)
 965             await asyncio.sleep(interval)
 966
 967     async def list_models(self, contains: str = None) -> [str]:
 968         """List models with certain names
 969
 970         :param: contains:   String that is contained in model name
 971
 972         :retur: [models] Returns list of model names
 973         """
 974
 975         controller = await self.get_controller()
 976         try:
 977             models = await controller.list_models()
 978             if contains:
 979                 models = [model for model in models if contains in model]
 980             return models
 981         finally:
 982             await self.disconnect_controller(controller)
 983
 984     async def list_offers(self, model_name: str) -> QueryApplicationOffersResults:
 985         """List models with certain names
 986
 987         :param: model_name: Model name
 988
 989         :return:            Returns list of offers
 990         """
 991
 992         controller = await self.get_controller()
 993         try:
 994             return await controller.list_offers(model_name)
 995         finally:
 996             await self.disconnect_controller(controller)