n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuModelAlreadyExists,
  33     JujuControllerFailedConnecting,
  34     JujuApplicationExists,
  35 )
  36 from n2vc.utils import DB_DATA
  37 from osm_common.dbbase import DbException
  38
  39
  40 class Libjuju:
  41     def __init__(
  42         self,
  43         endpoint: str,
  44         api_proxy: str,
  45         username: str,
  46         password: str,
  47         cacert: str,
  48         loop: asyncio.AbstractEventLoop = None,
  49         log: logging.Logger = None,
  50         db: dict = None,
  51         n2vc: N2VCConnector = None,
  52         apt_mirror: str = None,
  53         enable_os_upgrade: bool = True,
  54     ):
  55         """
  56         Constructor
  57
  58         :param: endpoint:               Endpoint of the juju controller (host:port)
  59         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  60         :param: username:               Juju username
  61         :param: password:               Juju password
  62         :param: cacert:                 Juju CA Certificate
  63         :param: loop:                   Asyncio loop
  64         :param: log:                    Logger
  65         :param: db:                     DB object
  66         :param: n2vc:                   N2VC object
  67         :param: apt_mirror:             APT Mirror
  68         :param: enable_os_upgrade:      Enable OS Upgrade
  69         """
  70
  71         self.log = log or logging.getLogger("Libjuju")
  72         self.db = db
  73         self.endpoints = self._get_api_endpoints_db() or [endpoint]
  74         self.api_proxy = api_proxy
  75         self.username = username
  76         self.password = password
  77         self.cacert = cacert
  78         self.loop = loop or asyncio.get_event_loop()
  79         self.n2vc = n2vc
  80
  81         # Generate config for models
  82         self.model_config = {}
  83         if apt_mirror:
  84             self.model_config["apt-mirror"] = apt_mirror
  85         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  86         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  87
  88         self.loop.set_exception_handler(self.handle_exception)
  89         self.creating_model = asyncio.Lock(loop=self.loop)
  90
  91         self.models = set()
  92         self.log.debug("Libjuju initialized!")
  93
  94         self.health_check_task = self.loop.create_task(self.health_check())
  95
  96     async def get_controller(self, timeout: float = 5.0) -> Controller:
  97         """
  98         Get controller
  99
 100         :param: timeout: Time in seconds to wait for controller to connect
 101         """
 102         controller = None
 103         try:
 104             controller = Controller(loop=self.loop)
 105             await asyncio.wait_for(
 106                 controller.connect(
 107                     endpoint=self.endpoints,
 108                     username=self.username,
 109                     password=self.password,
 110                     cacert=self.cacert,
 111                 ),
 112                 timeout=timeout,
 113             )
 114             endpoints = await controller.api_endpoints
 115             if self.endpoints != endpoints:
 116                 self.endpoints = endpoints
 117                 self._update_api_endpoints_db(self.endpoints)
 118             return controller
 119         except asyncio.CancelledError as e:
 120             raise e
 121         except Exception as e:
 122             self.log.error(
 123                 "Failed connecting to controller: {}...".format(self.endpoints)
 124             )
 125             if controller:
 126                 await self.disconnect_controller(controller)
 127             raise JujuControllerFailedConnecting(e)
 128
 129     async def disconnect(self):
 130         """Disconnect"""
 131         # Cancel health check task
 132         self.health_check_task.cancel()
 133         self.log.debug("Libjuju disconnected!")
 134
 135     async def disconnect_model(self, model: Model):
 136         """
 137         Disconnect model
 138
 139         :param: model: Model that will be disconnected
 140         """
 141         await model.disconnect()
 142
 143     async def disconnect_controller(self, controller: Controller):
 144         """
 145         Disconnect controller
 146
 147         :param: controller: Controller that will be disconnected
 148         """
 149         await controller.disconnect()
 150
 151     async def add_model(self, model_name: str, cloud_name: str):
 152         """
 153         Create model
 154
 155         :param: model_name: Model name
 156         :param: cloud_name: Cloud name
 157         """
 158
 159         # Get controller
 160         controller = await self.get_controller()
 161         model = None
 162         try:
 163             # Raise exception if model already exists
 164             if await self.model_exists(model_name, controller=controller):
 165                 raise JujuModelAlreadyExists(
 166                     "Model {} already exists.".format(model_name)
 167                 )
 168
 169             # Block until other workers have finished model creation
 170             while self.creating_model.locked():
 171                 await asyncio.sleep(0.1)
 172
 173             # If the model exists, return it from the controller
 174             if model_name in self.models:
 175                 return
 176
 177             # Create the model
 178             async with self.creating_model:
 179                 self.log.debug("Creating model {}".format(model_name))
 180                 model = await controller.add_model(
 181                     model_name,
 182                     config=self.model_config,
 183                     cloud_name=cloud_name,
 184                     credential_name=cloud_name,
 185                 )
 186                 self.models.add(model_name)
 187         finally:
 188             if model:
 189                 await self.disconnect_model(model)
 190             await self.disconnect_controller(controller)
 191
 192     async def get_model(
 193         self, controller: Controller, model_name: str, id=None
 194     ) -> Model:
 195         """
 196         Get model from controller
 197
 198         :param: controller: Controller
 199         :param: model_name: Model name
 200
 201         :return: Model: The created Juju model object
 202         """
 203         return await controller.get_model(model_name)
 204
 205     async def model_exists(
 206         self, model_name: str, controller: Controller = None
 207     ) -> bool:
 208         """
 209         Check if model exists
 210
 211         :param: controller: Controller
 212         :param: model_name: Model name
 213
 214         :return bool
 215         """
 216         need_to_disconnect = False
 217
 218         # Get controller if not passed
 219         if not controller:
 220             controller = await self.get_controller()
 221             need_to_disconnect = True
 222
 223         # Check if model exists
 224         try:
 225             return model_name in await controller.list_models()
 226         finally:
 227             if need_to_disconnect:
 228                 await self.disconnect_controller(controller)
 229
 230     async def get_model_status(self, model_name: str) -> FullStatus:
 231         """
 232         Get model status
 233
 234         :param: model_name: Model name
 235
 236         :return: Full status object
 237         """
 238         controller = await self.get_controller()
 239         model = await self.get_model(controller, model_name)
 240         try:
 241             return await model.get_status()
 242         finally:
 243             await self.disconnect_model(model)
 244             await self.disconnect_controller(controller)
 245
 246     async def create_machine(
 247         self,
 248         model_name: str,
 249         machine_id: str = None,
 250         db_dict: dict = None,
 251         progress_timeout: float = None,
 252         total_timeout: float = None,
 253         series: str = "xenial",
 254         wait: bool = True,
 255     ) -> (Machine, bool):
 256         """
 257         Create machine
 258
 259         :param: model_name:         Model name
 260         :param: machine_id:         Machine id
 261         :param: db_dict:            Dictionary with data of the DB to write the updates
 262         :param: progress_timeout:   Maximum time between two updates in the model
 263         :param: total_timeout:      Timeout for the entity to be active
 264         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 265         :param: wait:               Wait until machine is ready
 266
 267         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 268                                                 if the machine is new or it already existed
 269         """
 270         new = False
 271         machine = None
 272
 273         self.log.debug(
 274             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 275         )
 276
 277         # Get controller
 278         controller = await self.get_controller()
 279
 280         # Get model
 281         model = await self.get_model(controller, model_name)
 282         try:
 283             if machine_id is not None:
 284                 self.log.debug(
 285                     "Searching machine (id={}) in model {}".format(
 286                         machine_id, model_name
 287                     )
 288                 )
 289
 290                 # Get machines from model and get the machine with machine_id if exists
 291                 machines = await model.get_machines()
 292                 if machine_id in machines:
 293                     self.log.debug(
 294                         "Machine (id={}) found in model {}".format(
 295                             machine_id, model_name
 296                         )
 297                     )
 298                     machine = model.machines[machine_id]
 299                 else:
 300                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 301
 302             if machine is None:
 303                 self.log.debug("Creating a new machine in model {}".format(model_name))
 304
 305                 # Create machine
 306                 machine = await model.add_machine(
 307                     spec=None, constraints=None, disks=None, series=series
 308                 )
 309                 new = True
 310
 311                 # Wait until the machine is ready
 312                 self.log.debug(
 313                     "Wait until machine {} is ready in model {}".format(
 314                         machine.entity_id, model_name
 315                     )
 316                 )
 317                 if wait:
 318                     await JujuModelWatcher.wait_for(
 319                         model=model,
 320                         entity=machine,
 321                         progress_timeout=progress_timeout,
 322                         total_timeout=total_timeout,
 323                         db_dict=db_dict,
 324                         n2vc=self.n2vc,
 325                     )
 326         finally:
 327             await self.disconnect_model(model)
 328             await self.disconnect_controller(controller)
 329
 330         self.log.debug(
 331             "Machine {} ready at {} in model {}".format(
 332                 machine.entity_id, machine.dns_name, model_name
 333             )
 334         )
 335         return machine, new
 336
 337     async def provision_machine(
 338         self,
 339         model_name: str,
 340         hostname: str,
 341         username: str,
 342         private_key_path: str,
 343         db_dict: dict = None,
 344         progress_timeout: float = None,
 345         total_timeout: float = None,
 346     ) -> str:
 347         """
 348         Manually provisioning of a machine
 349
 350         :param: model_name:         Model name
 351         :param: hostname:           IP to access the machine
 352         :param: username:           Username to login to the machine
 353         :param: private_key_path:   Local path for the private key
 354         :param: db_dict:            Dictionary with data of the DB to write the updates
 355         :param: progress_timeout:   Maximum time between two updates in the model
 356         :param: total_timeout:      Timeout for the entity to be active
 357
 358         :return: (Entity): Machine id
 359         """
 360         self.log.debug(
 361             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 362                 model_name, hostname, username
 363             )
 364         )
 365
 366         # Get controller
 367         controller = await self.get_controller()
 368
 369         # Get model
 370         model = await self.get_model(controller, model_name)
 371
 372         try:
 373             # Get provisioner
 374             provisioner = AsyncSSHProvisioner(
 375                 host=hostname,
 376                 user=username,
 377                 private_key_path=private_key_path,
 378                 log=self.log,
 379             )
 380
 381             # Provision machine
 382             params = await provisioner.provision_machine()
 383
 384             params.jobs = ["JobHostUnits"]
 385
 386             self.log.debug("Adding machine to model")
 387             connection = model.connection()
 388             client_facade = client.ClientFacade.from_connection(connection)
 389
 390             results = await client_facade.AddMachines(params=[params])
 391             error = results.machines[0].error
 392
 393             if error:
 394                 msg = "Error adding machine: {}".format(error.message)
 395                 self.log.error(msg=msg)
 396                 raise ValueError(msg)
 397
 398             machine_id = results.machines[0].machine
 399
 400             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 401             asyncio.ensure_future(
 402                 provisioner.install_agent(
 403                     connection=connection,
 404                     nonce=params.nonce,
 405                     machine_id=machine_id,
 406                     api=self.api_proxy,
 407                 )
 408             )
 409
 410             machine = None
 411             for _ in range(10):
 412                 machine_list = await model.get_machines()
 413                 if machine_id in machine_list:
 414                     self.log.debug("Machine {} found in model!".format(machine_id))
 415                     machine = model.machines.get(machine_id)
 416                     break
 417                 await asyncio.sleep(2)
 418
 419             if machine is None:
 420                 msg = "Machine {} not found in model".format(machine_id)
 421                 self.log.error(msg=msg)
 422                 raise JujuMachineNotFound(msg)
 423
 424             self.log.debug(
 425                 "Wait until machine {} is ready in model {}".format(
 426                     machine.entity_id, model_name
 427                 )
 428             )
 429             await JujuModelWatcher.wait_for(
 430                 model=model,
 431                 entity=machine,
 432                 progress_timeout=progress_timeout,
 433                 total_timeout=total_timeout,
 434                 db_dict=db_dict,
 435                 n2vc=self.n2vc,
 436             )
 437         except Exception as e:
 438             raise e
 439         finally:
 440             await self.disconnect_model(model)
 441             await self.disconnect_controller(controller)
 442
 443         self.log.debug(
 444             "Machine provisioned {} in model {}".format(machine_id, model_name)
 445         )
 446
 447         return machine_id
 448
 449     async def deploy_charm(
 450         self,
 451         application_name: str,
 452         path: str,
 453         model_name: str,
 454         machine_id: str,
 455         db_dict: dict = None,
 456         progress_timeout: float = None,
 457         total_timeout: float = None,
 458         config: dict = None,
 459         series: str = None,
 460         num_units: int = 1,
 461     ):
 462         """Deploy charm
 463
 464         :param: application_name:   Application name
 465         :param: path:               Local path to the charm
 466         :param: model_name:         Model name
 467         :param: machine_id          ID of the machine
 468         :param: db_dict:            Dictionary with data of the DB to write the updates
 469         :param: progress_timeout:   Maximum time between two updates in the model
 470         :param: total_timeout:      Timeout for the entity to be active
 471         :param: config:             Config for the charm
 472         :param: series:             Series of the charm
 473         :param: num_units:          Number of units
 474
 475         :return: (juju.application.Application): Juju application
 476         """
 477         self.log.debug(
 478             "Deploying charm {} to machine {} in model ~{}".format(
 479                 application_name, machine_id, model_name
 480             )
 481         )
 482         self.log.debug("charm: {}".format(path))
 483
 484         # Get controller
 485         controller = await self.get_controller()
 486
 487         # Get model
 488         model = await self.get_model(controller, model_name)
 489
 490         try:
 491             application = None
 492             if application_name not in model.applications:
 493
 494                 if machine_id is not None:
 495                     if machine_id not in model.machines:
 496                         msg = "Machine {} not found in model".format(machine_id)
 497                         self.log.error(msg=msg)
 498                         raise JujuMachineNotFound(msg)
 499                     machine = model.machines[machine_id]
 500                     series = machine.series
 501
 502                 application = await model.deploy(
 503                     entity_url=path,
 504                     application_name=application_name,
 505                     channel="stable",
 506                     num_units=1,
 507                     series=series,
 508                     to=machine_id,
 509                     config=config,
 510                 )
 511
 512                 self.log.debug(
 513                     "Wait until application {} is ready in model {}".format(
 514                         application_name, model_name
 515                     )
 516                 )
 517                 if num_units > 1:
 518                     for _ in range(num_units - 1):
 519                         m, _ = await self.create_machine(model_name, wait=False)
 520                         await application.add_unit(to=m.entity_id)
 521
 522                 await JujuModelWatcher.wait_for(
 523                     model=model,
 524                     entity=application,
 525                     progress_timeout=progress_timeout,
 526                     total_timeout=total_timeout,
 527                     db_dict=db_dict,
 528                     n2vc=self.n2vc,
 529                 )
 530                 self.log.debug(
 531                     "Application {} is ready in model {}".format(
 532                         application_name, model_name
 533                     )
 534                 )
 535             else:
 536                 raise JujuApplicationExists(
 537                     "Application {} exists".format(application_name)
 538                 )
 539         finally:
 540             await self.disconnect_model(model)
 541             await self.disconnect_controller(controller)
 542
 543         return application
 544
 545     def _get_application(self, model: Model, application_name: str) -> Application:
 546         """Get application
 547
 548         :param: model:              Model object
 549         :param: application_name:   Application name
 550
 551         :return: juju.application.Application (or None if it doesn't exist)
 552         """
 553         if model.applications and application_name in model.applications:
 554             return model.applications[application_name]
 555
 556     async def execute_action(
 557         self,
 558         application_name: str,
 559         model_name: str,
 560         action_name: str,
 561         db_dict: dict = None,
 562         progress_timeout: float = None,
 563         total_timeout: float = None,
 564         **kwargs
 565     ):
 566         """Execute action
 567
 568         :param: application_name:   Application name
 569         :param: model_name:         Model name
 570         :param: cloud_name:         Cloud name
 571         :param: action_name:        Name of the action
 572         :param: db_dict:            Dictionary with data of the DB to write the updates
 573         :param: progress_timeout:   Maximum time between two updates in the model
 574         :param: total_timeout:      Timeout for the entity to be active
 575
 576         :return: (str, str): (output and status)
 577         """
 578         self.log.debug(
 579             "Executing action {} using params {}".format(action_name, kwargs)
 580         )
 581         # Get controller
 582         controller = await self.get_controller()
 583
 584         # Get model
 585         model = await self.get_model(controller, model_name)
 586
 587         try:
 588             # Get application
 589             application = self._get_application(
 590                 model, application_name=application_name,
 591             )
 592             if application is None:
 593                 raise JujuApplicationNotFound("Cannot execute action")
 594
 595             # Get unit
 596             unit = None
 597             for u in application.units:
 598                 if await u.is_leader_from_status():
 599                     unit = u
 600             if unit is None:
 601                 raise Exception("Cannot execute action: leader unit not found")
 602
 603             actions = await application.get_actions()
 604
 605             if action_name not in actions:
 606                 raise Exception(
 607                     "Action {} not in available actions".format(action_name)
 608                 )
 609
 610             action = await unit.run_action(action_name, **kwargs)
 611
 612             self.log.debug(
 613                 "Wait until action {} is completed in application {} (model={})".format(
 614                     action_name, application_name, model_name
 615                 )
 616             )
 617             await JujuModelWatcher.wait_for(
 618                 model=model,
 619                 entity=action,
 620                 progress_timeout=progress_timeout,
 621                 total_timeout=total_timeout,
 622                 db_dict=db_dict,
 623                 n2vc=self.n2vc,
 624             )
 625
 626             output = await model.get_action_output(action_uuid=action.entity_id)
 627             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 628             status = (
 629                 status[action.entity_id] if action.entity_id in status else "failed"
 630             )
 631
 632             self.log.debug(
 633                 "Action {} completed with status {} in application {} (model={})".format(
 634                     action_name, action.status, application_name, model_name
 635                 )
 636             )
 637         except Exception as e:
 638             raise e
 639         finally:
 640             await self.disconnect_model(model)
 641             await self.disconnect_controller(controller)
 642
 643         return output, status
 644
 645     async def get_actions(self, application_name: str, model_name: str) -> dict:
 646         """Get list of actions
 647
 648         :param: application_name: Application name
 649         :param: model_name: Model name
 650
 651         :return: Dict with this format
 652             {
 653                 "action_name": "Description of the action",
 654                 ...
 655             }
 656         """
 657         self.log.debug(
 658             "Getting list of actions for application {}".format(application_name)
 659         )
 660
 661         # Get controller
 662         controller = await self.get_controller()
 663
 664         # Get model
 665         model = await self.get_model(controller, model_name)
 666
 667         try:
 668             # Get application
 669             application = self._get_application(
 670                 model, application_name=application_name,
 671             )
 672
 673             # Return list of actions
 674             return await application.get_actions()
 675
 676         finally:
 677             # Disconnect from model and controller
 678             await self.disconnect_model(model)
 679             await self.disconnect_controller(controller)
 680
 681     async def add_relation(
 682         self,
 683         model_name: str,
 684         application_name_1: str,
 685         application_name_2: str,
 686         relation_1: str,
 687         relation_2: str,
 688     ):
 689         """Add relation
 690
 691         :param: model_name:             Model name
 692         :param: application_name_1      First application name
 693         :param: application_name_2:     Second application name
 694         :param: relation_1:             First relation name
 695         :param: relation_2:             Second relation name
 696         """
 697
 698         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 699
 700         # Get controller
 701         controller = await self.get_controller()
 702
 703         # Get model
 704         model = await self.get_model(controller, model_name)
 705
 706         # Build relation strings
 707         r1 = "{}:{}".format(application_name_1, relation_1)
 708         r2 = "{}:{}".format(application_name_2, relation_2)
 709
 710         # Add relation
 711         try:
 712             await model.add_relation(relation1=r1, relation2=r2)
 713         except JujuAPIError as e:
 714             if "not found" in e.message:
 715                 self.log.warning("Relation not found: {}".format(e.message))
 716                 return
 717             if "already exists" in e.message:
 718                 self.log.warning("Relation already exists: {}".format(e.message))
 719                 return
 720             # another exception, raise it
 721             raise e
 722         finally:
 723             await self.disconnect_model(model)
 724             await self.disconnect_controller(controller)
 725
 726     async def destroy_model(self, model_name: str, total_timeout: float):
 727         """
 728         Destroy model
 729
 730         :param: model_name:     Model name
 731         :param: total_timeout:  Timeout
 732         """
 733
 734         controller = await self.get_controller()
 735         model = await self.get_model(controller, model_name)
 736         try:
 737             self.log.debug("Destroying model {}".format(model_name))
 738             uuid = model.info.uuid
 739
 740             # Destroy machines
 741             machines = await model.get_machines()
 742             for machine_id in machines:
 743                 try:
 744                     await self.destroy_machine(
 745                         model, machine_id=machine_id, total_timeout=total_timeout,
 746                     )
 747                 except asyncio.CancelledError:
 748                     raise
 749                 except Exception:
 750                     pass
 751
 752             # Disconnect model
 753             await self.disconnect_model(model)
 754
 755             # Destroy model
 756             if model_name in self.models:
 757                 self.models.remove(model_name)
 758
 759             await controller.destroy_model(uuid)
 760
 761             # Wait until model is destroyed
 762             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 763             last_exception = ""
 764
 765             if total_timeout is None:
 766                 total_timeout = 3600
 767             end = time.time() + total_timeout
 768             while time.time() < end:
 769                 try:
 770                     models = await controller.list_models()
 771                     if model_name not in models:
 772                         self.log.debug(
 773                             "The model {} ({}) was destroyed".format(model_name, uuid)
 774                         )
 775                         return
 776                 except asyncio.CancelledError:
 777                     raise
 778                 except Exception as e:
 779                     last_exception = e
 780                 await asyncio.sleep(5)
 781             raise Exception(
 782                 "Timeout waiting for model {} to be destroyed {}".format(
 783                     model_name, last_exception
 784                 )
 785             )
 786         finally:
 787             await self.disconnect_controller(controller)
 788
 789     async def destroy_application(self, model: Model, application_name: str):
 790         """
 791         Destroy application
 792
 793         :param: model:              Model object
 794         :param: application_name:   Application name
 795         """
 796         self.log.debug(
 797             "Destroying application {} in model {}".format(
 798                 application_name, model.info.name
 799             )
 800         )
 801         application = model.applications.get(application_name)
 802         if application:
 803             await application.destroy()
 804         else:
 805             self.log.warning("Application not found: {}".format(application_name))
 806
 807     async def destroy_machine(
 808         self, model: Model, machine_id: str, total_timeout: float = 3600
 809     ):
 810         """
 811         Destroy machine
 812
 813         :param: model:          Model object
 814         :param: machine_id:     Machine id
 815         :param: total_timeout:  Timeout in seconds
 816         """
 817         machines = await model.get_machines()
 818         if machine_id in machines:
 819             machine = model.machines[machine_id]
 820             await machine.destroy(force=True)
 821             # max timeout
 822             end = time.time() + total_timeout
 823
 824             # wait for machine removal
 825             machines = await model.get_machines()
 826             while machine_id in machines and time.time() < end:
 827                 self.log.debug(
 828                     "Waiting for machine {} is destroyed".format(machine_id)
 829                 )
 830                 await asyncio.sleep(0.5)
 831                 machines = await model.get_machines()
 832             self.log.debug("Machine destroyed: {}".format(machine_id))
 833         else:
 834             self.log.debug("Machine not found: {}".format(machine_id))
 835
 836     async def configure_application(
 837         self, model_name: str, application_name: str, config: dict = None
 838     ):
 839         """Configure application
 840
 841         :param: model_name:         Model name
 842         :param: application_name:   Application name
 843         :param: config:             Config to apply to the charm
 844         """
 845         self.log.debug("Configuring application {}".format(application_name))
 846
 847         if config:
 848             try:
 849                 controller = await self.get_controller()
 850                 model = await self.get_model(controller, model_name)
 851                 application = self._get_application(
 852                     model, application_name=application_name,
 853                 )
 854                 await application.set_config(config)
 855             finally:
 856                 await self.disconnect_model(model)
 857                 await self.disconnect_controller(controller)
 858
 859     def _get_api_endpoints_db(self) -> [str]:
 860         """
 861         Get API Endpoints from DB
 862
 863         :return: List of API endpoints
 864         """
 865         self.log.debug("Getting endpoints from database")
 866
 867         juju_info = self.db.get_one(
 868             DB_DATA.api_endpoints.table,
 869             q_filter=DB_DATA.api_endpoints.filter,
 870             fail_on_empty=False,
 871         )
 872         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 873             return juju_info[DB_DATA.api_endpoints.key]
 874
 875     def _update_api_endpoints_db(self, endpoints: [str]):
 876         """
 877         Update API endpoints in Database
 878
 879         :param: List of endpoints
 880         """
 881         self.log.debug("Saving endpoints {} in database".format(endpoints))
 882
 883         juju_info = self.db.get_one(
 884             DB_DATA.api_endpoints.table,
 885             q_filter=DB_DATA.api_endpoints.filter,
 886             fail_on_empty=False,
 887         )
 888         # If it doesn't, then create it
 889         if not juju_info:
 890             try:
 891                 self.db.create(
 892                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 893                 )
 894             except DbException as e:
 895                 # Racing condition: check if another N2VC worker has created it
 896                 juju_info = self.db.get_one(
 897                     DB_DATA.api_endpoints.table,
 898                     q_filter=DB_DATA.api_endpoints.filter,
 899                     fail_on_empty=False,
 900                 )
 901                 if not juju_info:
 902                     raise e
 903         self.db.set_one(
 904             DB_DATA.api_endpoints.table,
 905             DB_DATA.api_endpoints.filter,
 906             {DB_DATA.api_endpoints.key: endpoints},
 907         )
 908
 909     def handle_exception(self, loop, context):
 910         # All unhandled exceptions by libjuju are handled here.
 911         pass
 912
 913     async def health_check(self, interval: float = 300.0):
 914         """
 915         Health check to make sure controller and controller_model connections are OK
 916
 917         :param: interval: Time in seconds between checks
 918         """
 919         while True:
 920             try:
 921                 controller = await self.get_controller()
 922                 # self.log.debug("VCA is alive")
 923             except Exception as e:
 924                 self.log.error("Health check to VCA failed: {}".format(e))
 925             finally:
 926                 await self.disconnect_controller(controller)
 927             await asyncio.sleep(interval)
 928
 929     async def list_models(self, contains: str = None) -> [str]:
 930         """List models with certain names
 931
 932         :param: contains:   String that is contained in model name
 933
 934         :retur: [models] Returns list of model names
 935         """
 936
 937         controller = await self.get_controller()
 938         try:
 939             models = await controller.list_models()
 940             if contains:
 941                 models = [model for model in models if contains in model]
 942             return models
 943         finally:
 944             await self.disconnect_controller(controller)