n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuLeaderUnitNotFound,
  33     JujuActionNotFound,
  34     JujuModelAlreadyExists,
  35     JujuControllerFailedConnecting,
  36     JujuApplicationExists,
  37 )
  38 from n2vc.utils import DB_DATA
  39 from osm_common.dbbase import DbException
  40
  41
  42 class Libjuju:
  43     def __init__(
  44         self,
  45         endpoint: str,
  46         api_proxy: str,
  47         username: str,
  48         password: str,
  49         cacert: str,
  50         loop: asyncio.AbstractEventLoop = None,
  51         log: logging.Logger = None,
  52         db: dict = None,
  53         n2vc: N2VCConnector = None,
  54         apt_mirror: str = None,
  55         enable_os_upgrade: bool = True,
  56     ):
  57         """
  58         Constructor
  59
  60         :param: endpoint:               Endpoint of the juju controller (host:port)
  61         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  62         :param: username:               Juju username
  63         :param: password:               Juju password
  64         :param: cacert:                 Juju CA Certificate
  65         :param: loop:                   Asyncio loop
  66         :param: log:                    Logger
  67         :param: db:                     DB object
  68         :param: n2vc:                   N2VC object
  69         :param: apt_mirror:             APT Mirror
  70         :param: enable_os_upgrade:      Enable OS Upgrade
  71         """
  72
  73         self.log = log or logging.getLogger("Libjuju")
  74         self.db = db
  75         db_endpoints = self._get_api_endpoints_db()
  76         self.endpoints = db_endpoints or [endpoint]
  77         if db_endpoints is None:
  78             self._update_api_endpoints_db(self.endpoints)
  79         self.api_proxy = api_proxy
  80         self.username = username
  81         self.password = password
  82         self.cacert = cacert
  83         self.loop = loop or asyncio.get_event_loop()
  84         self.n2vc = n2vc
  85
  86         # Generate config for models
  87         self.model_config = {}
  88         if apt_mirror:
  89             self.model_config["apt-mirror"] = apt_mirror
  90         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  91         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  92
  93         self.loop.set_exception_handler(self.handle_exception)
  94         self.creating_model = asyncio.Lock(loop=self.loop)
  95
  96         self.models = set()
  97         self.log.debug("Libjuju initialized!")
  98
  99         self.health_check_task = self.loop.create_task(self.health_check())
 100
 101     async def get_controller(self, timeout: float = 5.0) -> Controller:
 102         """
 103         Get controller
 104
 105         :param: timeout: Time in seconds to wait for controller to connect
 106         """
 107         controller = None
 108         try:
 109             controller = Controller(loop=self.loop)
 110             await asyncio.wait_for(
 111                 controller.connect(
 112                     endpoint=self.endpoints,
 113                     username=self.username,
 114                     password=self.password,
 115                     cacert=self.cacert,
 116                 ),
 117                 timeout=timeout,
 118             )
 119             endpoints = await controller.api_endpoints
 120             if self.endpoints != endpoints:
 121                 self.endpoints = endpoints
 122                 self._update_api_endpoints_db(self.endpoints)
 123             return controller
 124         except asyncio.CancelledError as e:
 125             raise e
 126         except Exception as e:
 127             self.log.error(
 128                 "Failed connecting to controller: {}...".format(self.endpoints)
 129             )
 130             if controller:
 131                 await self.disconnect_controller(controller)
 132             raise JujuControllerFailedConnecting(e)
 133
 134     async def disconnect(self):
 135         """Disconnect"""
 136         # Cancel health check task
 137         self.health_check_task.cancel()
 138         self.log.debug("Libjuju disconnected!")
 139
 140     async def disconnect_model(self, model: Model):
 141         """
 142         Disconnect model
 143
 144         :param: model: Model that will be disconnected
 145         """
 146         await model.disconnect()
 147
 148     async def disconnect_controller(self, controller: Controller):
 149         """
 150         Disconnect controller
 151
 152         :param: controller: Controller that will be disconnected
 153         """
 154         await controller.disconnect()
 155
 156     async def add_model(self, model_name: str, cloud_name: str):
 157         """
 158         Create model
 159
 160         :param: model_name: Model name
 161         :param: cloud_name: Cloud name
 162         """
 163
 164         # Get controller
 165         controller = await self.get_controller()
 166         model = None
 167         try:
 168             # Raise exception if model already exists
 169             if await self.model_exists(model_name, controller=controller):
 170                 raise JujuModelAlreadyExists(
 171                     "Model {} already exists.".format(model_name)
 172                 )
 173
 174             # Block until other workers have finished model creation
 175             while self.creating_model.locked():
 176                 await asyncio.sleep(0.1)
 177
 178             # If the model exists, return it from the controller
 179             if model_name in self.models:
 180                 return
 181
 182             # Create the model
 183             async with self.creating_model:
 184                 self.log.debug("Creating model {}".format(model_name))
 185                 model = await controller.add_model(
 186                     model_name,
 187                     config=self.model_config,
 188                     cloud_name=cloud_name,
 189                     credential_name=cloud_name,
 190                 )
 191                 self.models.add(model_name)
 192         finally:
 193             if model:
 194                 await self.disconnect_model(model)
 195             await self.disconnect_controller(controller)
 196
 197     async def get_model(
 198         self, controller: Controller, model_name: str, id=None
 199     ) -> Model:
 200         """
 201         Get model from controller
 202
 203         :param: controller: Controller
 204         :param: model_name: Model name
 205
 206         :return: Model: The created Juju model object
 207         """
 208         return await controller.get_model(model_name)
 209
 210     async def model_exists(
 211         self, model_name: str, controller: Controller = None
 212     ) -> bool:
 213         """
 214         Check if model exists
 215
 216         :param: controller: Controller
 217         :param: model_name: Model name
 218
 219         :return bool
 220         """
 221         need_to_disconnect = False
 222
 223         # Get controller if not passed
 224         if not controller:
 225             controller = await self.get_controller()
 226             need_to_disconnect = True
 227
 228         # Check if model exists
 229         try:
 230             return model_name in await controller.list_models()
 231         finally:
 232             if need_to_disconnect:
 233                 await self.disconnect_controller(controller)
 234
 235     async def get_model_status(self, model_name: str) -> FullStatus:
 236         """
 237         Get model status
 238
 239         :param: model_name: Model name
 240
 241         :return: Full status object
 242         """
 243         controller = await self.get_controller()
 244         model = await self.get_model(controller, model_name)
 245         try:
 246             return await model.get_status()
 247         finally:
 248             await self.disconnect_model(model)
 249             await self.disconnect_controller(controller)
 250
 251     async def create_machine(
 252         self,
 253         model_name: str,
 254         machine_id: str = None,
 255         db_dict: dict = None,
 256         progress_timeout: float = None,
 257         total_timeout: float = None,
 258         series: str = "xenial",
 259         wait: bool = True,
 260     ) -> (Machine, bool):
 261         """
 262         Create machine
 263
 264         :param: model_name:         Model name
 265         :param: machine_id:         Machine id
 266         :param: db_dict:            Dictionary with data of the DB to write the updates
 267         :param: progress_timeout:   Maximum time between two updates in the model
 268         :param: total_timeout:      Timeout for the entity to be active
 269         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 270         :param: wait:               Wait until machine is ready
 271
 272         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 273                                                 if the machine is new or it already existed
 274         """
 275         new = False
 276         machine = None
 277
 278         self.log.debug(
 279             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 280         )
 281
 282         # Get controller
 283         controller = await self.get_controller()
 284
 285         # Get model
 286         model = await self.get_model(controller, model_name)
 287         try:
 288             if machine_id is not None:
 289                 self.log.debug(
 290                     "Searching machine (id={}) in model {}".format(
 291                         machine_id, model_name
 292                     )
 293                 )
 294
 295                 # Get machines from model and get the machine with machine_id if exists
 296                 machines = await model.get_machines()
 297                 if machine_id in machines:
 298                     self.log.debug(
 299                         "Machine (id={}) found in model {}".format(
 300                             machine_id, model_name
 301                         )
 302                     )
 303                     machine = machines[machine_id]
 304                 else:
 305                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 306
 307             if machine is None:
 308                 self.log.debug("Creating a new machine in model {}".format(model_name))
 309
 310                 # Create machine
 311                 machine = await model.add_machine(
 312                     spec=None, constraints=None, disks=None, series=series
 313                 )
 314                 new = True
 315
 316                 # Wait until the machine is ready
 317                 self.log.debug(
 318                     "Wait until machine {} is ready in model {}".format(
 319                         machine.entity_id, model_name
 320                     )
 321                 )
 322                 if wait:
 323                     await JujuModelWatcher.wait_for(
 324                         model=model,
 325                         entity=machine,
 326                         progress_timeout=progress_timeout,
 327                         total_timeout=total_timeout,
 328                         db_dict=db_dict,
 329                         n2vc=self.n2vc,
 330                     )
 331         finally:
 332             await self.disconnect_model(model)
 333             await self.disconnect_controller(controller)
 334
 335         self.log.debug(
 336             "Machine {} ready at {} in model {}".format(
 337                 machine.entity_id, machine.dns_name, model_name
 338             )
 339         )
 340         return machine, new
 341
 342     async def provision_machine(
 343         self,
 344         model_name: str,
 345         hostname: str,
 346         username: str,
 347         private_key_path: str,
 348         db_dict: dict = None,
 349         progress_timeout: float = None,
 350         total_timeout: float = None,
 351     ) -> str:
 352         """
 353         Manually provisioning of a machine
 354
 355         :param: model_name:         Model name
 356         :param: hostname:           IP to access the machine
 357         :param: username:           Username to login to the machine
 358         :param: private_key_path:   Local path for the private key
 359         :param: db_dict:            Dictionary with data of the DB to write the updates
 360         :param: progress_timeout:   Maximum time between two updates in the model
 361         :param: total_timeout:      Timeout for the entity to be active
 362
 363         :return: (Entity): Machine id
 364         """
 365         self.log.debug(
 366             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 367                 model_name, hostname, username
 368             )
 369         )
 370
 371         # Get controller
 372         controller = await self.get_controller()
 373
 374         # Get model
 375         model = await self.get_model(controller, model_name)
 376
 377         try:
 378             # Get provisioner
 379             provisioner = AsyncSSHProvisioner(
 380                 host=hostname,
 381                 user=username,
 382                 private_key_path=private_key_path,
 383                 log=self.log,
 384             )
 385
 386             # Provision machine
 387             params = await provisioner.provision_machine()
 388
 389             params.jobs = ["JobHostUnits"]
 390
 391             self.log.debug("Adding machine to model")
 392             connection = model.connection()
 393             client_facade = client.ClientFacade.from_connection(connection)
 394
 395             results = await client_facade.AddMachines(params=[params])
 396             error = results.machines[0].error
 397
 398             if error:
 399                 msg = "Error adding machine: {}".format(error.message)
 400                 self.log.error(msg=msg)
 401                 raise ValueError(msg)
 402
 403             machine_id = results.machines[0].machine
 404
 405             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 406             asyncio.ensure_future(
 407                 provisioner.install_agent(
 408                     connection=connection,
 409                     nonce=params.nonce,
 410                     machine_id=machine_id,
 411                     api=self.api_proxy,
 412                 )
 413             )
 414
 415             machine = None
 416             for _ in range(10):
 417                 machine_list = await model.get_machines()
 418                 if machine_id in machine_list:
 419                     self.log.debug("Machine {} found in model!".format(machine_id))
 420                     machine = model.machines.get(machine_id)
 421                     break
 422                 await asyncio.sleep(2)
 423
 424             if machine is None:
 425                 msg = "Machine {} not found in model".format(machine_id)
 426                 self.log.error(msg=msg)
 427                 raise JujuMachineNotFound(msg)
 428
 429             self.log.debug(
 430                 "Wait until machine {} is ready in model {}".format(
 431                     machine.entity_id, model_name
 432                 )
 433             )
 434             await JujuModelWatcher.wait_for(
 435                 model=model,
 436                 entity=machine,
 437                 progress_timeout=progress_timeout,
 438                 total_timeout=total_timeout,
 439                 db_dict=db_dict,
 440                 n2vc=self.n2vc,
 441             )
 442         except Exception as e:
 443             raise e
 444         finally:
 445             await self.disconnect_model(model)
 446             await self.disconnect_controller(controller)
 447
 448         self.log.debug(
 449             "Machine provisioned {} in model {}".format(machine_id, model_name)
 450         )
 451
 452         return machine_id
 453
 454     async def deploy_charm(
 455         self,
 456         application_name: str,
 457         path: str,
 458         model_name: str,
 459         machine_id: str,
 460         db_dict: dict = None,
 461         progress_timeout: float = None,
 462         total_timeout: float = None,
 463         config: dict = None,
 464         series: str = None,
 465         num_units: int = 1,
 466     ):
 467         """Deploy charm
 468
 469         :param: application_name:   Application name
 470         :param: path:               Local path to the charm
 471         :param: model_name:         Model name
 472         :param: machine_id          ID of the machine
 473         :param: db_dict:            Dictionary with data of the DB to write the updates
 474         :param: progress_timeout:   Maximum time between two updates in the model
 475         :param: total_timeout:      Timeout for the entity to be active
 476         :param: config:             Config for the charm
 477         :param: series:             Series of the charm
 478         :param: num_units:          Number of units
 479
 480         :return: (juju.application.Application): Juju application
 481         """
 482         self.log.debug(
 483             "Deploying charm {} to machine {} in model ~{}".format(
 484                 application_name, machine_id, model_name
 485             )
 486         )
 487         self.log.debug("charm: {}".format(path))
 488
 489         # Get controller
 490         controller = await self.get_controller()
 491
 492         # Get model
 493         model = await self.get_model(controller, model_name)
 494
 495         try:
 496             application = None
 497             if application_name not in model.applications:
 498
 499                 if machine_id is not None:
 500                     if machine_id not in model.machines:
 501                         msg = "Machine {} not found in model".format(machine_id)
 502                         self.log.error(msg=msg)
 503                         raise JujuMachineNotFound(msg)
 504                     machine = model.machines[machine_id]
 505                     series = machine.series
 506
 507                 application = await model.deploy(
 508                     entity_url=path,
 509                     application_name=application_name,
 510                     channel="stable",
 511                     num_units=1,
 512                     series=series,
 513                     to=machine_id,
 514                     config=config,
 515                 )
 516
 517                 self.log.debug(
 518                     "Wait until application {} is ready in model {}".format(
 519                         application_name, model_name
 520                     )
 521                 )
 522                 if num_units > 1:
 523                     for _ in range(num_units - 1):
 524                         m, _ = await self.create_machine(model_name, wait=False)
 525                         await application.add_unit(to=m.entity_id)
 526
 527                 await JujuModelWatcher.wait_for(
 528                     model=model,
 529                     entity=application,
 530                     progress_timeout=progress_timeout,
 531                     total_timeout=total_timeout,
 532                     db_dict=db_dict,
 533                     n2vc=self.n2vc,
 534                 )
 535                 self.log.debug(
 536                     "Application {} is ready in model {}".format(
 537                         application_name, model_name
 538                     )
 539                 )
 540             else:
 541                 raise JujuApplicationExists(
 542                     "Application {} exists".format(application_name)
 543                 )
 544         finally:
 545             await self.disconnect_model(model)
 546             await self.disconnect_controller(controller)
 547
 548         return application
 549
 550     def _get_application(self, model: Model, application_name: str) -> Application:
 551         """Get application
 552
 553         :param: model:              Model object
 554         :param: application_name:   Application name
 555
 556         :return: juju.application.Application (or None if it doesn't exist)
 557         """
 558         if model.applications and application_name in model.applications:
 559             return model.applications[application_name]
 560
 561     async def execute_action(
 562         self,
 563         application_name: str,
 564         model_name: str,
 565         action_name: str,
 566         db_dict: dict = None,
 567         progress_timeout: float = None,
 568         total_timeout: float = None,
 569         **kwargs
 570     ):
 571         """Execute action
 572
 573         :param: application_name:   Application name
 574         :param: model_name:         Model name
 575         :param: action_name:        Name of the action
 576         :param: db_dict:            Dictionary with data of the DB to write the updates
 577         :param: progress_timeout:   Maximum time between two updates in the model
 578         :param: total_timeout:      Timeout for the entity to be active
 579
 580         :return: (str, str): (output and status)
 581         """
 582         self.log.debug(
 583             "Executing action {} using params {}".format(action_name, kwargs)
 584         )
 585         # Get controller
 586         controller = await self.get_controller()
 587
 588         # Get model
 589         model = await self.get_model(controller, model_name)
 590
 591         try:
 592             # Get application
 593             application = self._get_application(
 594                 model, application_name=application_name,
 595             )
 596             if application is None:
 597                 raise JujuApplicationNotFound("Cannot execute action")
 598
 599             # Get unit
 600             unit = None
 601             for u in application.units:
 602                 if await u.is_leader_from_status():
 603                     unit = u
 604             if unit is None:
 605                 raise JujuLeaderUnitNotFound("Cannot execute action: leader unit not found")
 606
 607             actions = await application.get_actions()
 608
 609             if action_name not in actions:
 610                 raise JujuActionNotFound(
 611                     "Action {} not in available actions".format(action_name)
 612                 )
 613
 614             action = await unit.run_action(action_name, **kwargs)
 615
 616             self.log.debug(
 617                 "Wait until action {} is completed in application {} (model={})".format(
 618                     action_name, application_name, model_name
 619                 )
 620             )
 621             await JujuModelWatcher.wait_for(
 622                 model=model,
 623                 entity=action,
 624                 progress_timeout=progress_timeout,
 625                 total_timeout=total_timeout,
 626                 db_dict=db_dict,
 627                 n2vc=self.n2vc,
 628             )
 629
 630             output = await model.get_action_output(action_uuid=action.entity_id)
 631             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 632             status = (
 633                 status[action.entity_id] if action.entity_id in status else "failed"
 634             )
 635
 636             self.log.debug(
 637                 "Action {} completed with status {} in application {} (model={})".format(
 638                     action_name, action.status, application_name, model_name
 639                 )
 640             )
 641         finally:
 642             await self.disconnect_model(model)
 643             await self.disconnect_controller(controller)
 644
 645         return output, status
 646
 647     async def get_actions(self, application_name: str, model_name: str) -> dict:
 648         """Get list of actions
 649
 650         :param: application_name: Application name
 651         :param: model_name: Model name
 652
 653         :return: Dict with this format
 654             {
 655                 "action_name": "Description of the action",
 656                 ...
 657             }
 658         """
 659         self.log.debug(
 660             "Getting list of actions for application {}".format(application_name)
 661         )
 662
 663         # Get controller
 664         controller = await self.get_controller()
 665
 666         # Get model
 667         model = await self.get_model(controller, model_name)
 668
 669         try:
 670             # Get application
 671             application = self._get_application(
 672                 model, application_name=application_name,
 673             )
 674
 675             # Return list of actions
 676             return await application.get_actions()
 677
 678         finally:
 679             # Disconnect from model and controller
 680             await self.disconnect_model(model)
 681             await self.disconnect_controller(controller)
 682
 683     async def add_relation(
 684         self,
 685         model_name: str,
 686         application_name_1: str,
 687         application_name_2: str,
 688         relation_1: str,
 689         relation_2: str,
 690     ):
 691         """Add relation
 692
 693         :param: model_name:             Model name
 694         :param: application_name_1      First application name
 695         :param: application_name_2:     Second application name
 696         :param: relation_1:             First relation name
 697         :param: relation_2:             Second relation name
 698         """
 699
 700         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 701
 702         # Get controller
 703         controller = await self.get_controller()
 704
 705         # Get model
 706         model = await self.get_model(controller, model_name)
 707
 708         # Build relation strings
 709         r1 = "{}:{}".format(application_name_1, relation_1)
 710         r2 = "{}:{}".format(application_name_2, relation_2)
 711
 712         # Add relation
 713         try:
 714             await model.add_relation(relation1=r1, relation2=r2)
 715         except JujuAPIError as e:
 716             if "not found" in e.message:
 717                 self.log.warning("Relation not found: {}".format(e.message))
 718                 return
 719             if "already exists" in e.message:
 720                 self.log.warning("Relation already exists: {}".format(e.message))
 721                 return
 722             # another exception, raise it
 723             raise e
 724         finally:
 725             await self.disconnect_model(model)
 726             await self.disconnect_controller(controller)
 727
 728     async def destroy_model(self, model_name: str, total_timeout: float):
 729         """
 730         Destroy model
 731
 732         :param: model_name:     Model name
 733         :param: total_timeout:  Timeout
 734         """
 735
 736         controller = await self.get_controller()
 737         model = await self.get_model(controller, model_name)
 738         try:
 739             self.log.debug("Destroying model {}".format(model_name))
 740             uuid = model.info.uuid
 741
 742             # Destroy machines
 743             machines = await model.get_machines()
 744             for machine_id in machines:
 745                 try:
 746                     await self.destroy_machine(
 747                         model, machine_id=machine_id, total_timeout=total_timeout,
 748                     )
 749                 except asyncio.CancelledError:
 750                     raise
 751                 except Exception:
 752                     pass
 753
 754             # Disconnect model
 755             await self.disconnect_model(model)
 756
 757             # Destroy model
 758             if model_name in self.models:
 759                 self.models.remove(model_name)
 760
 761             await controller.destroy_model(uuid)
 762
 763             # Wait until model is destroyed
 764             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 765             last_exception = ""
 766
 767             if total_timeout is None:
 768                 total_timeout = 3600
 769             end = time.time() + total_timeout
 770             while time.time() < end:
 771                 try:
 772                     models = await controller.list_models()
 773                     if model_name not in models:
 774                         self.log.debug(
 775                             "The model {} ({}) was destroyed".format(model_name, uuid)
 776                         )
 777                         return
 778                 except asyncio.CancelledError:
 779                     raise
 780                 except Exception as e:
 781                     last_exception = e
 782                 await asyncio.sleep(5)
 783             raise Exception(
 784                 "Timeout waiting for model {} to be destroyed {}".format(
 785                     model_name, last_exception
 786                 )
 787             )
 788         finally:
 789             await self.disconnect_controller(controller)
 790
 791     async def destroy_application(self, model: Model, application_name: str):
 792         """
 793         Destroy application
 794
 795         :param: model:              Model object
 796         :param: application_name:   Application name
 797         """
 798         self.log.debug(
 799             "Destroying application {} in model {}".format(
 800                 application_name, model.info.name
 801             )
 802         )
 803         application = model.applications.get(application_name)
 804         if application:
 805             await application.destroy()
 806         else:
 807             self.log.warning("Application not found: {}".format(application_name))
 808
 809     async def destroy_machine(
 810         self, model: Model, machine_id: str, total_timeout: float = 3600
 811     ):
 812         """
 813         Destroy machine
 814
 815         :param: model:          Model object
 816         :param: machine_id:     Machine id
 817         :param: total_timeout:  Timeout in seconds
 818         """
 819         machines = await model.get_machines()
 820         if machine_id in machines:
 821             machine = machines[machine_id]
 822             await machine.destroy(force=True)
 823             # max timeout
 824             end = time.time() + total_timeout
 825
 826             # wait for machine removal
 827             machines = await model.get_machines()
 828             while machine_id in machines and time.time() < end:
 829                 self.log.debug(
 830                     "Waiting for machine {} is destroyed".format(machine_id)
 831                 )
 832                 await asyncio.sleep(0.5)
 833                 machines = await model.get_machines()
 834             self.log.debug("Machine destroyed: {}".format(machine_id))
 835         else:
 836             self.log.debug("Machine not found: {}".format(machine_id))
 837
 838     async def configure_application(
 839         self, model_name: str, application_name: str, config: dict = None
 840     ):
 841         """Configure application
 842
 843         :param: model_name:         Model name
 844         :param: application_name:   Application name
 845         :param: config:             Config to apply to the charm
 846         """
 847         self.log.debug("Configuring application {}".format(application_name))
 848
 849         if config:
 850             try:
 851                 controller = await self.get_controller()
 852                 model = await self.get_model(controller, model_name)
 853                 application = self._get_application(
 854                     model, application_name=application_name,
 855                 )
 856                 await application.set_config(config)
 857             finally:
 858                 await self.disconnect_model(model)
 859                 await self.disconnect_controller(controller)
 860
 861     def _get_api_endpoints_db(self) -> [str]:
 862         """
 863         Get API Endpoints from DB
 864
 865         :return: List of API endpoints
 866         """
 867         self.log.debug("Getting endpoints from database")
 868
 869         juju_info = self.db.get_one(
 870             DB_DATA.api_endpoints.table,
 871             q_filter=DB_DATA.api_endpoints.filter,
 872             fail_on_empty=False,
 873         )
 874         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 875             return juju_info[DB_DATA.api_endpoints.key]
 876
 877     def _update_api_endpoints_db(self, endpoints: [str]):
 878         """
 879         Update API endpoints in Database
 880
 881         :param: List of endpoints
 882         """
 883         self.log.debug("Saving endpoints {} in database".format(endpoints))
 884
 885         juju_info = self.db.get_one(
 886             DB_DATA.api_endpoints.table,
 887             q_filter=DB_DATA.api_endpoints.filter,
 888             fail_on_empty=False,
 889         )
 890         # If it doesn't, then create it
 891         if not juju_info:
 892             try:
 893                 self.db.create(
 894                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 895                 )
 896             except DbException as e:
 897                 # Racing condition: check if another N2VC worker has created it
 898                 juju_info = self.db.get_one(
 899                     DB_DATA.api_endpoints.table,
 900                     q_filter=DB_DATA.api_endpoints.filter,
 901                     fail_on_empty=False,
 902                 )
 903                 if not juju_info:
 904                     raise e
 905         self.db.set_one(
 906             DB_DATA.api_endpoints.table,
 907             DB_DATA.api_endpoints.filter,
 908             {DB_DATA.api_endpoints.key: endpoints},
 909         )
 910
 911     def handle_exception(self, loop, context):
 912         # All unhandled exceptions by libjuju are handled here.
 913         pass
 914
 915     async def health_check(self, interval: float = 300.0):
 916         """
 917         Health check to make sure controller and controller_model connections are OK
 918
 919         :param: interval: Time in seconds between checks
 920         """
 921         while True:
 922             try:
 923                 controller = await self.get_controller()
 924                 # self.log.debug("VCA is alive")
 925             except Exception as e:
 926                 self.log.error("Health check to VCA failed: {}".format(e))
 927             finally:
 928                 await self.disconnect_controller(controller)
 929             await asyncio.sleep(interval)
 930
 931     async def list_models(self, contains: str = None) -> [str]:
 932         """List models with certain names
 933
 934         :param: contains:   String that is contained in model name
 935
 936         :retur: [models] Returns list of model names
 937         """
 938
 939         controller = await self.get_controller()
 940         try:
 941             models = await controller.list_models()
 942             if contains:
 943                 models = [model for model in models if contains in model]
 944             return models
 945         finally:
 946             await self.disconnect_controller(controller)