n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuModelAlreadyExists,
  33     JujuControllerFailedConnecting,
  34     JujuApplicationExists,
  35 )
  36 from n2vc.utils import DB_DATA
  37 from osm_common.dbbase import DbException
  38
  39
  40 class Libjuju:
  41     def __init__(
  42         self,
  43         endpoint: str,
  44         api_proxy: str,
  45         username: str,
  46         password: str,
  47         cacert: str,
  48         loop: asyncio.AbstractEventLoop = None,
  49         log: logging.Logger = None,
  50         db: dict = None,
  51         n2vc: N2VCConnector = None,
  52         apt_mirror: str = None,
  53         enable_os_upgrade: bool = True,
  54     ):
  55         """
  56         Constructor
  57
  58         :param: endpoint:               Endpoint of the juju controller (host:port)
  59         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  60         :param: username:               Juju username
  61         :param: password:               Juju password
  62         :param: cacert:                 Juju CA Certificate
  63         :param: loop:                   Asyncio loop
  64         :param: log:                    Logger
  65         :param: db:                     DB object
  66         :param: n2vc:                   N2VC object
  67         :param: apt_mirror:             APT Mirror
  68         :param: enable_os_upgrade:      Enable OS Upgrade
  69         """
  70
  71         self.log = log or logging.getLogger("Libjuju")
  72         self.db = db
  73         db_endpoints = self._get_api_endpoints_db()
  74         self.endpoints = db_endpoints or [endpoint]
  75         if db_endpoints is None:
  76             self._update_api_endpoints_db(self.endpoints)
  77         self.api_proxy = api_proxy
  78         self.username = username
  79         self.password = password
  80         self.cacert = cacert
  81         self.loop = loop or asyncio.get_event_loop()
  82         self.n2vc = n2vc
  83
  84         # Generate config for models
  85         self.model_config = {}
  86         if apt_mirror:
  87             self.model_config["apt-mirror"] = apt_mirror
  88         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  89         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  90
  91         self.loop.set_exception_handler(self.handle_exception)
  92         self.creating_model = asyncio.Lock(loop=self.loop)
  93
  94         self.models = set()
  95         self.log.debug("Libjuju initialized!")
  96
  97         self.health_check_task = self.loop.create_task(self.health_check())
  98
  99     async def get_controller(self, timeout: float = 5.0) -> Controller:
 100         """
 101         Get controller
 102
 103         :param: timeout: Time in seconds to wait for controller to connect
 104         """
 105         controller = None
 106         try:
 107             controller = Controller(loop=self.loop)
 108             await asyncio.wait_for(
 109                 controller.connect(
 110                     endpoint=self.endpoints,
 111                     username=self.username,
 112                     password=self.password,
 113                     cacert=self.cacert,
 114                 ),
 115                 timeout=timeout,
 116             )
 117             endpoints = await controller.api_endpoints
 118             if self.endpoints != endpoints:
 119                 self.endpoints = endpoints
 120                 self._update_api_endpoints_db(self.endpoints)
 121             return controller
 122         except asyncio.CancelledError as e:
 123             raise e
 124         except Exception as e:
 125             self.log.error(
 126                 "Failed connecting to controller: {}...".format(self.endpoints)
 127             )
 128             if controller:
 129                 await self.disconnect_controller(controller)
 130             raise JujuControllerFailedConnecting(e)
 131
 132     async def disconnect(self):
 133         """Disconnect"""
 134         # Cancel health check task
 135         self.health_check_task.cancel()
 136         self.log.debug("Libjuju disconnected!")
 137
 138     async def disconnect_model(self, model: Model):
 139         """
 140         Disconnect model
 141
 142         :param: model: Model that will be disconnected
 143         """
 144         await model.disconnect()
 145
 146     async def disconnect_controller(self, controller: Controller):
 147         """
 148         Disconnect controller
 149
 150         :param: controller: Controller that will be disconnected
 151         """
 152         await controller.disconnect()
 153
 154     async def add_model(self, model_name: str, cloud_name: str):
 155         """
 156         Create model
 157
 158         :param: model_name: Model name
 159         :param: cloud_name: Cloud name
 160         """
 161
 162         # Get controller
 163         controller = await self.get_controller()
 164         model = None
 165         try:
 166             # Raise exception if model already exists
 167             if await self.model_exists(model_name, controller=controller):
 168                 raise JujuModelAlreadyExists(
 169                     "Model {} already exists.".format(model_name)
 170                 )
 171
 172             # Block until other workers have finished model creation
 173             while self.creating_model.locked():
 174                 await asyncio.sleep(0.1)
 175
 176             # If the model exists, return it from the controller
 177             if model_name in self.models:
 178                 return
 179
 180             # Create the model
 181             async with self.creating_model:
 182                 self.log.debug("Creating model {}".format(model_name))
 183                 model = await controller.add_model(
 184                     model_name,
 185                     config=self.model_config,
 186                     cloud_name=cloud_name,
 187                     credential_name=cloud_name,
 188                 )
 189                 self.models.add(model_name)
 190         finally:
 191             if model:
 192                 await self.disconnect_model(model)
 193             await self.disconnect_controller(controller)
 194
 195     async def get_model(
 196         self, controller: Controller, model_name: str, id=None
 197     ) -> Model:
 198         """
 199         Get model from controller
 200
 201         :param: controller: Controller
 202         :param: model_name: Model name
 203
 204         :return: Model: The created Juju model object
 205         """
 206         return await controller.get_model(model_name)
 207
 208     async def model_exists(
 209         self, model_name: str, controller: Controller = None
 210     ) -> bool:
 211         """
 212         Check if model exists
 213
 214         :param: controller: Controller
 215         :param: model_name: Model name
 216
 217         :return bool
 218         """
 219         need_to_disconnect = False
 220
 221         # Get controller if not passed
 222         if not controller:
 223             controller = await self.get_controller()
 224             need_to_disconnect = True
 225
 226         # Check if model exists
 227         try:
 228             return model_name in await controller.list_models()
 229         finally:
 230             if need_to_disconnect:
 231                 await self.disconnect_controller(controller)
 232
 233     async def get_model_status(self, model_name: str) -> FullStatus:
 234         """
 235         Get model status
 236
 237         :param: model_name: Model name
 238
 239         :return: Full status object
 240         """
 241         controller = await self.get_controller()
 242         model = await self.get_model(controller, model_name)
 243         try:
 244             return await model.get_status()
 245         finally:
 246             await self.disconnect_model(model)
 247             await self.disconnect_controller(controller)
 248
 249     async def create_machine(
 250         self,
 251         model_name: str,
 252         machine_id: str = None,
 253         db_dict: dict = None,
 254         progress_timeout: float = None,
 255         total_timeout: float = None,
 256         series: str = "xenial",
 257         wait: bool = True,
 258     ) -> (Machine, bool):
 259         """
 260         Create machine
 261
 262         :param: model_name:         Model name
 263         :param: machine_id:         Machine id
 264         :param: db_dict:            Dictionary with data of the DB to write the updates
 265         :param: progress_timeout:   Maximum time between two updates in the model
 266         :param: total_timeout:      Timeout for the entity to be active
 267         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 268         :param: wait:               Wait until machine is ready
 269
 270         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 271                                                 if the machine is new or it already existed
 272         """
 273         new = False
 274         machine = None
 275
 276         self.log.debug(
 277             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 278         )
 279
 280         # Get controller
 281         controller = await self.get_controller()
 282
 283         # Get model
 284         model = await self.get_model(controller, model_name)
 285         try:
 286             if machine_id is not None:
 287                 self.log.debug(
 288                     "Searching machine (id={}) in model {}".format(
 289                         machine_id, model_name
 290                     )
 291                 )
 292
 293                 # Get machines from model and get the machine with machine_id if exists
 294                 machines = await model.get_machines()
 295                 if machine_id in machines:
 296                     self.log.debug(
 297                         "Machine (id={}) found in model {}".format(
 298                             machine_id, model_name
 299                         )
 300                     )
 301                     machine = model.machines[machine_id]
 302                 else:
 303                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 304
 305             if machine is None:
 306                 self.log.debug("Creating a new machine in model {}".format(model_name))
 307
 308                 # Create machine
 309                 machine = await model.add_machine(
 310                     spec=None, constraints=None, disks=None, series=series
 311                 )
 312                 new = True
 313
 314                 # Wait until the machine is ready
 315                 self.log.debug(
 316                     "Wait until machine {} is ready in model {}".format(
 317                         machine.entity_id, model_name
 318                     )
 319                 )
 320                 if wait:
 321                     await JujuModelWatcher.wait_for(
 322                         model=model,
 323                         entity=machine,
 324                         progress_timeout=progress_timeout,
 325                         total_timeout=total_timeout,
 326                         db_dict=db_dict,
 327                         n2vc=self.n2vc,
 328                     )
 329         finally:
 330             await self.disconnect_model(model)
 331             await self.disconnect_controller(controller)
 332
 333         self.log.debug(
 334             "Machine {} ready at {} in model {}".format(
 335                 machine.entity_id, machine.dns_name, model_name
 336             )
 337         )
 338         return machine, new
 339
 340     async def provision_machine(
 341         self,
 342         model_name: str,
 343         hostname: str,
 344         username: str,
 345         private_key_path: str,
 346         db_dict: dict = None,
 347         progress_timeout: float = None,
 348         total_timeout: float = None,
 349     ) -> str:
 350         """
 351         Manually provisioning of a machine
 352
 353         :param: model_name:         Model name
 354         :param: hostname:           IP to access the machine
 355         :param: username:           Username to login to the machine
 356         :param: private_key_path:   Local path for the private key
 357         :param: db_dict:            Dictionary with data of the DB to write the updates
 358         :param: progress_timeout:   Maximum time between two updates in the model
 359         :param: total_timeout:      Timeout for the entity to be active
 360
 361         :return: (Entity): Machine id
 362         """
 363         self.log.debug(
 364             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 365                 model_name, hostname, username
 366             )
 367         )
 368
 369         # Get controller
 370         controller = await self.get_controller()
 371
 372         # Get model
 373         model = await self.get_model(controller, model_name)
 374
 375         try:
 376             # Get provisioner
 377             provisioner = AsyncSSHProvisioner(
 378                 host=hostname,
 379                 user=username,
 380                 private_key_path=private_key_path,
 381                 log=self.log,
 382             )
 383
 384             # Provision machine
 385             params = await provisioner.provision_machine()
 386
 387             params.jobs = ["JobHostUnits"]
 388
 389             self.log.debug("Adding machine to model")
 390             connection = model.connection()
 391             client_facade = client.ClientFacade.from_connection(connection)
 392
 393             results = await client_facade.AddMachines(params=[params])
 394             error = results.machines[0].error
 395
 396             if error:
 397                 msg = "Error adding machine: {}".format(error.message)
 398                 self.log.error(msg=msg)
 399                 raise ValueError(msg)
 400
 401             machine_id = results.machines[0].machine
 402
 403             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 404             asyncio.ensure_future(
 405                 provisioner.install_agent(
 406                     connection=connection,
 407                     nonce=params.nonce,
 408                     machine_id=machine_id,
 409                     api=self.api_proxy,
 410                 )
 411             )
 412
 413             machine = None
 414             for _ in range(10):
 415                 machine_list = await model.get_machines()
 416                 if machine_id in machine_list:
 417                     self.log.debug("Machine {} found in model!".format(machine_id))
 418                     machine = model.machines.get(machine_id)
 419                     break
 420                 await asyncio.sleep(2)
 421
 422             if machine is None:
 423                 msg = "Machine {} not found in model".format(machine_id)
 424                 self.log.error(msg=msg)
 425                 raise JujuMachineNotFound(msg)
 426
 427             self.log.debug(
 428                 "Wait until machine {} is ready in model {}".format(
 429                     machine.entity_id, model_name
 430                 )
 431             )
 432             await JujuModelWatcher.wait_for(
 433                 model=model,
 434                 entity=machine,
 435                 progress_timeout=progress_timeout,
 436                 total_timeout=total_timeout,
 437                 db_dict=db_dict,
 438                 n2vc=self.n2vc,
 439             )
 440         except Exception as e:
 441             raise e
 442         finally:
 443             await self.disconnect_model(model)
 444             await self.disconnect_controller(controller)
 445
 446         self.log.debug(
 447             "Machine provisioned {} in model {}".format(machine_id, model_name)
 448         )
 449
 450         return machine_id
 451
 452     async def deploy_charm(
 453         self,
 454         application_name: str,
 455         path: str,
 456         model_name: str,
 457         machine_id: str,
 458         db_dict: dict = None,
 459         progress_timeout: float = None,
 460         total_timeout: float = None,
 461         config: dict = None,
 462         series: str = None,
 463         num_units: int = 1,
 464     ):
 465         """Deploy charm
 466
 467         :param: application_name:   Application name
 468         :param: path:               Local path to the charm
 469         :param: model_name:         Model name
 470         :param: machine_id          ID of the machine
 471         :param: db_dict:            Dictionary with data of the DB to write the updates
 472         :param: progress_timeout:   Maximum time between two updates in the model
 473         :param: total_timeout:      Timeout for the entity to be active
 474         :param: config:             Config for the charm
 475         :param: series:             Series of the charm
 476         :param: num_units:          Number of units
 477
 478         :return: (juju.application.Application): Juju application
 479         """
 480         self.log.debug(
 481             "Deploying charm {} to machine {} in model ~{}".format(
 482                 application_name, machine_id, model_name
 483             )
 484         )
 485         self.log.debug("charm: {}".format(path))
 486
 487         # Get controller
 488         controller = await self.get_controller()
 489
 490         # Get model
 491         model = await self.get_model(controller, model_name)
 492
 493         try:
 494             application = None
 495             if application_name not in model.applications:
 496
 497                 if machine_id is not None:
 498                     if machine_id not in model.machines:
 499                         msg = "Machine {} not found in model".format(machine_id)
 500                         self.log.error(msg=msg)
 501                         raise JujuMachineNotFound(msg)
 502                     machine = model.machines[machine_id]
 503                     series = machine.series
 504
 505                 application = await model.deploy(
 506                     entity_url=path,
 507                     application_name=application_name,
 508                     channel="stable",
 509                     num_units=1,
 510                     series=series,
 511                     to=machine_id,
 512                     config=config,
 513                 )
 514
 515                 self.log.debug(
 516                     "Wait until application {} is ready in model {}".format(
 517                         application_name, model_name
 518                     )
 519                 )
 520                 if num_units > 1:
 521                     for _ in range(num_units - 1):
 522                         m, _ = await self.create_machine(model_name, wait=False)
 523                         await application.add_unit(to=m.entity_id)
 524
 525                 await JujuModelWatcher.wait_for(
 526                     model=model,
 527                     entity=application,
 528                     progress_timeout=progress_timeout,
 529                     total_timeout=total_timeout,
 530                     db_dict=db_dict,
 531                     n2vc=self.n2vc,
 532                 )
 533                 self.log.debug(
 534                     "Application {} is ready in model {}".format(
 535                         application_name, model_name
 536                     )
 537                 )
 538             else:
 539                 raise JujuApplicationExists(
 540                     "Application {} exists".format(application_name)
 541                 )
 542         finally:
 543             await self.disconnect_model(model)
 544             await self.disconnect_controller(controller)
 545
 546         return application
 547
 548     def _get_application(self, model: Model, application_name: str) -> Application:
 549         """Get application
 550
 551         :param: model:              Model object
 552         :param: application_name:   Application name
 553
 554         :return: juju.application.Application (or None if it doesn't exist)
 555         """
 556         if model.applications and application_name in model.applications:
 557             return model.applications[application_name]
 558
 559     async def execute_action(
 560         self,
 561         application_name: str,
 562         model_name: str,
 563         action_name: str,
 564         db_dict: dict = None,
 565         progress_timeout: float = None,
 566         total_timeout: float = None,
 567         **kwargs
 568     ):
 569         """Execute action
 570
 571         :param: application_name:   Application name
 572         :param: model_name:         Model name
 573         :param: cloud_name:         Cloud name
 574         :param: action_name:        Name of the action
 575         :param: db_dict:            Dictionary with data of the DB to write the updates
 576         :param: progress_timeout:   Maximum time between two updates in the model
 577         :param: total_timeout:      Timeout for the entity to be active
 578
 579         :return: (str, str): (output and status)
 580         """
 581         self.log.debug(
 582             "Executing action {} using params {}".format(action_name, kwargs)
 583         )
 584         # Get controller
 585         controller = await self.get_controller()
 586
 587         # Get model
 588         model = await self.get_model(controller, model_name)
 589
 590         try:
 591             # Get application
 592             application = self._get_application(
 593                 model, application_name=application_name,
 594             )
 595             if application is None:
 596                 raise JujuApplicationNotFound("Cannot execute action")
 597
 598             # Get unit
 599             unit = None
 600             for u in application.units:
 601                 if await u.is_leader_from_status():
 602                     unit = u
 603             if unit is None:
 604                 raise Exception("Cannot execute action: leader unit not found")
 605
 606             actions = await application.get_actions()
 607
 608             if action_name not in actions:
 609                 raise Exception(
 610                     "Action {} not in available actions".format(action_name)
 611                 )
 612
 613             action = await unit.run_action(action_name, **kwargs)
 614
 615             self.log.debug(
 616                 "Wait until action {} is completed in application {} (model={})".format(
 617                     action_name, application_name, model_name
 618                 )
 619             )
 620             await JujuModelWatcher.wait_for(
 621                 model=model,
 622                 entity=action,
 623                 progress_timeout=progress_timeout,
 624                 total_timeout=total_timeout,
 625                 db_dict=db_dict,
 626                 n2vc=self.n2vc,
 627             )
 628
 629             output = await model.get_action_output(action_uuid=action.entity_id)
 630             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 631             status = (
 632                 status[action.entity_id] if action.entity_id in status else "failed"
 633             )
 634
 635             self.log.debug(
 636                 "Action {} completed with status {} in application {} (model={})".format(
 637                     action_name, action.status, application_name, model_name
 638                 )
 639             )
 640         except Exception as e:
 641             raise e
 642         finally:
 643             await self.disconnect_model(model)
 644             await self.disconnect_controller(controller)
 645
 646         return output, status
 647
 648     async def get_actions(self, application_name: str, model_name: str) -> dict:
 649         """Get list of actions
 650
 651         :param: application_name: Application name
 652         :param: model_name: Model name
 653
 654         :return: Dict with this format
 655             {
 656                 "action_name": "Description of the action",
 657                 ...
 658             }
 659         """
 660         self.log.debug(
 661             "Getting list of actions for application {}".format(application_name)
 662         )
 663
 664         # Get controller
 665         controller = await self.get_controller()
 666
 667         # Get model
 668         model = await self.get_model(controller, model_name)
 669
 670         try:
 671             # Get application
 672             application = self._get_application(
 673                 model, application_name=application_name,
 674             )
 675
 676             # Return list of actions
 677             return await application.get_actions()
 678
 679         finally:
 680             # Disconnect from model and controller
 681             await self.disconnect_model(model)
 682             await self.disconnect_controller(controller)
 683
 684     async def add_relation(
 685         self,
 686         model_name: str,
 687         application_name_1: str,
 688         application_name_2: str,
 689         relation_1: str,
 690         relation_2: str,
 691     ):
 692         """Add relation
 693
 694         :param: model_name:             Model name
 695         :param: application_name_1      First application name
 696         :param: application_name_2:     Second application name
 697         :param: relation_1:             First relation name
 698         :param: relation_2:             Second relation name
 699         """
 700
 701         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 702
 703         # Get controller
 704         controller = await self.get_controller()
 705
 706         # Get model
 707         model = await self.get_model(controller, model_name)
 708
 709         # Build relation strings
 710         r1 = "{}:{}".format(application_name_1, relation_1)
 711         r2 = "{}:{}".format(application_name_2, relation_2)
 712
 713         # Add relation
 714         try:
 715             await model.add_relation(relation1=r1, relation2=r2)
 716         except JujuAPIError as e:
 717             if "not found" in e.message:
 718                 self.log.warning("Relation not found: {}".format(e.message))
 719                 return
 720             if "already exists" in e.message:
 721                 self.log.warning("Relation already exists: {}".format(e.message))
 722                 return
 723             # another exception, raise it
 724             raise e
 725         finally:
 726             await self.disconnect_model(model)
 727             await self.disconnect_controller(controller)
 728
 729     async def destroy_model(self, model_name: str, total_timeout: float):
 730         """
 731         Destroy model
 732
 733         :param: model_name:     Model name
 734         :param: total_timeout:  Timeout
 735         """
 736
 737         controller = await self.get_controller()
 738         model = await self.get_model(controller, model_name)
 739         try:
 740             self.log.debug("Destroying model {}".format(model_name))
 741             uuid = model.info.uuid
 742
 743             # Destroy machines
 744             machines = await model.get_machines()
 745             for machine_id in machines:
 746                 try:
 747                     await self.destroy_machine(
 748                         model, machine_id=machine_id, total_timeout=total_timeout,
 749                     )
 750                 except asyncio.CancelledError:
 751                     raise
 752                 except Exception:
 753                     pass
 754
 755             # Disconnect model
 756             await self.disconnect_model(model)
 757
 758             # Destroy model
 759             if model_name in self.models:
 760                 self.models.remove(model_name)
 761
 762             await controller.destroy_model(uuid)
 763
 764             # Wait until model is destroyed
 765             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 766             last_exception = ""
 767
 768             if total_timeout is None:
 769                 total_timeout = 3600
 770             end = time.time() + total_timeout
 771             while time.time() < end:
 772                 try:
 773                     models = await controller.list_models()
 774                     if model_name not in models:
 775                         self.log.debug(
 776                             "The model {} ({}) was destroyed".format(model_name, uuid)
 777                         )
 778                         return
 779                 except asyncio.CancelledError:
 780                     raise
 781                 except Exception as e:
 782                     last_exception = e
 783                 await asyncio.sleep(5)
 784             raise Exception(
 785                 "Timeout waiting for model {} to be destroyed {}".format(
 786                     model_name, last_exception
 787                 )
 788             )
 789         finally:
 790             await self.disconnect_controller(controller)
 791
 792     async def destroy_application(self, model: Model, application_name: str):
 793         """
 794         Destroy application
 795
 796         :param: model:              Model object
 797         :param: application_name:   Application name
 798         """
 799         self.log.debug(
 800             "Destroying application {} in model {}".format(
 801                 application_name, model.info.name
 802             )
 803         )
 804         application = model.applications.get(application_name)
 805         if application:
 806             await application.destroy()
 807         else:
 808             self.log.warning("Application not found: {}".format(application_name))
 809
 810     async def destroy_machine(
 811         self, model: Model, machine_id: str, total_timeout: float = 3600
 812     ):
 813         """
 814         Destroy machine
 815
 816         :param: model:          Model object
 817         :param: machine_id:     Machine id
 818         :param: total_timeout:  Timeout in seconds
 819         """
 820         machines = await model.get_machines()
 821         if machine_id in machines:
 822             machine = model.machines[machine_id]
 823             await machine.destroy(force=True)
 824             # max timeout
 825             end = time.time() + total_timeout
 826
 827             # wait for machine removal
 828             machines = await model.get_machines()
 829             while machine_id in machines and time.time() < end:
 830                 self.log.debug(
 831                     "Waiting for machine {} is destroyed".format(machine_id)
 832                 )
 833                 await asyncio.sleep(0.5)
 834                 machines = await model.get_machines()
 835             self.log.debug("Machine destroyed: {}".format(machine_id))
 836         else:
 837             self.log.debug("Machine not found: {}".format(machine_id))
 838
 839     async def configure_application(
 840         self, model_name: str, application_name: str, config: dict = None
 841     ):
 842         """Configure application
 843
 844         :param: model_name:         Model name
 845         :param: application_name:   Application name
 846         :param: config:             Config to apply to the charm
 847         """
 848         self.log.debug("Configuring application {}".format(application_name))
 849
 850         if config:
 851             try:
 852                 controller = await self.get_controller()
 853                 model = await self.get_model(controller, model_name)
 854                 application = self._get_application(
 855                     model, application_name=application_name,
 856                 )
 857                 await application.set_config(config)
 858             finally:
 859                 await self.disconnect_model(model)
 860                 await self.disconnect_controller(controller)
 861
 862     def _get_api_endpoints_db(self) -> [str]:
 863         """
 864         Get API Endpoints from DB
 865
 866         :return: List of API endpoints
 867         """
 868         self.log.debug("Getting endpoints from database")
 869
 870         juju_info = self.db.get_one(
 871             DB_DATA.api_endpoints.table,
 872             q_filter=DB_DATA.api_endpoints.filter,
 873             fail_on_empty=False,
 874         )
 875         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 876             return juju_info[DB_DATA.api_endpoints.key]
 877
 878     def _update_api_endpoints_db(self, endpoints: [str]):
 879         """
 880         Update API endpoints in Database
 881
 882         :param: List of endpoints
 883         """
 884         self.log.debug("Saving endpoints {} in database".format(endpoints))
 885
 886         juju_info = self.db.get_one(
 887             DB_DATA.api_endpoints.table,
 888             q_filter=DB_DATA.api_endpoints.filter,
 889             fail_on_empty=False,
 890         )
 891         # If it doesn't, then create it
 892         if not juju_info:
 893             try:
 894                 self.db.create(
 895                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 896                 )
 897             except DbException as e:
 898                 # Racing condition: check if another N2VC worker has created it
 899                 juju_info = self.db.get_one(
 900                     DB_DATA.api_endpoints.table,
 901                     q_filter=DB_DATA.api_endpoints.filter,
 902                     fail_on_empty=False,
 903                 )
 904                 if not juju_info:
 905                     raise e
 906         self.db.set_one(
 907             DB_DATA.api_endpoints.table,
 908             DB_DATA.api_endpoints.filter,
 909             {DB_DATA.api_endpoints.key: endpoints},
 910         )
 911
 912     def handle_exception(self, loop, context):
 913         # All unhandled exceptions by libjuju are handled here.
 914         pass
 915
 916     async def health_check(self, interval: float = 300.0):
 917         """
 918         Health check to make sure controller and controller_model connections are OK
 919
 920         :param: interval: Time in seconds between checks
 921         """
 922         while True:
 923             try:
 924                 controller = await self.get_controller()
 925                 # self.log.debug("VCA is alive")
 926             except Exception as e:
 927                 self.log.error("Health check to VCA failed: {}".format(e))
 928             finally:
 929                 await self.disconnect_controller(controller)
 930             await asyncio.sleep(interval)
 931
 932     async def list_models(self, contains: str = None) -> [str]:
 933         """List models with certain names
 934
 935         :param: contains:   String that is contained in model name
 936
 937         :retur: [models] Returns list of model names
 938         """
 939
 940         controller = await self.get_controller()
 941         try:
 942             models = await controller.list_models()
 943             if contains:
 944                 models = [model for model in models if contains in model]
 945             return models
 946         finally:
 947             await self.disconnect_controller(controller)