n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuModelAlreadyExists,
  33     JujuControllerFailedConnecting,
  34     JujuApplicationExists,
  35 )
  36 from n2vc.utils import DB_DATA
  37 from osm_common.dbbase import DbException
  38
  39
  40 class Libjuju:
  41     def __init__(
  42         self,
  43         endpoint: str,
  44         api_proxy: str,
  45         username: str,
  46         password: str,
  47         cacert: str,
  48         loop: asyncio.AbstractEventLoop = None,
  49         log: logging.Logger = None,
  50         db: dict = None,
  51         n2vc: N2VCConnector = None,
  52         apt_mirror: str = None,
  53         enable_os_upgrade: bool = True,
  54     ):
  55         """
  56         Constructor
  57
  58         :param: endpoint:               Endpoint of the juju controller (host:port)
  59         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  60         :param: username:               Juju username
  61         :param: password:               Juju password
  62         :param: cacert:                 Juju CA Certificate
  63         :param: loop:                   Asyncio loop
  64         :param: log:                    Logger
  65         :param: db:                     DB object
  66         :param: n2vc:                   N2VC object
  67         :param: apt_mirror:             APT Mirror
  68         :param: enable_os_upgrade:      Enable OS Upgrade
  69         """
  70
  71         self.log = log or logging.getLogger("Libjuju")
  72         self.db = db
  73         self.endpoints = self._get_api_endpoints_db() or [endpoint]
  74         self.api_proxy = api_proxy
  75         self.username = username
  76         self.password = password
  77         self.cacert = cacert
  78         self.loop = loop or asyncio.get_event_loop()
  79         self.n2vc = n2vc
  80
  81         # Generate config for models
  82         self.model_config = {}
  83         if apt_mirror:
  84             self.model_config["apt-mirror"] = apt_mirror
  85         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  86         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  87
  88         self.loop.set_exception_handler(self.handle_exception)
  89         self.creating_model = asyncio.Lock(loop=self.loop)
  90
  91         self.models = set()
  92         self.log.debug("Libjuju initialized!")
  93
  94         self.health_check_task = self.loop.create_task(self.health_check())
  95
  96     async def get_controller(self, timeout: float = 5.0) -> Controller:
  97         """
  98         Get controller
  99
 100         :param: timeout: Time in seconds to wait for controller to connect
 101         """
 102         controller = None
 103         try:
 104             controller = Controller(loop=self.loop)
 105             await asyncio.wait_for(
 106                 controller.connect(
 107                     endpoint=self.endpoints,
 108                     username=self.username,
 109                     password=self.password,
 110                     cacert=self.cacert,
 111                 ),
 112                 timeout=timeout,
 113             )
 114             endpoints = await controller.api_endpoints
 115             if self.endpoints != endpoints:
 116                 self.endpoints = endpoints
 117                 self._update_api_endpoints_db(self.endpoints)
 118             return controller
 119         except asyncio.CancelledError as e:
 120             raise e
 121         except Exception as e:
 122             self.log.error(
 123                 "Failed connecting to controller: {}...".format(self.endpoints)
 124             )
 125             if controller:
 126                 await self.disconnect_controller(controller)
 127             raise JujuControllerFailedConnecting(e)
 128
 129     async def disconnect(self):
 130         """Disconnect"""
 131         # Cancel health check task
 132         self.health_check_task.cancel()
 133         self.log.debug("Libjuju disconnected!")
 134
 135     async def disconnect_model(self, model: Model):
 136         """
 137         Disconnect model
 138
 139         :param: model: Model that will be disconnected
 140         """
 141         await model.disconnect()
 142
 143     async def disconnect_controller(self, controller: Controller):
 144         """
 145         Disconnect controller
 146
 147         :param: controller: Controller that will be disconnected
 148         """
 149         await controller.disconnect()
 150
 151     async def add_model(self, model_name: str, cloud_name: str):
 152         """
 153         Create model
 154
 155         :param: model_name: Model name
 156         :param: cloud_name: Cloud name
 157         """
 158
 159         # Get controller
 160         controller = await self.get_controller()
 161         model = None
 162         try:
 163             # Raise exception if model already exists
 164             if await self.model_exists(model_name, controller=controller):
 165                 raise JujuModelAlreadyExists(
 166                     "Model {} already exists.".format(model_name)
 167                 )
 168
 169             # Block until other workers have finished model creation
 170             while self.creating_model.locked():
 171                 await asyncio.sleep(0.1)
 172
 173             # If the model exists, return it from the controller
 174             if model_name in self.models:
 175                 return
 176
 177             # Create the model
 178             async with self.creating_model:
 179                 self.log.debug("Creating model {}".format(model_name))
 180                 model = await controller.add_model(
 181                     model_name,
 182                     config=self.model_config,
 183                     cloud_name=cloud_name,
 184                     credential_name=cloud_name,
 185                 )
 186                 self.models.add(model_name)
 187         finally:
 188             if model:
 189                 await self.disconnect_model(model)
 190             await self.disconnect_controller(controller)
 191
 192     async def get_model(
 193         self, controller: Controller, model_name: str, id=None
 194     ) -> Model:
 195         """
 196         Get model from controller
 197
 198         :param: controller: Controller
 199         :param: model_name: Model name
 200
 201         :return: Model: The created Juju model object
 202         """
 203         return await controller.get_model(model_name)
 204
 205     async def model_exists(
 206         self, model_name: str, controller: Controller = None
 207     ) -> bool:
 208         """
 209         Check if model exists
 210
 211         :param: controller: Controller
 212         :param: model_name: Model name
 213
 214         :return bool
 215         """
 216         need_to_disconnect = False
 217
 218         # Get controller if not passed
 219         if not controller:
 220             controller = await self.get_controller()
 221             need_to_disconnect = True
 222
 223         # Check if model exists
 224         try:
 225             return model_name in await controller.list_models()
 226         finally:
 227             if need_to_disconnect:
 228                 await self.disconnect_controller(controller)
 229
 230     async def get_model_status(self, model_name: str) -> FullStatus:
 231         """
 232         Get model status
 233
 234         :param: model_name: Model name
 235
 236         :return: Full status object
 237         """
 238         controller = await self.get_controller()
 239         model = await self.get_model(controller, model_name)
 240         try:
 241             return await model.get_status()
 242         finally:
 243             await self.disconnect_model(model)
 244             await self.disconnect_controller(controller)
 245
 246     async def create_machine(
 247         self,
 248         model_name: str,
 249         machine_id: str = None,
 250         db_dict: dict = None,
 251         progress_timeout: float = None,
 252         total_timeout: float = None,
 253         series: str = "xenial",
 254         wait: bool = True,
 255     ) -> (Machine, bool):
 256         """
 257         Create machine
 258
 259         :param: model_name:         Model name
 260         :param: machine_id:         Machine id
 261         :param: db_dict:            Dictionary with data of the DB to write the updates
 262         :param: progress_timeout:   Maximum time between two updates in the model
 263         :param: total_timeout:      Timeout for the entity to be active
 264         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 265         :param: wait:               Wait until machine is ready
 266
 267         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 268                                                 if the machine is new or it already existed
 269         """
 270         new = False
 271         machine = None
 272
 273         self.log.debug(
 274             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 275         )
 276
 277         # Get controller
 278         controller = await self.get_controller()
 279
 280         # Get model
 281         model = await self.get_model(controller, model_name)
 282         try:
 283             if machine_id is not None:
 284                 self.log.debug(
 285                     "Searching machine (id={}) in model {}".format(
 286                         machine_id, model_name
 287                     )
 288                 )
 289
 290                 # Get machines from model and get the machine with machine_id if exists
 291                 machines = await model.get_machines()
 292                 if machine_id in machines:
 293                     self.log.debug(
 294                         "Machine (id={}) found in model {}".format(
 295                             machine_id, model_name
 296                         )
 297                     )
 298                     machine = model.machines[machine_id]
 299                 else:
 300                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 301
 302             if machine is None:
 303                 self.log.debug("Creating a new machine in model {}".format(model_name))
 304
 305                 # Create machine
 306                 machine = await model.add_machine(
 307                     spec=None, constraints=None, disks=None, series=series
 308                 )
 309                 new = True
 310
 311                 # Wait until the machine is ready
 312                 self.log.debug(
 313                     "Wait until machine {} is ready in model {}".format(
 314                         machine.entity_id, model_name
 315                     )
 316                 )
 317                 if wait:
 318                     await JujuModelWatcher.wait_for(
 319                         model=model,
 320                         entity=machine,
 321                         progress_timeout=progress_timeout,
 322                         total_timeout=total_timeout,
 323                         db_dict=db_dict,
 324                         n2vc=self.n2vc,
 325                     )
 326         finally:
 327             await self.disconnect_model(model)
 328             await self.disconnect_controller(controller)
 329
 330         self.log.debug(
 331             "Machine {} ready at {} in model {}".format(
 332                 machine.entity_id, machine.dns_name, model_name
 333             )
 334         )
 335         return machine, new
 336
 337     async def provision_machine(
 338         self,
 339         model_name: str,
 340         hostname: str,
 341         username: str,
 342         private_key_path: str,
 343         db_dict: dict = None,
 344         progress_timeout: float = None,
 345         total_timeout: float = None,
 346     ) -> str:
 347         """
 348         Manually provisioning of a machine
 349
 350         :param: model_name:         Model name
 351         :param: hostname:           IP to access the machine
 352         :param: username:           Username to login to the machine
 353         :param: private_key_path:   Local path for the private key
 354         :param: db_dict:            Dictionary with data of the DB to write the updates
 355         :param: progress_timeout:   Maximum time between two updates in the model
 356         :param: total_timeout:      Timeout for the entity to be active
 357
 358         :return: (Entity): Machine id
 359         """
 360         self.log.debug(
 361             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 362                 model_name, hostname, username
 363             )
 364         )
 365
 366         # Get controller
 367         controller = await self.get_controller()
 368
 369         # Get model
 370         model = await self.get_model(controller, model_name)
 371
 372         try:
 373             # Get provisioner
 374             provisioner = AsyncSSHProvisioner(
 375                 host=hostname,
 376                 user=username,
 377                 private_key_path=private_key_path,
 378                 log=self.log,
 379             )
 380
 381             # Provision machine
 382             params = await provisioner.provision_machine()
 383
 384             params.jobs = ["JobHostUnits"]
 385
 386             self.log.debug("Adding machine to model")
 387             connection = model.connection()
 388             client_facade = client.ClientFacade.from_connection(connection)
 389
 390             results = await client_facade.AddMachines(params=[params])
 391             error = results.machines[0].error
 392
 393             if error:
 394                 msg = "Error adding machine: {}".format(error.message)
 395                 self.log.error(msg=msg)
 396                 raise ValueError(msg)
 397
 398             machine_id = results.machines[0].machine
 399
 400             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 401             asyncio.ensure_future(
 402                 provisioner.install_agent(
 403                     connection=connection,
 404                     nonce=params.nonce,
 405                     machine_id=machine_id,
 406                     api=self.api_proxy,
 407                 )
 408             )
 409
 410             machine = None
 411             for _ in range(10):
 412                 machine_list = await model.get_machines()
 413                 if machine_id in machine_list:
 414                     self.log.debug("Machine {} found in model!".format(machine_id))
 415                     machine = model.machines.get(machine_id)
 416                     break
 417                 await asyncio.sleep(2)
 418
 419             if machine is None:
 420                 msg = "Machine {} not found in model".format(machine_id)
 421                 self.log.error(msg=msg)
 422                 raise JujuMachineNotFound(msg)
 423
 424             self.log.debug(
 425                 "Wait until machine {} is ready in model {}".format(
 426                     machine.entity_id, model_name
 427                 )
 428             )
 429             await JujuModelWatcher.wait_for(
 430                 model=model,
 431                 entity=machine,
 432                 progress_timeout=progress_timeout,
 433                 total_timeout=total_timeout,
 434                 db_dict=db_dict,
 435                 n2vc=self.n2vc,
 436             )
 437         except Exception as e:
 438             raise e
 439         finally:
 440             await self.disconnect_model(model)
 441             await self.disconnect_controller(controller)
 442
 443         self.log.debug(
 444             "Machine provisioned {} in model {}".format(machine_id, model_name)
 445         )
 446
 447         return machine_id
 448
 449     async def deploy_charm(
 450         self,
 451         application_name: str,
 452         path: str,
 453         model_name: str,
 454         machine_id: str,
 455         db_dict: dict = None,
 456         progress_timeout: float = None,
 457         total_timeout: float = None,
 458         config: dict = None,
 459         series: str = None,
 460         num_units: int = 1,
 461     ):
 462         """Deploy charm
 463
 464         :param: application_name:   Application name
 465         :param: path:               Local path to the charm
 466         :param: model_name:         Model name
 467         :param: machine_id          ID of the machine
 468         :param: db_dict:            Dictionary with data of the DB to write the updates
 469         :param: progress_timeout:   Maximum time between two updates in the model
 470         :param: total_timeout:      Timeout for the entity to be active
 471         :param: config:             Config for the charm
 472         :param: series:             Series of the charm
 473         :param: num_units:          Number of units
 474
 475         :return: (juju.application.Application): Juju application
 476         """
 477         self.log.debug(
 478             "Deploying charm {} to machine {} in model ~{}".format(
 479                 application_name, machine_id, model_name
 480             )
 481         )
 482         self.log.debug("charm: {}".format(path))
 483
 484         # Get controller
 485         controller = await self.get_controller()
 486
 487         # Get model
 488         model = await self.get_model(controller, model_name)
 489
 490         try:
 491             application = None
 492             if application_name not in model.applications:
 493
 494                 if machine_id is not None:
 495                     if machine_id not in model.machines:
 496                         msg = "Machine {} not found in model".format(machine_id)
 497                         self.log.error(msg=msg)
 498                         raise JujuMachineNotFound(msg)
 499                     machine = model.machines[machine_id]
 500                     series = machine.series
 501
 502                 application = await model.deploy(
 503                     entity_url=path,
 504                     application_name=application_name,
 505                     channel="stable",
 506                     num_units=1,
 507                     series=series,
 508                     to=machine_id,
 509                     config=config,
 510                 )
 511
 512                 self.log.debug(
 513                     "Wait until application {} is ready in model {}".format(
 514                         application_name, model_name
 515                     )
 516                 )
 517                 if num_units > 1:
 518                     for _ in range(num_units - 1):
 519                         m, _ = await self.create_machine(model_name, wait=False)
 520                         await application.add_unit(to=m.entity_id)
 521
 522                 await JujuModelWatcher.wait_for(
 523                     model=model,
 524                     entity=application,
 525                     progress_timeout=progress_timeout,
 526                     total_timeout=total_timeout,
 527                     db_dict=db_dict,
 528                     n2vc=self.n2vc,
 529                 )
 530                 self.log.debug(
 531                     "Application {} is ready in model {}".format(
 532                         application_name, model_name
 533                     )
 534                 )
 535             else:
 536                 raise JujuApplicationExists(
 537                     "Application {} exists".format(application_name)
 538                 )
 539         finally:
 540             await self.disconnect_model(model)
 541             await self.disconnect_controller(controller)
 542
 543         return application
 544
 545     def _get_application(self, model: Model, application_name: str) -> Application:
 546         """Get application
 547
 548         :param: model:              Model object
 549         :param: application_name:   Application name
 550
 551         :return: juju.application.Application (or None if it doesn't exist)
 552         """
 553         if model.applications and application_name in model.applications:
 554             return model.applications[application_name]
 555
 556     async def execute_action(
 557         self,
 558         application_name: str,
 559         model_name: str,
 560         action_name: str,
 561         db_dict: dict = None,
 562         progress_timeout: float = None,
 563         total_timeout: float = None,
 564         **kwargs
 565     ):
 566         """Execute action
 567
 568         :param: application_name:   Application name
 569         :param: model_name:         Model name
 570         :param: cloud_name:         Cloud name
 571         :param: action_name:        Name of the action
 572         :param: db_dict:            Dictionary with data of the DB to write the updates
 573         :param: progress_timeout:   Maximum time between two updates in the model
 574         :param: total_timeout:      Timeout for the entity to be active
 575
 576         :return: (str, str): (output and status)
 577         """
 578         self.log.debug(
 579             "Executing action {} using params {}".format(action_name, kwargs)
 580         )
 581         # Get controller
 582         controller = await self.get_controller()
 583
 584         # Get model
 585         model = await self.get_model(controller, model_name)
 586
 587         try:
 588             # Get application
 589             application = self._get_application(
 590                 model, application_name=application_name,
 591             )
 592             if application is None:
 593                 raise JujuApplicationNotFound("Cannot execute action")
 594
 595             # Get unit
 596             unit = None
 597             for u in application.units:
 598                 if await u.is_leader_from_status():
 599                     unit = u
 600             if unit is None:
 601                 raise Exception("Cannot execute action: leader unit not found")
 602
 603             actions = await application.get_actions()
 604
 605             if action_name not in actions:
 606                 raise Exception(
 607                     "Action {} not in available actions".format(action_name)
 608                 )
 609
 610             action = await unit.run_action(action_name, **kwargs)
 611
 612             self.log.debug(
 613                 "Wait until action {} is completed in application {} (model={})".format(
 614                     action_name, application_name, model_name
 615                 )
 616             )
 617             await JujuModelWatcher.wait_for(
 618                 model=model,
 619                 entity=action,
 620                 progress_timeout=progress_timeout,
 621                 total_timeout=total_timeout,
 622                 db_dict=db_dict,
 623                 n2vc=self.n2vc,
 624             )
 625
 626             output = await model.get_action_output(action_uuid=action.entity_id)
 627             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 628             status = (
 629                 status[action.entity_id] if action.entity_id in status else "failed"
 630             )
 631
 632             self.log.debug(
 633                 "Action {} completed with status {} in application {} (model={})".format(
 634                     action_name, action.status, application_name, model_name
 635                 )
 636             )
 637         except Exception as e:
 638             raise e
 639         finally:
 640             await self.disconnect_model(model)
 641             await self.disconnect_controller(controller)
 642
 643         return output, status
 644
 645     async def get_actions(self, application_name: str, model_name: str) -> dict:
 646         """Get list of actions
 647
 648         :param: application_name: Application name
 649         :param: model_name: Model name
 650
 651         :return: Dict with this format
 652             {
 653                 "action_name": "Description of the action",
 654                 ...
 655             }
 656         """
 657         self.log.debug(
 658             "Getting list of actions for application {}".format(application_name)
 659         )
 660
 661         # Get controller
 662         controller = await self.get_controller()
 663
 664         # Get model
 665         model = await self.get_model(controller, model_name)
 666
 667         try:
 668             # Get application
 669             application = self._get_application(
 670                 model, application_name=application_name,
 671             )
 672
 673             # Return list of actions
 674             return await application.get_actions()
 675
 676         finally:
 677             # Disconnect from model and controller
 678             await self.disconnect_model(model)
 679             await self.disconnect_controller(controller)
 680
 681     async def add_relation(
 682         self,
 683         model_name: str,
 684         application_name_1: str,
 685         application_name_2: str,
 686         relation_1: str,
 687         relation_2: str,
 688     ):
 689         """Add relation
 690
 691         :param: model_name:             Model name
 692         :param: application_name_1      First application name
 693         :param: application_name_2:     Second application name
 694         :param: relation_1:             First relation name
 695         :param: relation_2:             Second relation name
 696         """
 697
 698         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 699
 700         # Get controller
 701         controller = await self.get_controller()
 702
 703         # Get model
 704         model = await self.get_model(controller, model_name)
 705
 706         # Build relation strings
 707         r1 = "{}:{}".format(application_name_1, relation_1)
 708         r2 = "{}:{}".format(application_name_2, relation_2)
 709
 710         # Add relation
 711         try:
 712             await model.add_relation(relation1=r1, relation2=r2)
 713         except JujuAPIError as e:
 714             if "not found" in e.message:
 715                 self.log.warning("Relation not found: {}".format(e.message))
 716                 return
 717             if "already exists" in e.message:
 718                 self.log.warning("Relation already exists: {}".format(e.message))
 719                 return
 720             # another exception, raise it
 721             raise e
 722         finally:
 723             await self.disconnect_model(model)
 724             await self.disconnect_controller(controller)
 725
 726     async def destroy_model(self, model_name: str, total_timeout: float):
 727         """
 728         Destroy model
 729
 730         :param: model_name:     Model name
 731         :param: total_timeout:  Timeout
 732         """
 733
 734         controller = await self.get_controller()
 735         model = await self.get_model(controller, model_name)
 736         try:
 737             self.log.debug("Destroying model {}".format(model_name))
 738             uuid = model.info.uuid
 739
 740             # Destroy applications
 741             for application_name in model.applications:
 742                 try:
 743                     await self.destroy_application(
 744                         model, application_name=application_name,
 745                     )
 746                 except Exception as e:
 747                     self.log.error(
 748                         "Error destroying application {} in model {}: {}".format(
 749                             application_name, model_name, e
 750                         )
 751                     )
 752
 753             # Destroy machines
 754             machines = await model.get_machines()
 755             for machine_id in machines:
 756                 try:
 757                     await self.destroy_machine(
 758                         model, machine_id=machine_id, total_timeout=total_timeout,
 759                     )
 760                 except asyncio.CancelledError:
 761                     raise
 762                 except Exception:
 763                     pass
 764
 765             # Disconnect model
 766             await self.disconnect_model(model)
 767
 768             # Destroy model
 769             if model_name in self.models:
 770                 self.models.remove(model_name)
 771
 772             await controller.destroy_model(uuid)
 773
 774             # Wait until model is destroyed
 775             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 776             last_exception = ""
 777
 778             if total_timeout is None:
 779                 total_timeout = 3600
 780             end = time.time() + total_timeout
 781             while time.time() < end:
 782                 try:
 783                     models = await controller.list_models()
 784                     if model_name not in models:
 785                         self.log.debug(
 786                             "The model {} ({}) was destroyed".format(model_name, uuid)
 787                         )
 788                         return
 789                 except asyncio.CancelledError:
 790                     raise
 791                 except Exception as e:
 792                     last_exception = e
 793                 await asyncio.sleep(5)
 794             raise Exception(
 795                 "Timeout waiting for model {} to be destroyed {}".format(
 796                     model_name, last_exception
 797                 )
 798             )
 799         finally:
 800             await self.disconnect_controller(controller)
 801
 802     async def destroy_application(self, model: Model, application_name: str):
 803         """
 804         Destroy application
 805
 806         :param: model:              Model object
 807         :param: application_name:   Application name
 808         """
 809         self.log.debug(
 810             "Destroying application {} in model {}".format(
 811                 application_name, model.info.name
 812             )
 813         )
 814         application = model.applications.get(application_name)
 815         if application:
 816             await application.destroy()
 817         else:
 818             self.log.warning("Application not found: {}".format(application_name))
 819
 820     async def destroy_machine(
 821         self, model: Model, machine_id: str, total_timeout: float = 3600
 822     ):
 823         """
 824         Destroy machine
 825
 826         :param: model:          Model object
 827         :param: machine_id:     Machine id
 828         :param: total_timeout:  Timeout in seconds
 829         """
 830         machines = await model.get_machines()
 831         if machine_id in machines:
 832             machine = model.machines[machine_id]
 833             # TODO: change this by machine.is_manual when this is upstreamed:
 834             # https://github.com/juju/python-libjuju/pull/396
 835             if "instance-id" in machine.safe_data and machine.safe_data[
 836                 "instance-id"
 837             ].startswith("manual:"):
 838                 await machine.destroy(force=True)
 839
 840                 # max timeout
 841                 end = time.time() + total_timeout
 842
 843                 # wait for machine removal
 844                 machines = await model.get_machines()
 845                 while machine_id in machines and time.time() < end:
 846                     self.log.debug(
 847                         "Waiting for machine {} is destroyed".format(machine_id)
 848                     )
 849                     await asyncio.sleep(0.5)
 850                     machines = await model.get_machines()
 851                 self.log.debug("Machine destroyed: {}".format(machine_id))
 852         else:
 853             self.log.debug("Machine not found: {}".format(machine_id))
 854
 855     async def configure_application(
 856         self, model_name: str, application_name: str, config: dict = None
 857     ):
 858         """Configure application
 859
 860         :param: model_name:         Model name
 861         :param: application_name:   Application name
 862         :param: config:             Config to apply to the charm
 863         """
 864         self.log.debug("Configuring application {}".format(application_name))
 865
 866         if config:
 867             try:
 868                 controller = await self.get_controller()
 869                 model = await self.get_model(controller, model_name)
 870                 application = self._get_application(
 871                     model, application_name=application_name,
 872                 )
 873                 await application.set_config(config)
 874             finally:
 875                 await self.disconnect_model(model)
 876                 await self.disconnect_controller(controller)
 877
 878     def _get_api_endpoints_db(self) -> [str]:
 879         """
 880         Get API Endpoints from DB
 881
 882         :return: List of API endpoints
 883         """
 884         self.log.debug("Getting endpoints from database")
 885
 886         juju_info = self.db.get_one(
 887             DB_DATA.api_endpoints.table,
 888             q_filter=DB_DATA.api_endpoints.filter,
 889             fail_on_empty=False,
 890         )
 891         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 892             return juju_info[DB_DATA.api_endpoints.key]
 893
 894     def _update_api_endpoints_db(self, endpoints: [str]):
 895         """
 896         Update API endpoints in Database
 897
 898         :param: List of endpoints
 899         """
 900         self.log.debug("Saving endpoints {} in database".format(endpoints))
 901
 902         juju_info = self.db.get_one(
 903             DB_DATA.api_endpoints.table,
 904             q_filter=DB_DATA.api_endpoints.filter,
 905             fail_on_empty=False,
 906         )
 907         # If it doesn't, then create it
 908         if not juju_info:
 909             try:
 910                 self.db.create(
 911                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 912                 )
 913             except DbException as e:
 914                 # Racing condition: check if another N2VC worker has created it
 915                 juju_info = self.db.get_one(
 916                     DB_DATA.api_endpoints.table,
 917                     q_filter=DB_DATA.api_endpoints.filter,
 918                     fail_on_empty=False,
 919                 )
 920                 if not juju_info:
 921                     raise e
 922         self.db.set_one(
 923             DB_DATA.api_endpoints.table,
 924             DB_DATA.api_endpoints.filter,
 925             {DB_DATA.api_endpoints.key: endpoints},
 926         )
 927
 928     def handle_exception(self, loop, context):
 929         # All unhandled exceptions by libjuju are handled here.
 930         pass
 931
 932     async def health_check(self, interval: float = 300.0):
 933         """
 934         Health check to make sure controller and controller_model connections are OK
 935
 936         :param: interval: Time in seconds between checks
 937         """
 938         while True:
 939             try:
 940                 controller = await self.get_controller()
 941                 # self.log.debug("VCA is alive")
 942             except Exception as e:
 943                 self.log.error("Health check to VCA failed: {}".format(e))
 944             finally:
 945                 await self.disconnect_controller(controller)
 946             await asyncio.sleep(interval)
 947
 948     async def list_models(self, contains: str = None) -> [str]:
 949         """List models with certain names
 950
 951         :param: contains:   String that is contained in model name
 952
 953         :retur: [models] Returns list of model names
 954         """
 955
 956         controller = await self.get_controller()
 957         try:
 958             models = await controller.list_models()
 959             if contains:
 960                 models = [model for model in models if contains in model]
 961             return models
 962         finally:
 963             await self.disconnect_controller(controller)