n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 from juju.controller import Controller
  18 from juju.client import client
  19 import time
  20
  21 from juju.errors import JujuAPIError
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.client._definitions import FullStatus
  26 from n2vc.juju_watcher import JujuModelWatcher
  27 from n2vc.provisioner import AsyncSSHProvisioner
  28 from n2vc.n2vc_conn import N2VCConnector
  29 from n2vc.exceptions import (
  30     JujuMachineNotFound,
  31     JujuApplicationNotFound,
  32     JujuModelAlreadyExists,
  33     JujuControllerFailedConnecting,
  34     JujuApplicationExists,
  35 )
  36 from n2vc.utils import DB_DATA
  37 from osm_common.dbbase import DbException
  38
  39
  40 class Libjuju:
  41     def __init__(
  42         self,
  43         endpoint: str,
  44         api_proxy: str,
  45         username: str,
  46         password: str,
  47         cacert: str,
  48         loop: asyncio.AbstractEventLoop = None,
  49         log: logging.Logger = None,
  50         db: dict = None,
  51         n2vc: N2VCConnector = None,
  52         apt_mirror: str = None,
  53         enable_os_upgrade: bool = True,
  54     ):
  55         """
  56         Constructor
  57
  58         :param: endpoint:               Endpoint of the juju controller (host:port)
  59         :param: api_proxy:              Endpoint of the juju controller - Reachable from the VNFs
  60         :param: username:               Juju username
  61         :param: password:               Juju password
  62         :param: cacert:                 Juju CA Certificate
  63         :param: loop:                   Asyncio loop
  64         :param: log:                    Logger
  65         :param: db:                     DB object
  66         :param: n2vc:                   N2VC object
  67         :param: apt_mirror:             APT Mirror
  68         :param: enable_os_upgrade:      Enable OS Upgrade
  69         """
  70
  71         self.log = log or logging.getLogger("Libjuju")
  72         self.db = db
  73         self.endpoints = self._get_api_endpoints_db() or [endpoint]
  74         self.api_proxy = api_proxy
  75         self.username = username
  76         self.password = password
  77         self.cacert = cacert
  78         self.loop = loop or asyncio.get_event_loop()
  79         self.n2vc = n2vc
  80
  81         # Generate config for models
  82         self.model_config = {}
  83         if apt_mirror:
  84             self.model_config["apt-mirror"] = apt_mirror
  85         self.model_config["enable-os-refresh-update"] = enable_os_upgrade
  86         self.model_config["enable-os-upgrade"] = enable_os_upgrade
  87
  88         self.loop.set_exception_handler(self.handle_exception)
  89         self.creating_model = asyncio.Lock(loop=self.loop)
  90
  91         self.models = set()
  92         self.log.debug("Libjuju initialized!")
  93
  94         self.health_check_task = self.loop.create_task(self.health_check())
  95
  96     async def get_controller(self, timeout: float = 5.0) -> Controller:
  97         """
  98         Get controller
  99
 100         :param: timeout: Time in seconds to wait for controller to connect
 101         """
 102         controller = None
 103         try:
 104             controller = Controller(loop=self.loop)
 105             await asyncio.wait_for(
 106                 controller.connect(
 107                     endpoint=self.endpoints,
 108                     username=self.username,
 109                     password=self.password,
 110                     cacert=self.cacert,
 111                 ),
 112                 timeout=timeout,
 113             )
 114             endpoints = await controller.api_endpoints
 115             if self.endpoints != endpoints:
 116                 self.endpoints = endpoints
 117                 self._update_api_endpoints_db(self.endpoints)
 118             return controller
 119         except asyncio.CancelledError as e:
 120             raise e
 121         except Exception as e:
 122             self.log.error(
 123                 "Failed connecting to controller: {}...".format(self.endpoints)
 124             )
 125             if controller:
 126                 await self.disconnect_controller(controller)
 127             raise JujuControllerFailedConnecting(e)
 128
 129     async def disconnect(self):
 130         """Disconnect"""
 131         # Cancel health check task
 132         self.health_check_task.cancel()
 133         self.log.debug("Libjuju disconnected!")
 134
 135     async def disconnect_model(self, model: Model):
 136         """
 137         Disconnect model
 138
 139         :param: model: Model that will be disconnected
 140         """
 141         await model.disconnect()
 142
 143     async def disconnect_controller(self, controller: Controller):
 144         """
 145         Disconnect controller
 146
 147         :param: controller: Controller that will be disconnected
 148         """
 149         await controller.disconnect()
 150
 151     async def add_model(self, model_name: str, cloud_name: str):
 152         """
 153         Create model
 154
 155         :param: model_name: Model name
 156         :param: cloud_name: Cloud name
 157         """
 158
 159         # Get controller
 160         controller = await self.get_controller()
 161         model = None
 162         try:
 163             # Raise exception if model already exists
 164             if await self.model_exists(model_name, controller=controller):
 165                 raise JujuModelAlreadyExists(
 166                     "Model {} already exists.".format(model_name)
 167                 )
 168
 169             # Block until other workers have finished model creation
 170             while self.creating_model.locked():
 171                 await asyncio.sleep(0.1)
 172
 173             # If the model exists, return it from the controller
 174             if model_name in self.models:
 175                 return
 176
 177             # Create the model
 178             async with self.creating_model:
 179                 self.log.debug("Creating model {}".format(model_name))
 180                 model = await controller.add_model(
 181                     model_name,
 182                     config=self.model_config,
 183                     cloud_name=cloud_name,
 184                     credential_name=cloud_name,
 185                 )
 186                 self.models.add(model_name)
 187         finally:
 188             if model:
 189                 await self.disconnect_model(model)
 190             await self.disconnect_controller(controller)
 191
 192     async def get_model(
 193         self, controller: Controller, model_name: str, id=None
 194     ) -> Model:
 195         """
 196         Get model from controller
 197
 198         :param: controller: Controller
 199         :param: model_name: Model name
 200
 201         :return: Model: The created Juju model object
 202         """
 203         return await controller.get_model(model_name)
 204
 205     async def model_exists(
 206         self, model_name: str, controller: Controller = None
 207     ) -> bool:
 208         """
 209         Check if model exists
 210
 211         :param: controller: Controller
 212         :param: model_name: Model name
 213
 214         :return bool
 215         """
 216         need_to_disconnect = False
 217
 218         # Get controller if not passed
 219         if not controller:
 220             controller = await self.get_controller()
 221             need_to_disconnect = True
 222
 223         # Check if model exists
 224         try:
 225             return model_name in await controller.list_models()
 226         finally:
 227             if need_to_disconnect:
 228                 await self.disconnect_controller(controller)
 229
 230     async def get_model_status(self, model_name: str) -> FullStatus:
 231         """
 232         Get model status
 233
 234         :param: model_name: Model name
 235
 236         :return: Full status object
 237         """
 238         controller = await self.get_controller()
 239         model = await self.get_model(controller, model_name)
 240         try:
 241             return await model.get_status()
 242         finally:
 243             await self.disconnect_model(model)
 244             await self.disconnect_controller(controller)
 245
 246     async def create_machine(
 247         self,
 248         model_name: str,
 249         machine_id: str = None,
 250         db_dict: dict = None,
 251         progress_timeout: float = None,
 252         total_timeout: float = None,
 253         series: str = "xenial",
 254     ) -> (Machine, bool):
 255         """
 256         Create machine
 257
 258         :param: model_name:         Model name
 259         :param: machine_id:         Machine id
 260         :param: db_dict:            Dictionary with data of the DB to write the updates
 261         :param: progress_timeout:   Maximum time between two updates in the model
 262         :param: total_timeout:      Timeout for the entity to be active
 263
 264         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 265                                                 if the machine is new or it already existed
 266         """
 267         new = False
 268         machine = None
 269
 270         self.log.debug(
 271             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 272         )
 273
 274         # Get controller
 275         controller = await self.get_controller()
 276
 277         # Get model
 278         model = await self.get_model(controller, model_name)
 279         try:
 280             if machine_id is not None:
 281                 self.log.debug(
 282                     "Searching machine (id={}) in model {}".format(
 283                         machine_id, model_name
 284                     )
 285                 )
 286
 287                 # Get machines from model and get the machine with machine_id if exists
 288                 machines = await model.get_machines()
 289                 if machine_id in machines:
 290                     self.log.debug(
 291                         "Machine (id={}) found in model {}".format(
 292                             machine_id, model_name
 293                         )
 294                     )
 295                     machine = model.machines[machine_id]
 296                 else:
 297                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 298
 299             if machine is None:
 300                 self.log.debug("Creating a new machine in model {}".format(model_name))
 301
 302                 # Create machine
 303                 machine = await model.add_machine(
 304                     spec=None, constraints=None, disks=None, series=series
 305                 )
 306                 new = True
 307
 308                 # Wait until the machine is ready
 309                 self.log.debug(
 310                     "Wait until machine {} is ready in model {}".format(
 311                         machine.entity_id, model_name
 312                     )
 313                 )
 314                 await JujuModelWatcher.wait_for(
 315                     model=model,
 316                     entity=machine,
 317                     progress_timeout=progress_timeout,
 318                     total_timeout=total_timeout,
 319                     db_dict=db_dict,
 320                     n2vc=self.n2vc,
 321                 )
 322         finally:
 323             await self.disconnect_model(model)
 324             await self.disconnect_controller(controller)
 325
 326         self.log.debug(
 327             "Machine {} ready at {} in model {}".format(
 328                 machine.entity_id, machine.dns_name, model_name
 329             )
 330         )
 331         return machine, new
 332
 333     async def provision_machine(
 334         self,
 335         model_name: str,
 336         hostname: str,
 337         username: str,
 338         private_key_path: str,
 339         db_dict: dict = None,
 340         progress_timeout: float = None,
 341         total_timeout: float = None,
 342     ) -> str:
 343         """
 344         Manually provisioning of a machine
 345
 346         :param: model_name:         Model name
 347         :param: hostname:           IP to access the machine
 348         :param: username:           Username to login to the machine
 349         :param: private_key_path:   Local path for the private key
 350         :param: db_dict:            Dictionary with data of the DB to write the updates
 351         :param: progress_timeout:   Maximum time between two updates in the model
 352         :param: total_timeout:      Timeout for the entity to be active
 353
 354         :return: (Entity): Machine id
 355         """
 356         self.log.debug(
 357             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 358                 model_name, hostname, username
 359             )
 360         )
 361
 362         # Get controller
 363         controller = await self.get_controller()
 364
 365         # Get model
 366         model = await self.get_model(controller, model_name)
 367
 368         try:
 369             # Get provisioner
 370             provisioner = AsyncSSHProvisioner(
 371                 host=hostname,
 372                 user=username,
 373                 private_key_path=private_key_path,
 374                 log=self.log,
 375             )
 376
 377             # Provision machine
 378             params = await provisioner.provision_machine()
 379
 380             params.jobs = ["JobHostUnits"]
 381
 382             self.log.debug("Adding machine to model")
 383             connection = model.connection()
 384             client_facade = client.ClientFacade.from_connection(connection)
 385
 386             results = await client_facade.AddMachines(params=[params])
 387             error = results.machines[0].error
 388
 389             if error:
 390                 msg = "Error adding machine: {}".format(error.message)
 391                 self.log.error(msg=msg)
 392                 raise ValueError(msg)
 393
 394             machine_id = results.machines[0].machine
 395
 396             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 397             asyncio.ensure_future(
 398                 provisioner.install_agent(
 399                     connection=connection,
 400                     nonce=params.nonce,
 401                     machine_id=machine_id,
 402                     api=self.api_proxy,
 403                 )
 404             )
 405
 406             machine = None
 407             for _ in range(10):
 408                 machine_list = await model.get_machines()
 409                 if machine_id in machine_list:
 410                     self.log.debug("Machine {} found in model!".format(machine_id))
 411                     machine = model.machines.get(machine_id)
 412                     break
 413                 await asyncio.sleep(2)
 414
 415             if machine is None:
 416                 msg = "Machine {} not found in model".format(machine_id)
 417                 self.log.error(msg=msg)
 418                 raise JujuMachineNotFound(msg)
 419
 420             self.log.debug(
 421                 "Wait until machine {} is ready in model {}".format(
 422                     machine.entity_id, model_name
 423                 )
 424             )
 425             await JujuModelWatcher.wait_for(
 426                 model=model,
 427                 entity=machine,
 428                 progress_timeout=progress_timeout,
 429                 total_timeout=total_timeout,
 430                 db_dict=db_dict,
 431                 n2vc=self.n2vc,
 432             )
 433         except Exception as e:
 434             raise e
 435         finally:
 436             await self.disconnect_model(model)
 437             await self.disconnect_controller(controller)
 438
 439         self.log.debug(
 440             "Machine provisioned {} in model {}".format(machine_id, model_name)
 441         )
 442
 443         return machine_id
 444
 445     async def deploy_charm(
 446         self,
 447         application_name: str,
 448         path: str,
 449         model_name: str,
 450         machine_id: str,
 451         db_dict: dict = None,
 452         progress_timeout: float = None,
 453         total_timeout: float = None,
 454         config: dict = None,
 455         series: str = None,
 456     ):
 457         """Deploy charm
 458
 459         :param: application_name:   Application name
 460         :param: path:               Local path to the charm
 461         :param: model_name:         Model name
 462         :param: machine_id          ID of the machine
 463         :param: db_dict:            Dictionary with data of the DB to write the updates
 464         :param: progress_timeout:   Maximum time between two updates in the model
 465         :param: total_timeout:      Timeout for the entity to be active
 466         :param: config:             Config for the charm
 467         :param: series:             Series of the charm
 468
 469         :return: (juju.application.Application): Juju application
 470         """
 471         self.log.debug(
 472             "Deploying charm {} to machine {} in model ~{}".format(
 473                 application_name, machine_id, model_name
 474             )
 475         )
 476         self.log.debug("charm: {}".format(path))
 477
 478         # Get controller
 479         controller = await self.get_controller()
 480
 481         # Get model
 482         model = await self.get_model(controller, model_name)
 483
 484         try:
 485             application = None
 486             if application_name not in model.applications:
 487
 488                 if machine_id is not None:
 489                     if machine_id not in model.machines:
 490                         msg = "Machine {} not found in model".format(machine_id)
 491                         self.log.error(msg=msg)
 492                         raise JujuMachineNotFound(msg)
 493                     machine = model.machines[machine_id]
 494                     series = machine.series
 495
 496                 application = await model.deploy(
 497                     entity_url=path,
 498                     application_name=application_name,
 499                     channel="stable",
 500                     num_units=1,
 501                     series=series,
 502                     to=machine_id,
 503                     config=config,
 504                 )
 505
 506                 self.log.debug(
 507                     "Wait until application {} is ready in model {}".format(
 508                         application_name, model_name
 509                     )
 510                 )
 511                 await JujuModelWatcher.wait_for(
 512                     model=model,
 513                     entity=application,
 514                     progress_timeout=progress_timeout,
 515                     total_timeout=total_timeout,
 516                     db_dict=db_dict,
 517                     n2vc=self.n2vc,
 518                 )
 519                 self.log.debug(
 520                     "Application {} is ready in model {}".format(
 521                         application_name, model_name
 522                     )
 523                 )
 524             else:
 525                 raise JujuApplicationExists(
 526                     "Application {} exists".format(application_name)
 527                 )
 528         finally:
 529             await self.disconnect_model(model)
 530             await self.disconnect_controller(controller)
 531
 532         return application
 533
 534     def _get_application(self, model: Model, application_name: str) -> Application:
 535         """Get application
 536
 537         :param: model:              Model object
 538         :param: application_name:   Application name
 539
 540         :return: juju.application.Application (or None if it doesn't exist)
 541         """
 542         if model.applications and application_name in model.applications:
 543             return model.applications[application_name]
 544
 545     async def execute_action(
 546         self,
 547         application_name: str,
 548         model_name: str,
 549         action_name: str,
 550         db_dict: dict = None,
 551         progress_timeout: float = None,
 552         total_timeout: float = None,
 553         **kwargs
 554     ):
 555         """Execute action
 556
 557         :param: application_name:   Application name
 558         :param: model_name:         Model name
 559         :param: cloud_name:         Cloud name
 560         :param: action_name:        Name of the action
 561         :param: db_dict:            Dictionary with data of the DB to write the updates
 562         :param: progress_timeout:   Maximum time between two updates in the model
 563         :param: total_timeout:      Timeout for the entity to be active
 564
 565         :return: (str, str): (output and status)
 566         """
 567         self.log.debug(
 568             "Executing action {} using params {}".format(action_name, kwargs)
 569         )
 570         # Get controller
 571         controller = await self.get_controller()
 572
 573         # Get model
 574         model = await self.get_model(controller, model_name)
 575
 576         try:
 577             # Get application
 578             application = self._get_application(
 579                 model, application_name=application_name,
 580             )
 581             if application is None:
 582                 raise JujuApplicationNotFound("Cannot execute action")
 583
 584             # Get unit
 585             unit = None
 586             for u in application.units:
 587                 if await u.is_leader_from_status():
 588                     unit = u
 589             if unit is None:
 590                 raise Exception("Cannot execute action: leader unit not found")
 591
 592             actions = await application.get_actions()
 593
 594             if action_name not in actions:
 595                 raise Exception(
 596                     "Action {} not in available actions".format(action_name)
 597                 )
 598
 599             action = await unit.run_action(action_name, **kwargs)
 600
 601             self.log.debug(
 602                 "Wait until action {} is completed in application {} (model={})".format(
 603                     action_name, application_name, model_name
 604                 )
 605             )
 606             await JujuModelWatcher.wait_for(
 607                 model=model,
 608                 entity=action,
 609                 progress_timeout=progress_timeout,
 610                 total_timeout=total_timeout,
 611                 db_dict=db_dict,
 612                 n2vc=self.n2vc,
 613             )
 614
 615             output = await model.get_action_output(action_uuid=action.entity_id)
 616             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
 617             status = (
 618                 status[action.entity_id] if action.entity_id in status else "failed"
 619             )
 620
 621             self.log.debug(
 622                 "Action {} completed with status {} in application {} (model={})".format(
 623                     action_name, action.status, application_name, model_name
 624                 )
 625             )
 626         except Exception as e:
 627             raise e
 628         finally:
 629             await self.disconnect_model(model)
 630             await self.disconnect_controller(controller)
 631
 632         return output, status
 633
 634     async def get_actions(self, application_name: str, model_name: str) -> dict:
 635         """Get list of actions
 636
 637         :param: application_name: Application name
 638         :param: model_name: Model name
 639
 640         :return: Dict with this format
 641             {
 642                 "action_name": "Description of the action",
 643                 ...
 644             }
 645         """
 646         self.log.debug(
 647             "Getting list of actions for application {}".format(application_name)
 648         )
 649
 650         # Get controller
 651         controller = await self.get_controller()
 652
 653         # Get model
 654         model = await self.get_model(controller, model_name)
 655
 656         try:
 657             # Get application
 658             application = self._get_application(
 659                 model, application_name=application_name,
 660             )
 661
 662             # Return list of actions
 663             return await application.get_actions()
 664
 665         finally:
 666             # Disconnect from model and controller
 667             await self.disconnect_model(model)
 668             await self.disconnect_controller(controller)
 669
 670     async def add_relation(
 671         self,
 672         model_name: str,
 673         application_name_1: str,
 674         application_name_2: str,
 675         relation_1: str,
 676         relation_2: str,
 677     ):
 678         """Add relation
 679
 680         :param: model_name:             Model name
 681         :param: application_name_1      First application name
 682         :param: application_name_2:     Second application name
 683         :param: relation_1:             First relation name
 684         :param: relation_2:             Second relation name
 685         """
 686
 687         self.log.debug("Adding relation: {} -> {}".format(relation_1, relation_2))
 688
 689         # Get controller
 690         controller = await self.get_controller()
 691
 692         # Get model
 693         model = await self.get_model(controller, model_name)
 694
 695         # Build relation strings
 696         r1 = "{}:{}".format(application_name_1, relation_1)
 697         r2 = "{}:{}".format(application_name_2, relation_2)
 698
 699         # Add relation
 700         try:
 701             await model.add_relation(relation1=r1, relation2=r2)
 702         except JujuAPIError as e:
 703             if "not found" in e.message:
 704                 self.log.warning("Relation not found: {}".format(e.message))
 705                 return
 706             if "already exists" in e.message:
 707                 self.log.warning("Relation already exists: {}".format(e.message))
 708                 return
 709             # another exception, raise it
 710             raise e
 711         finally:
 712             await self.disconnect_model(model)
 713             await self.disconnect_controller(controller)
 714
 715     async def destroy_model(
 716         self, model_name: str, total_timeout: float,
 717     ):
 718         """
 719         Destroy model
 720
 721         :param: model_name:     Model name
 722         :param: total_timeout:  Timeout
 723         """
 724
 725         controller = await self.get_controller()
 726         model = await self.get_model(controller, model_name)
 727         try:
 728             self.log.debug("Destroying model {}".format(model_name))
 729             uuid = model.info.uuid
 730
 731             # Destroy applications
 732             for application_name in model.applications:
 733                 try:
 734                     await self.destroy_application(
 735                         model, application_name=application_name,
 736                     )
 737                 except Exception as e:
 738                     self.log.error(
 739                         "Error destroying application {} in model {}: {}".format(
 740                             application_name, model_name, e
 741                         )
 742                     )
 743
 744             # Destroy machines
 745             machines = await model.get_machines()
 746             for machine_id in machines:
 747                 try:
 748                     await self.destroy_machine(
 749                         model, machine_id=machine_id, total_timeout=total_timeout,
 750                     )
 751                 except asyncio.CancelledError:
 752                     raise
 753                 except Exception:
 754                     pass
 755
 756             # Disconnect model
 757             await self.disconnect_model(model)
 758
 759             # Destroy model
 760             if model_name in self.models:
 761                 self.models.remove(model_name)
 762
 763             await controller.destroy_model(uuid)
 764
 765             # Wait until model is destroyed
 766             self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
 767             last_exception = ""
 768
 769             if total_timeout is None:
 770                 total_timeout = 3600
 771             end = time.time() + total_timeout
 772             while time.time() < end:
 773                 try:
 774                     models = await controller.list_models()
 775                     if model_name not in models:
 776                         self.log.debug(
 777                             "The model {} ({}) was destroyed".format(model_name, uuid)
 778                         )
 779                         return
 780                 except asyncio.CancelledError:
 781                     raise
 782                 except Exception as e:
 783                     last_exception = e
 784                 await asyncio.sleep(5)
 785             raise Exception(
 786                 "Timeout waiting for model {} to be destroyed {}".format(
 787                     model_name, last_exception
 788                 )
 789             )
 790         finally:
 791             await self.disconnect_controller(controller)
 792
 793     async def destroy_application(self, model: Model, application_name: str):
 794         """
 795         Destroy application
 796
 797         :param: model:              Model object
 798         :param: application_name:   Application name
 799         """
 800         self.log.debug(
 801             "Destroying application {} in model {}".format(
 802                 application_name, model.info.name
 803             )
 804         )
 805         application = model.applications.get(application_name)
 806         if application:
 807             await application.destroy()
 808         else:
 809             self.log.warning("Application not found: {}".format(application_name))
 810
 811     async def destroy_machine(
 812         self, model: Model, machine_id: str, total_timeout: float = 3600
 813     ):
 814         """
 815         Destroy machine
 816
 817         :param: model:          Model object
 818         :param: machine_id:     Machine id
 819         :param: total_timeout:  Timeout in seconds
 820         """
 821         machines = await model.get_machines()
 822         if machine_id in machines:
 823             machine = model.machines[machine_id]
 824             # TODO: change this by machine.is_manual when this is upstreamed:
 825             # https://github.com/juju/python-libjuju/pull/396
 826             if "instance-id" in machine.safe_data and machine.safe_data[
 827                 "instance-id"
 828             ].startswith("manual:"):
 829                 await machine.destroy(force=True)
 830
 831                 # max timeout
 832                 end = time.time() + total_timeout
 833
 834                 # wait for machine removal
 835                 machines = await model.get_machines()
 836                 while machine_id in machines and time.time() < end:
 837                     self.log.debug(
 838                         "Waiting for machine {} is destroyed".format(machine_id)
 839                     )
 840                     await asyncio.sleep(0.5)
 841                     machines = await model.get_machines()
 842                 self.log.debug("Machine destroyed: {}".format(machine_id))
 843         else:
 844             self.log.debug("Machine not found: {}".format(machine_id))
 845
 846     async def configure_application(
 847         self, model_name: str, application_name: str, config: dict = None
 848     ):
 849         """Configure application
 850
 851         :param: model_name:         Model name
 852         :param: application_name:   Application name
 853         :param: config:             Config to apply to the charm
 854         """
 855         self.log.debug("Configuring application {}".format(application_name))
 856
 857         if config:
 858             try:
 859                 controller = await self.get_controller()
 860                 model = await self.get_model(controller, model_name)
 861                 application = self._get_application(
 862                     model, application_name=application_name,
 863                 )
 864                 await application.set_config(config)
 865             finally:
 866                 await self.disconnect_model(model)
 867                 await self.disconnect_controller(controller)
 868
 869     def _get_api_endpoints_db(self) -> [str]:
 870         """
 871         Get API Endpoints from DB
 872
 873         :return: List of API endpoints
 874         """
 875         self.log.debug("Getting endpoints from database")
 876
 877         juju_info = self.db.get_one(
 878             DB_DATA.api_endpoints.table,
 879             q_filter=DB_DATA.api_endpoints.filter,
 880             fail_on_empty=False,
 881         )
 882         if juju_info and DB_DATA.api_endpoints.key in juju_info:
 883             return juju_info[DB_DATA.api_endpoints.key]
 884
 885     def _update_api_endpoints_db(self, endpoints: [str]):
 886         """
 887         Update API endpoints in Database
 888
 889         :param: List of endpoints
 890         """
 891         self.log.debug("Saving endpoints {} in database".format(endpoints))
 892
 893         juju_info = self.db.get_one(
 894             DB_DATA.api_endpoints.table,
 895             q_filter=DB_DATA.api_endpoints.filter,
 896             fail_on_empty=False,
 897         )
 898         # If it doesn't, then create it
 899         if not juju_info:
 900             try:
 901                 self.db.create(
 902                     DB_DATA.api_endpoints.table, DB_DATA.api_endpoints.filter,
 903                 )
 904             except DbException as e:
 905                 # Racing condition: check if another N2VC worker has created it
 906                 juju_info = self.db.get_one(
 907                     DB_DATA.api_endpoints.table,
 908                     q_filter=DB_DATA.api_endpoints.filter,
 909                     fail_on_empty=False,
 910                 )
 911                 if not juju_info:
 912                     raise e
 913         self.db.set_one(
 914             DB_DATA.api_endpoints.table,
 915             DB_DATA.api_endpoints.filter,
 916             {DB_DATA.api_endpoints.key: endpoints},
 917         )
 918
 919     def handle_exception(self, loop, context):
 920         # All unhandled exceptions by libjuju are handled here.
 921         pass
 922
 923     async def health_check(self, interval: float = 300.0):
 924         """
 925         Health check to make sure controller and controller_model connections are OK
 926
 927         :param: interval: Time in seconds between checks
 928         """
 929         while True:
 930             try:
 931                 controller = await self.get_controller()
 932                 # self.log.debug("VCA is alive")
 933             except Exception as e:
 934                 self.log.error("Health check to VCA failed: {}".format(e))
 935             finally:
 936                 await self.disconnect_controller(controller)
 937             await asyncio.sleep(interval)