n2vc/libjuju.py

   1 # Copyright 2020 Canonical Ltd.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #     http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #     Unless required by applicable law or agreed to in writing, software
  10 #     distributed under the License is distributed on an "AS IS" BASIS,
  11 #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 #     See the License for the specific language governing permissions and
  13 #     limitations under the License.
  14
  15 import asyncio
  16 import logging
  17 import typing
  18
  19 import time
  20
  21 import juju.errors
  22 from juju.model import Model
  23 from juju.machine import Machine
  24 from juju.application import Application
  25 from juju.unit import Unit
  26 from juju.client._definitions import (
  27     FullStatus,
  28     QueryApplicationOffersResults,
  29     Cloud,
  30     CloudCredential,
  31 )
  32 from juju.controller import Controller
  33 from juju.client import client
  34 from juju import tag
  35
  36 from n2vc.definitions import Offer, RelationEndpoint
  37 from n2vc.juju_watcher import JujuModelWatcher
  38 from n2vc.provisioner import AsyncSSHProvisioner
  39 from n2vc.n2vc_conn import N2VCConnector
  40 from n2vc.exceptions import (
  41     JujuMachineNotFound,
  42     JujuApplicationNotFound,
  43     JujuLeaderUnitNotFound,
  44     JujuActionNotFound,
  45     JujuControllerFailedConnecting,
  46     JujuApplicationExists,
  47     JujuInvalidK8sConfiguration,
  48     JujuError,
  49 )
  50 from n2vc.vca.cloud import Cloud as VcaCloud
  51 from n2vc.vca.connection import Connection
  52 from kubernetes.client.configuration import Configuration
  53 from retrying_async import retry
  54
  55
  56 RBAC_LABEL_KEY_NAME = "rbac-id"
  57
  58
  59 class Libjuju:
  60     def __init__(
  61         self,
  62         vca_connection: Connection,
  63         loop: asyncio.AbstractEventLoop = None,
  64         log: logging.Logger = None,
  65         n2vc: N2VCConnector = None,
  66     ):
  67         """
  68         Constructor
  69
  70         :param: vca_connection:         n2vc.vca.connection object
  71         :param: loop:                   Asyncio loop
  72         :param: log:                    Logger
  73         :param: n2vc:                   N2VC object
  74         """
  75
  76         self.log = log or logging.getLogger("Libjuju")
  77         self.n2vc = n2vc
  78         self.vca_connection = vca_connection
  79
  80         self.loop = loop or asyncio.get_event_loop()
  81         self.loop.set_exception_handler(self.handle_exception)
  82         self.creating_model = asyncio.Lock(loop=self.loop)
  83
  84         if self.vca_connection.is_default:
  85             self.health_check_task = self._create_health_check_task()
  86
  87     def _create_health_check_task(self):
  88         return self.loop.create_task(self.health_check())
  89
  90     async def get_controller(self, timeout: float = 60.0) -> Controller:
  91         """
  92         Get controller
  93
  94         :param: timeout: Time in seconds to wait for controller to connect
  95         """
  96         controller = None
  97         try:
  98             controller = Controller()
  99             await asyncio.wait_for(
 100                 controller.connect(
 101                     endpoint=self.vca_connection.data.endpoints,
 102                     username=self.vca_connection.data.user,
 103                     password=self.vca_connection.data.secret,
 104                     cacert=self.vca_connection.data.cacert,
 105                 ),
 106                 timeout=timeout,
 107             )
 108             if self.vca_connection.is_default:
 109                 endpoints = await controller.api_endpoints
 110                 if not all(
 111                     endpoint in self.vca_connection.endpoints for endpoint in endpoints
 112                 ):
 113                     await self.vca_connection.update_endpoints(endpoints)
 114             return controller
 115         except asyncio.CancelledError as e:
 116             raise e
 117         except Exception as e:
 118             self.log.error(
 119                 "Failed connecting to controller: {}... {}".format(
 120                     self.vca_connection.data.endpoints, e
 121                 )
 122             )
 123             if controller:
 124                 await self.disconnect_controller(controller)
 125
 126             raise JujuControllerFailedConnecting(
 127                 f"Error connecting to Juju controller: {e}"
 128             )
 129
 130     async def disconnect(self):
 131         """Disconnect"""
 132         # Cancel health check task
 133         self.health_check_task.cancel()
 134         self.log.debug("Libjuju disconnected!")
 135
 136     async def disconnect_model(self, model: Model):
 137         """
 138         Disconnect model
 139
 140         :param: model: Model that will be disconnected
 141         """
 142         await model.disconnect()
 143
 144     async def disconnect_controller(self, controller: Controller):
 145         """
 146         Disconnect controller
 147
 148         :param: controller: Controller that will be disconnected
 149         """
 150         if controller:
 151             await controller.disconnect()
 152
 153     @retry(attempts=3, delay=5, timeout=None)
 154     async def add_model(self, model_name: str, cloud: VcaCloud):
 155         """
 156         Create model
 157
 158         :param: model_name: Model name
 159         :param: cloud: Cloud object
 160         """
 161
 162         # Get controller
 163         controller = await self.get_controller()
 164         model = None
 165         try:
 166             # Block until other workers have finished model creation
 167             while self.creating_model.locked():
 168                 await asyncio.sleep(0.1)
 169
 170             # Create the model
 171             async with self.creating_model:
 172                 if await self.model_exists(model_name, controller=controller):
 173                     return
 174                 self.log.debug("Creating model {}".format(model_name))
 175                 model = await controller.add_model(
 176                     model_name,
 177                     config=self.vca_connection.data.model_config,
 178                     cloud_name=cloud.name,
 179                     credential_name=cloud.credential_name,
 180                 )
 181         except juju.errors.JujuAPIError as e:
 182             if "already exists" in e.message:
 183                 pass
 184             else:
 185                 raise e
 186         finally:
 187             if model:
 188                 await self.disconnect_model(model)
 189             await self.disconnect_controller(controller)
 190
 191     async def get_executed_actions(self, model_name: str) -> list:
 192         """
 193         Get executed/history of actions for a model.
 194
 195         :param: model_name: Model name, str.
 196         :return: List of executed actions for a model.
 197         """
 198         model = None
 199         executed_actions = []
 200         controller = await self.get_controller()
 201         try:
 202             model = await self.get_model(controller, model_name)
 203             # Get all unique action names
 204             actions = {}
 205             for application in model.applications:
 206                 application_actions = await self.get_actions(application, model_name)
 207                 actions.update(application_actions)
 208             # Get status of all actions
 209             for application_action in actions:
 210                 app_action_status_list = await model.get_action_status(
 211                     name=application_action
 212                 )
 213                 for action_id, action_status in app_action_status_list.items():
 214                     executed_action = {
 215                         "id": action_id,
 216                         "action": application_action,
 217                         "status": action_status,
 218                     }
 219                     # Get action output by id
 220                     action_status = await model.get_action_output(executed_action["id"])
 221                     for k, v in action_status.items():
 222                         executed_action[k] = v
 223                     executed_actions.append(executed_action)
 224         except Exception as e:
 225             raise JujuError(
 226                 "Error in getting executed actions for model: {}. Error: {}".format(
 227                     model_name, str(e)
 228                 )
 229             )
 230         finally:
 231             if model:
 232                 await self.disconnect_model(model)
 233             await self.disconnect_controller(controller)
 234         return executed_actions
 235
 236     async def get_application_configs(
 237         self, model_name: str, application_name: str
 238     ) -> dict:
 239         """
 240         Get available configs for an application.
 241
 242         :param: model_name: Model name, str.
 243         :param: application_name: Application name, str.
 244
 245         :return: A dict which has key - action name, value - action description
 246         """
 247         model = None
 248         application_configs = {}
 249         controller = await self.get_controller()
 250         try:
 251             model = await self.get_model(controller, model_name)
 252             application = self._get_application(
 253                 model, application_name=application_name
 254             )
 255             application_configs = await application.get_config()
 256         except Exception as e:
 257             raise JujuError(
 258                 "Error in getting configs for application: {} in model: {}. Error: {}".format(
 259                     application_name, model_name, str(e)
 260                 )
 261             )
 262         finally:
 263             if model:
 264                 await self.disconnect_model(model)
 265             await self.disconnect_controller(controller)
 266         return application_configs
 267
 268     @retry(attempts=3, delay=5)
 269     async def get_model(self, controller: Controller, model_name: str) -> Model:
 270         """
 271         Get model from controller
 272
 273         :param: controller: Controller
 274         :param: model_name: Model name
 275
 276         :return: Model: The created Juju model object
 277         """
 278         return await controller.get_model(model_name)
 279
 280     async def model_exists(
 281         self, model_name: str, controller: Controller = None
 282     ) -> bool:
 283         """
 284         Check if model exists
 285
 286         :param: controller: Controller
 287         :param: model_name: Model name
 288
 289         :return bool
 290         """
 291         need_to_disconnect = False
 292
 293         # Get controller if not passed
 294         if not controller:
 295             controller = await self.get_controller()
 296             need_to_disconnect = True
 297
 298         # Check if model exists
 299         try:
 300             return model_name in await controller.list_models()
 301         finally:
 302             if need_to_disconnect:
 303                 await self.disconnect_controller(controller)
 304
 305     async def models_exist(self, model_names: [str]) -> (bool, list):
 306         """
 307         Check if models exists
 308
 309         :param: model_names: List of strings with model names
 310
 311         :return (bool, list[str]): (True if all models exists, List of model names that don't exist)
 312         """
 313         if not model_names:
 314             raise Exception(
 315                 "model_names must be a non-empty array. Given value: {}".format(
 316                     model_names
 317                 )
 318             )
 319         non_existing_models = []
 320         models = await self.list_models()
 321         existing_models = list(set(models).intersection(model_names))
 322         non_existing_models = list(set(model_names) - set(existing_models))
 323
 324         return (
 325             len(non_existing_models) == 0,
 326             non_existing_models,
 327         )
 328
 329     async def get_model_status(self, model_name: str) -> FullStatus:
 330         """
 331         Get model status
 332
 333         :param: model_name: Model name
 334
 335         :return: Full status object
 336         """
 337         controller = await self.get_controller()
 338         model = await self.get_model(controller, model_name)
 339         try:
 340             return await model.get_status()
 341         finally:
 342             await self.disconnect_model(model)
 343             await self.disconnect_controller(controller)
 344
 345     async def create_machine(
 346         self,
 347         model_name: str,
 348         machine_id: str = None,
 349         db_dict: dict = None,
 350         progress_timeout: float = None,
 351         total_timeout: float = None,
 352         series: str = "bionic",
 353         wait: bool = True,
 354     ) -> (Machine, bool):
 355         """
 356         Create machine
 357
 358         :param: model_name:         Model name
 359         :param: machine_id:         Machine id
 360         :param: db_dict:            Dictionary with data of the DB to write the updates
 361         :param: progress_timeout:   Maximum time between two updates in the model
 362         :param: total_timeout:      Timeout for the entity to be active
 363         :param: series:             Series of the machine (xenial, bionic, focal, ...)
 364         :param: wait:               Wait until machine is ready
 365
 366         :return: (juju.machine.Machine, bool):  Machine object and a boolean saying
 367                                                 if the machine is new or it already existed
 368         """
 369         new = False
 370         machine = None
 371
 372         self.log.debug(
 373             "Creating machine (id={}) in model: {}".format(machine_id, model_name)
 374         )
 375
 376         # Get controller
 377         controller = await self.get_controller()
 378
 379         # Get model
 380         model = await self.get_model(controller, model_name)
 381         try:
 382             if machine_id is not None:
 383                 self.log.debug(
 384                     "Searching machine (id={}) in model {}".format(
 385                         machine_id, model_name
 386                     )
 387                 )
 388
 389                 # Get machines from model and get the machine with machine_id if exists
 390                 machines = await model.get_machines()
 391                 if machine_id in machines:
 392                     self.log.debug(
 393                         "Machine (id={}) found in model {}".format(
 394                             machine_id, model_name
 395                         )
 396                     )
 397                     machine = machines[machine_id]
 398                 else:
 399                     raise JujuMachineNotFound("Machine {} not found".format(machine_id))
 400
 401             if machine is None:
 402                 self.log.debug("Creating a new machine in model {}".format(model_name))
 403
 404                 # Create machine
 405                 machine = await model.add_machine(
 406                     spec=None, constraints=None, disks=None, series=series
 407                 )
 408                 new = True
 409
 410                 # Wait until the machine is ready
 411                 self.log.debug(
 412                     "Wait until machine {} is ready in model {}".format(
 413                         machine.entity_id, model_name
 414                     )
 415                 )
 416                 if wait:
 417                     await JujuModelWatcher.wait_for(
 418                         model=model,
 419                         entity=machine,
 420                         progress_timeout=progress_timeout,
 421                         total_timeout=total_timeout,
 422                         db_dict=db_dict,
 423                         n2vc=self.n2vc,
 424                         vca_id=self.vca_connection._vca_id,
 425                     )
 426         finally:
 427             await self.disconnect_model(model)
 428             await self.disconnect_controller(controller)
 429
 430         self.log.debug(
 431             "Machine {} ready at {} in model {}".format(
 432                 machine.entity_id, machine.dns_name, model_name
 433             )
 434         )
 435         return machine, new
 436
 437     async def provision_machine(
 438         self,
 439         model_name: str,
 440         hostname: str,
 441         username: str,
 442         private_key_path: str,
 443         db_dict: dict = None,
 444         progress_timeout: float = None,
 445         total_timeout: float = None,
 446     ) -> str:
 447         """
 448         Manually provisioning of a machine
 449
 450         :param: model_name:         Model name
 451         :param: hostname:           IP to access the machine
 452         :param: username:           Username to login to the machine
 453         :param: private_key_path:   Local path for the private key
 454         :param: db_dict:            Dictionary with data of the DB to write the updates
 455         :param: progress_timeout:   Maximum time between two updates in the model
 456         :param: total_timeout:      Timeout for the entity to be active
 457
 458         :return: (Entity): Machine id
 459         """
 460         self.log.debug(
 461             "Provisioning machine. model: {}, hostname: {}, username: {}".format(
 462                 model_name, hostname, username
 463             )
 464         )
 465
 466         # Get controller
 467         controller = await self.get_controller()
 468
 469         # Get model
 470         model = await self.get_model(controller, model_name)
 471
 472         try:
 473             # Get provisioner
 474             provisioner = AsyncSSHProvisioner(
 475                 host=hostname,
 476                 user=username,
 477                 private_key_path=private_key_path,
 478                 log=self.log,
 479             )
 480
 481             # Provision machine
 482             params = await provisioner.provision_machine()
 483
 484             params.jobs = ["JobHostUnits"]
 485
 486             self.log.debug("Adding machine to model")
 487             connection = model.connection()
 488             client_facade = client.ClientFacade.from_connection(connection)
 489
 490             results = await client_facade.AddMachines(params=[params])
 491             error = results.machines[0].error
 492
 493             if error:
 494                 msg = "Error adding machine: {}".format(error.message)
 495                 self.log.error(msg=msg)
 496                 raise ValueError(msg)
 497
 498             machine_id = results.machines[0].machine
 499
 500             self.log.debug("Installing Juju agent into machine {}".format(machine_id))
 501             asyncio.ensure_future(
 502                 provisioner.install_agent(
 503                     connection=connection,
 504                     nonce=params.nonce,
 505                     machine_id=machine_id,
 506                     proxy=self.vca_connection.data.api_proxy,
 507                     series=params.series,
 508                 )
 509             )
 510
 511             machine = None
 512             for _ in range(10):
 513                 machine_list = await model.get_machines()
 514                 if machine_id in machine_list:
 515                     self.log.debug("Machine {} found in model!".format(machine_id))
 516                     machine = model.machines.get(machine_id)
 517                     break
 518                 await asyncio.sleep(2)
 519
 520             if machine is None:
 521                 msg = "Machine {} not found in model".format(machine_id)
 522                 self.log.error(msg=msg)
 523                 raise JujuMachineNotFound(msg)
 524
 525             self.log.debug(
 526                 "Wait until machine {} is ready in model {}".format(
 527                     machine.entity_id, model_name
 528                 )
 529             )
 530             await JujuModelWatcher.wait_for(
 531                 model=model,
 532                 entity=machine,
 533                 progress_timeout=progress_timeout,
 534                 total_timeout=total_timeout,
 535                 db_dict=db_dict,
 536                 n2vc=self.n2vc,
 537                 vca_id=self.vca_connection._vca_id,
 538             )
 539         except Exception as e:
 540             raise e
 541         finally:
 542             await self.disconnect_model(model)
 543             await self.disconnect_controller(controller)
 544
 545         self.log.debug(
 546             "Machine provisioned {} in model {}".format(machine_id, model_name)
 547         )
 548
 549         return machine_id
 550
 551     async def deploy(
 552         self, uri: str, model_name: str, wait: bool = True, timeout: float = 3600
 553     ):
 554         """
 555         Deploy bundle or charm: Similar to the juju CLI command `juju deploy`
 556
 557         :param: uri:            Path or Charm Store uri in which the charm or bundle can be found
 558         :param: model_name:     Model name
 559         :param: wait:           Indicates whether to wait or not until all applications are active
 560         :param: timeout:        Time in seconds to wait until all applications are active
 561         """
 562         controller = await self.get_controller()
 563         model = await self.get_model(controller, model_name)
 564         try:
 565             await model.deploy(uri, trust=True)
 566             if wait:
 567                 await JujuModelWatcher.wait_for_model(model, timeout=timeout)
 568                 self.log.debug("All units active in model {}".format(model_name))
 569         finally:
 570             await self.disconnect_model(model)
 571             await self.disconnect_controller(controller)
 572
 573     async def add_unit(
 574         self,
 575         application_name: str,
 576         model_name: str,
 577         machine_id: str,
 578         db_dict: dict = None,
 579         progress_timeout: float = None,
 580         total_timeout: float = None,
 581     ):
 582         """Add unit
 583
 584         :param: application_name:   Application name
 585         :param: model_name:         Model name
 586         :param: machine_id          Machine id
 587         :param: db_dict:            Dictionary with data of the DB to write the updates
 588         :param: progress_timeout:   Maximum time between two updates in the model
 589         :param: total_timeout:      Timeout for the entity to be active
 590
 591         :return: None
 592         """
 593
 594         model = None
 595         controller = await self.get_controller()
 596         try:
 597             model = await self.get_model(controller, model_name)
 598             application = self._get_application(model, application_name)
 599
 600             if application is not None:
 601                 # Checks if the given machine id in the model,
 602                 # otherwise function raises an error
 603                 _machine, _series = self._get_machine_info(model, machine_id)
 604
 605                 self.log.debug(
 606                     "Adding unit (machine {}) to application {} in model ~{}".format(
 607                         machine_id, application_name, model_name
 608                     )
 609                 )
 610
 611                 await application.add_unit(to=machine_id)
 612
 613                 await JujuModelWatcher.wait_for(
 614                     model=model,
 615                     entity=application,
 616                     progress_timeout=progress_timeout,
 617                     total_timeout=total_timeout,
 618                     db_dict=db_dict,
 619                     n2vc=self.n2vc,
 620                     vca_id=self.vca_connection._vca_id,
 621                 )
 622                 self.log.debug(
 623                     "Unit is added to application {} in model {}".format(
 624                         application_name, model_name
 625                     )
 626                 )
 627             else:
 628                 raise JujuApplicationNotFound(
 629                     "Application {} not exists".format(application_name)
 630                 )
 631         finally:
 632             if model:
 633                 await self.disconnect_model(model)
 634             await self.disconnect_controller(controller)
 635
 636     async def destroy_unit(
 637         self,
 638         application_name: str,
 639         model_name: str,
 640         machine_id: str,
 641         total_timeout: float = None,
 642     ):
 643         """Destroy unit
 644
 645         :param: application_name:   Application name
 646         :param: model_name:         Model name
 647         :param: machine_id          Machine id
 648         :param: total_timeout:      Timeout for the entity to be active
 649
 650         :return: None
 651         """
 652
 653         model = None
 654         controller = await self.get_controller()
 655         try:
 656             model = await self.get_model(controller, model_name)
 657             application = self._get_application(model, application_name)
 658
 659             if application is None:
 660                 raise JujuApplicationNotFound(
 661                     "Application not found: {} (model={})".format(
 662                         application_name, model_name
 663                     )
 664                 )
 665
 666             unit = self._get_unit(application, machine_id)
 667             if not unit:
 668                 raise JujuError(
 669                     "A unit with machine id {} not in available units".format(
 670                         machine_id
 671                     )
 672                 )
 673
 674             unit_name = unit.name
 675
 676             self.log.debug(
 677                 "Destroying unit {} from application {} in model {}".format(
 678                     unit_name, application_name, model_name
 679                 )
 680             )
 681             await application.destroy_unit(unit_name)
 682
 683             self.log.debug(
 684                 "Waiting for unit {} to be destroyed in application {} (model={})...".format(
 685                     unit_name, application_name, model_name
 686                 )
 687             )
 688
 689             # TODO: Add functionality in the Juju watcher to replace this kind of blocks
 690             if total_timeout is None:
 691                 total_timeout = 3600
 692             end = time.time() + total_timeout
 693             while time.time() < end:
 694                 if not self._get_unit(application, machine_id):
 695                     self.log.debug(
 696                         "The unit {} was destroyed in application {} (model={}) ".format(
 697                             unit_name, application_name, model_name
 698                         )
 699                     )
 700                     return
 701                 await asyncio.sleep(5)
 702             self.log.debug(
 703                 "Unit {} is destroyed from application {} in model {}".format(
 704                     unit_name, application_name, model_name
 705                 )
 706             )
 707         finally:
 708             if model:
 709                 await self.disconnect_model(model)
 710             await self.disconnect_controller(controller)
 711
 712     async def deploy_charm(
 713         self,
 714         application_name: str,
 715         path: str,
 716         model_name: str,
 717         machine_id: str,
 718         db_dict: dict = None,
 719         progress_timeout: float = None,
 720         total_timeout: float = None,
 721         config: dict = None,
 722         series: str = None,
 723         num_units: int = 1,
 724     ):
 725         """Deploy charm
 726
 727         :param: application_name:   Application name
 728         :param: path:               Local path to the charm
 729         :param: model_name:         Model name
 730         :param: machine_id          ID of the machine
 731         :param: db_dict:            Dictionary with data of the DB to write the updates
 732         :param: progress_timeout:   Maximum time between two updates in the model
 733         :param: total_timeout:      Timeout for the entity to be active
 734         :param: config:             Config for the charm
 735         :param: series:             Series of the charm
 736         :param: num_units:          Number of units
 737
 738         :return: (juju.application.Application): Juju application
 739         """
 740         self.log.debug(
 741             "Deploying charm {} to machine {} in model ~{}".format(
 742                 application_name, machine_id, model_name
 743             )
 744         )
 745         self.log.debug("charm: {}".format(path))
 746
 747         # Get controller
 748         controller = await self.get_controller()
 749
 750         # Get model
 751         model = await self.get_model(controller, model_name)
 752
 753         try:
 754             if application_name not in model.applications:
 755                 if machine_id is not None:
 756                     machine, series = self._get_machine_info(model, machine_id)
 757
 758                 application = await model.deploy(
 759                     entity_url=path,
 760                     application_name=application_name,
 761                     channel="stable",
 762                     num_units=1,
 763                     series=series,
 764                     to=machine_id,
 765                     config=config,
 766                 )
 767
 768                 self.log.debug(
 769                     "Wait until application {} is ready in model {}".format(
 770                         application_name, model_name
 771                     )
 772                 )
 773                 if num_units > 1:
 774                     for _ in range(num_units - 1):
 775                         m, _ = await self.create_machine(model_name, wait=False)
 776                         await application.add_unit(to=m.entity_id)
 777
 778                 await JujuModelWatcher.wait_for(
 779                     model=model,
 780                     entity=application,
 781                     progress_timeout=progress_timeout,
 782                     total_timeout=total_timeout,
 783                     db_dict=db_dict,
 784                     n2vc=self.n2vc,
 785                     vca_id=self.vca_connection._vca_id,
 786                 )
 787                 self.log.debug(
 788                     "Application {} is ready in model {}".format(
 789                         application_name, model_name
 790                     )
 791                 )
 792             else:
 793                 raise JujuApplicationExists(
 794                     "Application {} exists".format(application_name)
 795                 )
 796         except juju.errors.JujuError as e:
 797             if "already exists" in e.message:
 798                 raise JujuApplicationExists(
 799                     "Application {} exists".format(application_name)
 800                 )
 801             else:
 802                 raise e
 803         finally:
 804             await self.disconnect_model(model)
 805             await self.disconnect_controller(controller)
 806
 807         return application
 808
 809     async def upgrade_charm(
 810         self,
 811         application_name: str,
 812         path: str,
 813         model_name: str,
 814         total_timeout: float = None,
 815         **kwargs,
 816     ):
 817         """Upgrade Charm
 818
 819         :param: application_name:   Application name
 820         :param: model_name:         Model name
 821         :param: path:               Local path to the charm
 822         :param: total_timeout:      Timeout for the entity to be active
 823
 824         :return: (str, str): (output and status)
 825         """
 826
 827         self.log.debug(
 828             "Upgrading charm {} in model {} from path {}".format(
 829                 application_name, model_name, path
 830             )
 831         )
 832
 833         await self.resolve_application(
 834             model_name=model_name, application_name=application_name
 835         )
 836
 837         # Get controller
 838         controller = await self.get_controller()
 839
 840         # Get model
 841         model = await self.get_model(controller, model_name)
 842
 843         try:
 844             # Get application
 845             application = self._get_application(
 846                 model,
 847                 application_name=application_name,
 848             )
 849             if application is None:
 850                 raise JujuApplicationNotFound(
 851                     "Cannot find application {} to upgrade".format(application_name)
 852                 )
 853
 854             await application.refresh(path=path)
 855
 856             self.log.debug(
 857                 "Wait until charm upgrade is completed for application {} (model={})".format(
 858                     application_name, model_name
 859                 )
 860             )
 861
 862             await JujuModelWatcher.ensure_units_idle(
 863                 model=model, application=application
 864             )
 865
 866             if application.status == "error":
 867                 error_message = "Unknown"
 868                 for unit in application.units:
 869                     if (
 870                         unit.workload_status == "error"
 871                         and unit.workload_status_message != ""
 872                     ):
 873                         error_message = unit.workload_status_message
 874
 875                 message = "Application {} failed update in {}: {}".format(
 876                     application_name, model_name, error_message
 877                 )
 878                 self.log.error(message)
 879                 raise JujuError(message=message)
 880
 881             self.log.debug(
 882                 "Application {} is ready in model {}".format(
 883                     application_name, model_name
 884                 )
 885             )
 886
 887         finally:
 888             await self.disconnect_model(model)
 889             await self.disconnect_controller(controller)
 890
 891         return application
 892
 893     async def resolve_application(self, model_name: str, application_name: str):
 894         controller = await self.get_controller()
 895         model = await self.get_model(controller, model_name)
 896
 897         try:
 898             application = self._get_application(
 899                 model,
 900                 application_name=application_name,
 901             )
 902             if application is None:
 903                 raise JujuApplicationNotFound(
 904                     "Cannot find application {} to resolve".format(application_name)
 905                 )
 906
 907             while application.status == "error":
 908                 for unit in application.units:
 909                     if unit.workload_status == "error":
 910                         self.log.debug(
 911                             "Model {}, Application {}, Unit {} in error state, resolving".format(
 912                                 model_name, application_name, unit.entity_id
 913                             )
 914                         )
 915                         try:
 916                             await unit.resolved(retry=False)
 917                         except Exception:
 918                             pass
 919
 920                 await asyncio.sleep(1)
 921
 922         finally:
 923             await self.disconnect_model(model)
 924             await self.disconnect_controller(controller)
 925
 926     async def resolve(self, model_name: str):
 927         controller = await self.get_controller()
 928         model = await self.get_model(controller, model_name)
 929         all_units_active = False
 930         try:
 931             while not all_units_active:
 932                 all_units_active = True
 933                 for application_name, application in model.applications.items():
 934                     if application.status == "error":
 935                         for unit in application.units:
 936                             if unit.workload_status == "error":
 937                                 self.log.debug(
 938                                     "Model {}, Application {}, Unit {} in error state, resolving".format(
 939                                         model_name, application_name, unit.entity_id
 940                                     )
 941                                 )
 942                                 try:
 943                                     await unit.resolved(retry=False)
 944                                     all_units_active = False
 945                                 except Exception:
 946                                     pass
 947
 948                 if not all_units_active:
 949                     await asyncio.sleep(5)
 950         finally:
 951             await self.disconnect_model(model)
 952             await self.disconnect_controller(controller)
 953
 954     async def scale_application(
 955         self,
 956         model_name: str,
 957         application_name: str,
 958         scale: int = 1,
 959         total_timeout: float = None,
 960     ):
 961         """
 962         Scale application (K8s)
 963
 964         :param: model_name:         Model name
 965         :param: application_name:   Application name
 966         :param: scale:              Scale to which to set this application
 967         :param: total_timeout:      Timeout for the entity to be active
 968         """
 969
 970         model = None
 971         controller = await self.get_controller()
 972         try:
 973             model = await self.get_model(controller, model_name)
 974
 975             self.log.debug(
 976                 "Scaling application {} in model {}".format(
 977                     application_name, model_name
 978                 )
 979             )
 980             application = self._get_application(model, application_name)
 981             if application is None:
 982                 raise JujuApplicationNotFound("Cannot scale application")
 983             await application.scale(scale=scale)
 984             # Wait until application is scaled in model
 985             self.log.debug(
 986                 "Waiting for application {} to be scaled in model {}...".format(
 987                     application_name, model_name
 988                 )
 989             )
 990             if total_timeout is None:
 991                 total_timeout = 1800
 992             end = time.time() + total_timeout
 993             while time.time() < end:
 994                 application_scale = self._get_application_count(model, application_name)
 995                 # Before calling wait_for_model function,
 996                 # wait until application unit count and scale count are equal.
 997                 # Because there is a delay before scaling triggers in Juju model.
 998                 if application_scale == scale:
 999                     await JujuModelWatcher.wait_for_model(
1000                         model=model, timeout=total_timeout
1001                     )
1002                     self.log.debug(
1003                         "Application {} is scaled in model {}".format(
1004                             application_name, model_name
1005                         )
1006                     )
1007                     return
1008                 await asyncio.sleep(5)
1009             raise Exception(
1010                 "Timeout waiting for application {} in model {} to be scaled".format(
1011                     application_name, model_name
1012                 )
1013             )
1014         finally:
1015             if model:
1016                 await self.disconnect_model(model)
1017             await self.disconnect_controller(controller)
1018
1019     def _get_application_count(self, model: Model, application_name: str) -> int:
1020         """Get number of units of the application
1021
1022         :param: model:              Model object
1023         :param: application_name:   Application name
1024
1025         :return: int (or None if application doesn't exist)
1026         """
1027         application = self._get_application(model, application_name)
1028         if application is not None:
1029             return len(application.units)
1030
1031     def _get_application(self, model: Model, application_name: str) -> Application:
1032         """Get application
1033
1034         :param: model:              Model object
1035         :param: application_name:   Application name
1036
1037         :return: juju.application.Application (or None if it doesn't exist)
1038         """
1039         if model.applications and application_name in model.applications:
1040             return model.applications[application_name]
1041
1042     def _get_unit(self, application: Application, machine_id: str) -> Unit:
1043         """Get unit
1044
1045         :param: application:        Application object
1046         :param: machine_id:         Machine id
1047
1048         :return: Unit
1049         """
1050         unit = None
1051         for u in application.units:
1052             if u.machine_id == machine_id:
1053                 unit = u
1054                 break
1055         return unit
1056
1057     def _get_machine_info(
1058         self,
1059         model,
1060         machine_id: str,
1061     ) -> (str, str):
1062         """Get machine info
1063
1064         :param: model:          Model object
1065         :param: machine_id:     Machine id
1066
1067         :return: (str, str): (machine, series)
1068         """
1069         if machine_id not in model.machines:
1070             msg = "Machine {} not found in model".format(machine_id)
1071             self.log.error(msg=msg)
1072             raise JujuMachineNotFound(msg)
1073         machine = model.machines[machine_id]
1074         return machine, machine.series
1075
1076     async def execute_action(
1077         self,
1078         application_name: str,
1079         model_name: str,
1080         action_name: str,
1081         db_dict: dict = None,
1082         machine_id: str = None,
1083         progress_timeout: float = None,
1084         total_timeout: float = None,
1085         **kwargs,
1086     ):
1087         """Execute action
1088
1089         :param: application_name:   Application name
1090         :param: model_name:         Model name
1091         :param: action_name:        Name of the action
1092         :param: db_dict:            Dictionary with data of the DB to write the updates
1093         :param: machine_id          Machine id
1094         :param: progress_timeout:   Maximum time between two updates in the model
1095         :param: total_timeout:      Timeout for the entity to be active
1096
1097         :return: (str, str): (output and status)
1098         """
1099         self.log.debug(
1100             "Executing action {} using params {}".format(action_name, kwargs)
1101         )
1102         # Get controller
1103         controller = await self.get_controller()
1104
1105         # Get model
1106         model = await self.get_model(controller, model_name)
1107
1108         try:
1109             # Get application
1110             application = self._get_application(
1111                 model,
1112                 application_name=application_name,
1113             )
1114             if application is None:
1115                 raise JujuApplicationNotFound("Cannot execute action")
1116             # Racing condition:
1117             #   Ocassionally, self._get_leader_unit() will return None
1118             #   because the leader elected hook has not been triggered yet.
1119             #   Therefore, we are doing some retries. If it happens again,
1120             #   re-open bug 1236
1121             if machine_id is None:
1122                 unit = await self._get_leader_unit(application)
1123                 self.log.debug(
1124                     "Action {} is being executed on the leader unit {}".format(
1125                         action_name, unit.name
1126                     )
1127                 )
1128             else:
1129                 unit = self._get_unit(application, machine_id)
1130                 if not unit:
1131                     raise JujuError(
1132                         "A unit with machine id {} not in available units".format(
1133                             machine_id
1134                         )
1135                     )
1136                 self.log.debug(
1137                     "Action {} is being executed on {} unit".format(
1138                         action_name, unit.name
1139                     )
1140                 )
1141
1142             actions = await application.get_actions()
1143
1144             if action_name not in actions:
1145                 raise JujuActionNotFound(
1146                     "Action {} not in available actions".format(action_name)
1147                 )
1148
1149             action = await unit.run_action(action_name, **kwargs)
1150
1151             self.log.debug(
1152                 "Wait until action {} is completed in application {} (model={})".format(
1153                     action_name, application_name, model_name
1154                 )
1155             )
1156             await JujuModelWatcher.wait_for(
1157                 model=model,
1158                 entity=action,
1159                 progress_timeout=progress_timeout,
1160                 total_timeout=total_timeout,
1161                 db_dict=db_dict,
1162                 n2vc=self.n2vc,
1163                 vca_id=self.vca_connection._vca_id,
1164             )
1165
1166             output = await model.get_action_output(action_uuid=action.entity_id)
1167             status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1168             status = (
1169                 status[action.entity_id] if action.entity_id in status else "failed"
1170             )
1171
1172             self.log.debug(
1173                 "Action {} completed with status {} in application {} (model={})".format(
1174                     action_name, action.status, application_name, model_name
1175                 )
1176             )
1177         finally:
1178             await self.disconnect_model(model)
1179             await self.disconnect_controller(controller)
1180
1181         return output, status
1182
1183     async def get_actions(self, application_name: str, model_name: str) -> dict:
1184         """Get list of actions
1185
1186         :param: application_name: Application name
1187         :param: model_name: Model name
1188
1189         :return: Dict with this format
1190             {
1191                 "action_name": "Description of the action",
1192                 ...
1193             }
1194         """
1195         self.log.debug(
1196             "Getting list of actions for application {}".format(application_name)
1197         )
1198
1199         # Get controller
1200         controller = await self.get_controller()
1201
1202         # Get model
1203         model = await self.get_model(controller, model_name)
1204
1205         try:
1206             # Get application
1207             application = self._get_application(
1208                 model,
1209                 application_name=application_name,
1210             )
1211
1212             # Return list of actions
1213             return await application.get_actions()
1214
1215         finally:
1216             # Disconnect from model and controller
1217             await self.disconnect_model(model)
1218             await self.disconnect_controller(controller)
1219
1220     async def get_metrics(self, model_name: str, application_name: str) -> dict:
1221         """Get the metrics collected by the VCA.
1222
1223         :param model_name The name or unique id of the network service
1224         :param application_name The name of the application
1225         """
1226         if not model_name or not application_name:
1227             raise Exception("model_name and application_name must be non-empty strings")
1228         metrics = {}
1229         controller = await self.get_controller()
1230         model = await self.get_model(controller, model_name)
1231         try:
1232             application = self._get_application(model, application_name)
1233             if application is not None:
1234                 metrics = await application.get_metrics()
1235         finally:
1236             self.disconnect_model(model)
1237             self.disconnect_controller(controller)
1238         return metrics
1239
1240     async def add_relation(
1241         self,
1242         model_name: str,
1243         endpoint_1: str,
1244         endpoint_2: str,
1245     ):
1246         """Add relation
1247
1248         :param: model_name:     Model name
1249         :param: endpoint_1      First endpoint name
1250                                 ("app:endpoint" format or directly the saas name)
1251         :param: endpoint_2:     Second endpoint name (^ same format)
1252         """
1253
1254         self.log.debug("Adding relation: {} -> {}".format(endpoint_1, endpoint_2))
1255
1256         # Get controller
1257         controller = await self.get_controller()
1258
1259         # Get model
1260         model = await self.get_model(controller, model_name)
1261
1262         # Add relation
1263         try:
1264             await model.add_relation(endpoint_1, endpoint_2)
1265         except juju.errors.JujuAPIError as e:
1266             if self._relation_is_not_found(e):
1267                 self.log.warning("Relation not found: {}".format(e.message))
1268                 return
1269             if self._relation_already_exist(e):
1270                 self.log.warning("Relation already exists: {}".format(e.message))
1271                 return
1272             # another exception, raise it
1273             raise e
1274         finally:
1275             await self.disconnect_model(model)
1276             await self.disconnect_controller(controller)
1277
1278     def _relation_is_not_found(self, juju_error):
1279         text = "not found"
1280         return (text in juju_error.message) or (
1281             juju_error.error_code and text in juju_error.error_code
1282         )
1283
1284     def _relation_already_exist(self, juju_error):
1285         text = "already exists"
1286         return (text in juju_error.message) or (
1287             juju_error.error_code and text in juju_error.error_code
1288         )
1289
1290     async def offer(self, endpoint: RelationEndpoint) -> Offer:
1291         """
1292         Create an offer from a RelationEndpoint
1293
1294         :param: endpoint: Relation endpoint
1295
1296         :return: Offer object
1297         """
1298         model_name = endpoint.model_name
1299         offer_name = f"{endpoint.application_name}-{endpoint.endpoint_name}"
1300         controller = await self.get_controller()
1301         model = None
1302         try:
1303             model = await self.get_model(controller, model_name)
1304             await model.create_offer(endpoint.endpoint, offer_name=offer_name)
1305             offer_list = await self._list_offers(model_name, offer_name=offer_name)
1306             if offer_list:
1307                 return Offer(offer_list[0].offer_url)
1308             else:
1309                 raise Exception("offer was not created")
1310         except juju.errors.JujuError as e:
1311             if "application offer already exists" not in e.message:
1312                 raise e
1313         finally:
1314             if model:
1315                 self.disconnect_model(model)
1316             self.disconnect_controller(controller)
1317
1318     async def consume(
1319         self,
1320         model_name: str,
1321         offer: Offer,
1322         provider_libjuju: "Libjuju",
1323     ) -> str:
1324         """
1325         Consumes a remote offer in the model. Relations can be created later using "juju relate".
1326
1327         :param: model_name:             Model name
1328         :param: offer:                  Offer object to consume
1329         :param: provider_libjuju:       Libjuju object of the provider endpoint
1330
1331         :raises ParseError if there's a problem parsing the offer_url
1332         :raises JujuError if remote offer includes and endpoint
1333         :raises JujuAPIError if the operation is not successful
1334
1335         :returns: Saas name. It is the application name in the model that reference the remote application.
1336         """
1337         saas_name = f'{offer.name}-{offer.model_name.replace("-", "")}'
1338         if offer.vca_id:
1339             saas_name = f"{saas_name}-{offer.vca_id}"
1340         controller = await self.get_controller()
1341         model = None
1342         provider_controller = None
1343         try:
1344             model = await controller.get_model(model_name)
1345             provider_controller = await provider_libjuju.get_controller()
1346             await model.consume(
1347                 offer.url, application_alias=saas_name, controller=provider_controller
1348             )
1349             return saas_name
1350         finally:
1351             if model:
1352                 await self.disconnect_model(model)
1353             if provider_controller:
1354                 await provider_libjuju.disconnect_controller(provider_controller)
1355             await self.disconnect_controller(controller)
1356
1357     async def destroy_model(self, model_name: str, total_timeout: float = 1800):
1358         """
1359         Destroy model
1360
1361         :param: model_name:     Model name
1362         :param: total_timeout:  Timeout
1363         """
1364
1365         controller = await self.get_controller()
1366         model = None
1367         try:
1368             if not await self.model_exists(model_name, controller=controller):
1369                 self.log.warn(f"Model {model_name} doesn't exist")
1370                 return
1371
1372             self.log.debug(f"Getting model {model_name} to be destroyed")
1373             model = await self.get_model(controller, model_name)
1374             self.log.debug(f"Destroying manual machines in model {model_name}")
1375             # Destroy machines that are manually provisioned
1376             # and still are in pending state
1377             await self._destroy_pending_machines(model, only_manual=True)
1378             await self.disconnect_model(model)
1379
1380             await asyncio.wait_for(
1381                 self._destroy_model(model_name, controller),
1382                 timeout=total_timeout,
1383             )
1384         except Exception as e:
1385             if not await self.model_exists(model_name, controller=controller):
1386                 self.log.warn(
1387                     f"Failed deleting model {model_name}: model doesn't exist"
1388                 )
1389                 return
1390             self.log.warn(f"Failed deleting model {model_name}: {e}")
1391             raise e
1392         finally:
1393             if model:
1394                 await self.disconnect_model(model)
1395             await self.disconnect_controller(controller)
1396
1397     async def _destroy_model(
1398         self,
1399         model_name: str,
1400         controller: Controller,
1401     ):
1402         """
1403         Destroy model from controller
1404
1405         :param: model: Model name to be removed
1406         :param: controller: Controller object
1407         :param: timeout: Timeout in seconds
1408         """
1409         self.log.debug(f"Destroying model {model_name}")
1410
1411         async def _destroy_model_gracefully(model_name: str, controller: Controller):
1412             self.log.info(f"Gracefully deleting model {model_name}")
1413             resolved = False
1414             while model_name in await controller.list_models():
1415                 if not resolved:
1416                     await self.resolve(model_name)
1417                     resolved = True
1418                 await controller.destroy_model(model_name, destroy_storage=True)
1419
1420                 await asyncio.sleep(5)
1421             self.log.info(f"Model {model_name} deleted gracefully")
1422
1423         async def _destroy_model_forcefully(model_name: str, controller: Controller):
1424             self.log.info(f"Forcefully deleting model {model_name}")
1425             while model_name in await controller.list_models():
1426                 await controller.destroy_model(
1427                     model_name, destroy_storage=True, force=True, max_wait=60
1428                 )
1429                 await asyncio.sleep(5)
1430             self.log.info(f"Model {model_name} deleted forcefully")
1431
1432         try:
1433             try:
1434                 await asyncio.wait_for(
1435                     _destroy_model_gracefully(model_name, controller), timeout=120
1436                 )
1437             except asyncio.TimeoutError:
1438                 await _destroy_model_forcefully(model_name, controller)
1439         except juju.errors.JujuError as e:
1440             if any("has been removed" in error for error in e.errors):
1441                 return
1442             if any("model not found" in error for error in e.errors):
1443                 return
1444             raise e
1445
1446     async def destroy_application(
1447         self, model_name: str, application_name: str, total_timeout: float
1448     ):
1449         """
1450         Destroy application
1451
1452         :param: model_name:         Model name
1453         :param: application_name:   Application name
1454         :param: total_timeout:      Timeout
1455         """
1456
1457         controller = await self.get_controller()
1458         model = None
1459
1460         try:
1461             model = await self.get_model(controller, model_name)
1462             self.log.debug(
1463                 "Destroying application {} in model {}".format(
1464                     application_name, model_name
1465                 )
1466             )
1467             application = self._get_application(model, application_name)
1468             if application:
1469                 await application.destroy()
1470             else:
1471                 self.log.warning("Application not found: {}".format(application_name))
1472
1473             self.log.debug(
1474                 "Waiting for application {} to be destroyed in model {}...".format(
1475                     application_name, model_name
1476                 )
1477             )
1478             if total_timeout is None:
1479                 total_timeout = 3600
1480             end = time.time() + total_timeout
1481             while time.time() < end:
1482                 if not self._get_application(model, application_name):
1483                     self.log.debug(
1484                         "The application {} was destroyed in model {} ".format(
1485                             application_name, model_name
1486                         )
1487                     )
1488                     return
1489                 await asyncio.sleep(5)
1490             raise Exception(
1491                 "Timeout waiting for application {} to be destroyed in model {}".format(
1492                     application_name, model_name
1493                 )
1494             )
1495         finally:
1496             if model is not None:
1497                 await self.disconnect_model(model)
1498             await self.disconnect_controller(controller)
1499
1500     async def _destroy_pending_machines(self, model: Model, only_manual: bool = False):
1501         """
1502         Destroy pending machines in a given model
1503
1504         :param: only_manual:    Bool that indicates only manually provisioned
1505                                 machines should be destroyed (if True), or that
1506                                 all pending machines should be destroyed
1507         """
1508         status = await model.get_status()
1509         for machine_id in status.machines:
1510             machine_status = status.machines[machine_id]
1511             if machine_status.agent_status.status == "pending":
1512                 if only_manual and not machine_status.instance_id.startswith("manual:"):
1513                     break
1514                 machine = model.machines[machine_id]
1515                 await machine.destroy(force=True)
1516
1517     async def configure_application(
1518         self, model_name: str, application_name: str, config: dict = None
1519     ):
1520         """Configure application
1521
1522         :param: model_name:         Model name
1523         :param: application_name:   Application name
1524         :param: config:             Config to apply to the charm
1525         """
1526         self.log.debug("Configuring application {}".format(application_name))
1527
1528         if config:
1529             controller = await self.get_controller()
1530             model = None
1531             try:
1532                 model = await self.get_model(controller, model_name)
1533                 application = self._get_application(
1534                     model,
1535                     application_name=application_name,
1536                 )
1537                 await application.set_config(config)
1538             finally:
1539                 if model:
1540                     await self.disconnect_model(model)
1541                 await self.disconnect_controller(controller)
1542
1543     def handle_exception(self, loop, context):
1544         # All unhandled exceptions by libjuju are handled here.
1545         pass
1546
1547     async def health_check(self, interval: float = 300.0):
1548         """
1549         Health check to make sure controller and controller_model connections are OK
1550
1551         :param: interval: Time in seconds between checks
1552         """
1553         controller = None
1554         while True:
1555             try:
1556                 controller = await self.get_controller()
1557                 # self.log.debug("VCA is alive")
1558             except Exception as e:
1559                 self.log.error("Health check to VCA failed: {}".format(e))
1560             finally:
1561                 await self.disconnect_controller(controller)
1562             await asyncio.sleep(interval)
1563
1564     async def list_models(self, contains: str = None) -> [str]:
1565         """List models with certain names
1566
1567         :param: contains:   String that is contained in model name
1568
1569         :retur: [models] Returns list of model names
1570         """
1571
1572         controller = await self.get_controller()
1573         try:
1574             models = await controller.list_models()
1575             if contains:
1576                 models = [model for model in models if contains in model]
1577             return models
1578         finally:
1579             await self.disconnect_controller(controller)
1580
1581     async def _list_offers(
1582         self, model_name: str, offer_name: str = None
1583     ) -> QueryApplicationOffersResults:
1584         """
1585         List offers within a model
1586
1587         :param: model_name: Model name
1588         :param: offer_name: Offer name to filter.
1589
1590         :return: Returns application offers results in the model
1591         """
1592
1593         controller = await self.get_controller()
1594         try:
1595             offers = (await controller.list_offers(model_name)).results
1596             if offer_name:
1597                 matching_offer = []
1598                 for offer in offers:
1599                     if offer.offer_name == offer_name:
1600                         matching_offer.append(offer)
1601                         break
1602                 offers = matching_offer
1603             return offers
1604         finally:
1605             await self.disconnect_controller(controller)
1606
1607     async def add_k8s(
1608         self,
1609         name: str,
1610         rbac_id: str,
1611         token: str,
1612         client_cert_data: str,
1613         configuration: Configuration,
1614         storage_class: str,
1615         credential_name: str = None,
1616     ):
1617         """
1618         Add a Kubernetes cloud to the controller
1619
1620         Similar to the `juju add-k8s` command in the CLI
1621
1622         :param: name:               Name for the K8s cloud
1623         :param: configuration:      Kubernetes configuration object
1624         :param: storage_class:      Storage Class to use in the cloud
1625         :param: credential_name:    Storage Class to use in the cloud
1626         """
1627
1628         if not storage_class:
1629             raise Exception("storage_class must be a non-empty string")
1630         if not name:
1631             raise Exception("name must be a non-empty string")
1632         if not configuration:
1633             raise Exception("configuration must be provided")
1634
1635         endpoint = configuration.host
1636         credential = self.get_k8s_cloud_credential(
1637             configuration,
1638             client_cert_data,
1639             token,
1640         )
1641         credential.attrs[RBAC_LABEL_KEY_NAME] = rbac_id
1642         cloud = client.Cloud(
1643             type_="kubernetes",
1644             auth_types=[credential.auth_type],
1645             endpoint=endpoint,
1646             ca_certificates=[client_cert_data],
1647             config={
1648                 "operator-storage": storage_class,
1649                 "workload-storage": storage_class,
1650             },
1651         )
1652
1653         return await self.add_cloud(
1654             name, cloud, credential, credential_name=credential_name
1655         )
1656
1657     def get_k8s_cloud_credential(
1658         self,
1659         configuration: Configuration,
1660         client_cert_data: str,
1661         token: str = None,
1662     ) -> client.CloudCredential:
1663         attrs = {}
1664         # TODO: Test with AKS
1665         key = None  # open(configuration.key_file, "r").read()
1666         username = configuration.username
1667         password = configuration.password
1668
1669         if client_cert_data:
1670             attrs["ClientCertificateData"] = client_cert_data
1671         if key:
1672             attrs["ClientKeyData"] = key
1673         if token:
1674             if username or password:
1675                 raise JujuInvalidK8sConfiguration("Cannot set both token and user/pass")
1676             attrs["Token"] = token
1677
1678         auth_type = None
1679         if key:
1680             auth_type = "oauth2"
1681             if client_cert_data:
1682                 auth_type = "oauth2withcert"
1683             if not token:
1684                 raise JujuInvalidK8sConfiguration(
1685                     "missing token for auth type {}".format(auth_type)
1686                 )
1687         elif username:
1688             if not password:
1689                 self.log.debug(
1690                     "credential for user {} has empty password".format(username)
1691                 )
1692             attrs["username"] = username
1693             attrs["password"] = password
1694             if client_cert_data:
1695                 auth_type = "userpasswithcert"
1696             else:
1697                 auth_type = "userpass"
1698         elif client_cert_data and token:
1699             auth_type = "certificate"
1700         else:
1701             raise JujuInvalidK8sConfiguration("authentication method not supported")
1702         return client.CloudCredential(auth_type=auth_type, attrs=attrs)
1703
1704     async def add_cloud(
1705         self,
1706         name: str,
1707         cloud: Cloud,
1708         credential: CloudCredential = None,
1709         credential_name: str = None,
1710     ) -> Cloud:
1711         """
1712         Add cloud to the controller
1713
1714         :param: name:               Name of the cloud to be added
1715         :param: cloud:              Cloud object
1716         :param: credential:         CloudCredentials object for the cloud
1717         :param: credential_name:    Credential name.
1718                                     If not defined, cloud of the name will be used.
1719         """
1720         controller = await self.get_controller()
1721         try:
1722             _ = await controller.add_cloud(name, cloud)
1723             if credential:
1724                 await controller.add_credential(
1725                     credential_name or name, credential=credential, cloud=name
1726                 )
1727             # Need to return the object returned by the controller.add_cloud() function
1728             # I'm returning the original value now until this bug is fixed:
1729             #   https://github.com/juju/python-libjuju/issues/443
1730             return cloud
1731         finally:
1732             await self.disconnect_controller(controller)
1733
1734     async def remove_cloud(self, name: str):
1735         """
1736         Remove cloud
1737
1738         :param: name:   Name of the cloud to be removed
1739         """
1740         controller = await self.get_controller()
1741         try:
1742             await controller.remove_cloud(name)
1743         except juju.errors.JujuError as e:
1744             if len(e.errors) == 1 and f'cloud "{name}" not found' == e.errors[0]:
1745                 self.log.warning(f"Cloud {name} not found, so it could not be deleted.")
1746             else:
1747                 raise e
1748         finally:
1749             await self.disconnect_controller(controller)
1750
1751     @retry(attempts=20, delay=5, fallback=JujuLeaderUnitNotFound())
1752     async def _get_leader_unit(self, application: Application) -> Unit:
1753         unit = None
1754         for u in application.units:
1755             if await u.is_leader_from_status():
1756                 unit = u
1757                 break
1758         if not unit:
1759             raise Exception()
1760         return unit
1761
1762     async def get_cloud_credentials(self, cloud: Cloud) -> typing.List:
1763         """
1764         Get cloud credentials
1765
1766         :param: cloud: Cloud object. The returned credentials will be from this cloud.
1767
1768         :return: List of credentials object associated to the specified cloud
1769
1770         """
1771         controller = await self.get_controller()
1772         try:
1773             facade = client.CloudFacade.from_connection(controller.connection())
1774             cloud_cred_tag = tag.credential(
1775                 cloud.name, self.vca_connection.data.user, cloud.credential_name
1776             )
1777             params = [client.Entity(cloud_cred_tag)]
1778             return (await facade.Credential(params)).results
1779         finally:
1780             await self.disconnect_controller(controller)
1781
1782     async def check_application_exists(self, model_name, application_name) -> bool:
1783         """Check application exists
1784
1785         :param: model_name:         Model Name
1786         :param: application_name:   Application Name
1787
1788         :return: Boolean
1789         """
1790
1791         model = None
1792         controller = await self.get_controller()
1793         try:
1794             model = await self.get_model(controller, model_name)
1795             self.log.debug(
1796                 "Checking if application {} exists in model {}".format(
1797                     application_name, model_name
1798                 )
1799             )
1800             return self._get_application(model, application_name) is not None
1801         finally:
1802             if model:
1803                 await self.disconnect_model(model)
1804             await self.disconnect_controller(controller)