n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46 )
  47 from n2vc.juju_observer import JujuModelObserver
  48 from n2vc.n2vc_conn import N2VCConnector
  49 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  50 from n2vc.provisioner import AsyncSSHProvisioner
  51 from n2vc.libjuju import Libjuju
  52
  53
  54 class N2VCJujuConnector(N2VCConnector):
  55
  56     """
  57 ####################################################################################
  58 ################################### P U B L I C ####################################
  59 ####################################################################################
  60     """
  61
  62     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  63
  64     def __init__(
  65         self,
  66         db: object,
  67         fs: object,
  68         log: object = None,
  69         loop: object = None,
  70         url: str = "127.0.0.1:17070",
  71         username: str = "admin",
  72         vca_config: dict = None,
  73         on_update_db=None,
  74     ):
  75         """Initialize juju N2VC connector
  76         """
  77
  78         # parent class constructor
  79         N2VCConnector.__init__(
  80             self,
  81             db=db,
  82             fs=fs,
  83             log=log,
  84             loop=loop,
  85             url=url,
  86             username=username,
  87             vca_config=vca_config,
  88             on_update_db=on_update_db,
  89         )
  90
  91         # silence websocket traffic log
  92         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  93         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  94         logging.getLogger("model").setLevel(logging.WARN)
  95
  96         self.log.info("Initializing N2VC juju connector...")
  97
  98         """
  99         ##############################################################
 100         # check arguments
 101         ##############################################################
 102         """
 103
 104         # juju URL
 105         if url is None:
 106             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 107         url_parts = url.split(":")
 108         if len(url_parts) != 2:
 109             raise N2VCBadArgumentsException(
 110                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 111             )
 112         self.hostname = url_parts[0]
 113         try:
 114             self.port = int(url_parts[1])
 115         except ValueError:
 116             raise N2VCBadArgumentsException(
 117                 "url port must be a number -> {}".format(url), ["url"]
 118             )
 119
 120         # juju USERNAME
 121         if username is None:
 122             raise N2VCBadArgumentsException(
 123                 "Argument username is mandatory", ["username"]
 124             )
 125
 126         # juju CONFIGURATION
 127         if vca_config is None:
 128             raise N2VCBadArgumentsException(
 129                 "Argument vca_config is mandatory", ["vca_config"]
 130             )
 131
 132         if "secret" in vca_config:
 133             self.secret = vca_config["secret"]
 134         else:
 135             raise N2VCBadArgumentsException(
 136                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 137             )
 138
 139         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 140         # if exists, it will be written in lcm container: _create_juju_public_key()
 141         if "public_key" in vca_config:
 142             self.public_key = vca_config["public_key"]
 143         else:
 144             self.public_key = None
 145
 146         # TODO: Verify ca_cert is valid before using. VCA will crash
 147         # if the ca_cert isn't formatted correctly.
 148         def base64_to_cacert(b64string):
 149             """Convert the base64-encoded string containing the VCA CACERT.
 150
 151             The input string....
 152
 153             """
 154             try:
 155                 cacert = base64.b64decode(b64string).decode("utf-8")
 156
 157                 cacert = re.sub(r"\\n", r"\n", cacert,)
 158             except binascii.Error as e:
 159                 self.log.debug("Caught binascii.Error: {}".format(e))
 160                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 161
 162             return cacert
 163
 164         self.ca_cert = vca_config.get("ca_cert")
 165         if self.ca_cert:
 166             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 167
 168         if "api_proxy" in vca_config:
 169             self.api_proxy = vca_config["api_proxy"]
 170             self.log.debug(
 171                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 172             )
 173         else:
 174             self.warning(
 175                 "api_proxy is not configured. Support for native charms is disabled"
 176             )
 177
 178         if "enable_os_upgrade" in vca_config:
 179             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 180         else:
 181             self.enable_os_upgrade = True
 182
 183         if "apt_mirror" in vca_config:
 184             self.apt_mirror = vca_config["apt_mirror"]
 185         else:
 186             self.apt_mirror = None
 187
 188         self.cloud = vca_config.get("cloud")
 189         # self.log.debug('Arguments have been checked')
 190
 191         # juju data
 192         self.controller = None  # it will be filled when connect to juju
 193         self.juju_models = {}  # model objects for every model_name
 194         self.juju_observers = {}  # model observers for every model_name
 195         self._connecting = (
 196             False  # while connecting to juju (to avoid duplicate connections)
 197         )
 198         self._authenticated = (
 199             False  # it will be True when juju connection be stablished
 200         )
 201         self._creating_model = False  # True during model creation
 202         self.libjuju = Libjuju(
 203             endpoint=self.url,
 204             api_proxy=self.api_proxy,
 205             enable_os_upgrade=self.enable_os_upgrade,
 206             apt_mirror=self.apt_mirror,
 207             username=self.username,
 208             password=self.secret,
 209             cacert=self.ca_cert,
 210             loop=self.loop,
 211             log=self.log,
 212             db=self.db,
 213             n2vc=self,
 214         )
 215
 216         # create juju pub key file in lcm container at
 217         # ./local/share/juju/ssh/juju_id_rsa.pub
 218         self._create_juju_public_key()
 219
 220         self.log.info("N2VC juju connector initialized")
 221
 222     async def get_status(self, namespace: str, yaml_format: bool = True):
 223
 224         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 225
 226         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 227             namespace=namespace
 228         )
 229         # model name is ns_id
 230         model_name = ns_id
 231         if model_name is None:
 232             msg = "Namespace {} not valid".format(namespace)
 233             self.log.error(msg)
 234             raise N2VCBadArgumentsException(msg, ["namespace"])
 235
 236         status = await self.libjuju.get_model_status(model_name)
 237
 238         if yaml_format:
 239             return obj_to_yaml(status)
 240         else:
 241             return obj_to_dict(status)
 242
 243     async def create_execution_environment(
 244         self,
 245         namespace: str,
 246         db_dict: dict,
 247         reuse_ee_id: str = None,
 248         progress_timeout: float = None,
 249         total_timeout: float = None,
 250     ) -> (str, dict):
 251
 252         self.log.info(
 253             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 254                 namespace, reuse_ee_id
 255             )
 256         )
 257
 258         machine_id = None
 259         if reuse_ee_id:
 260             model_name, application_name, machine_id = self._get_ee_id_components(
 261                 ee_id=reuse_ee_id
 262             )
 263         else:
 264             (
 265                 _nsi_id,
 266                 ns_id,
 267                 _vnf_id,
 268                 _vdu_id,
 269                 _vdu_count,
 270             ) = self._get_namespace_components(namespace=namespace)
 271             # model name is ns_id
 272             model_name = ns_id
 273             # application name
 274             application_name = self._get_application_name(namespace=namespace)
 275
 276         self.log.debug(
 277             "model name: {}, application name:  {}, machine_id: {}".format(
 278                 model_name, application_name, machine_id
 279             )
 280         )
 281
 282         # create or reuse a new juju machine
 283         try:
 284             if not await self.libjuju.model_exists(model_name):
 285                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 286             machine, new = await self.libjuju.create_machine(
 287                 model_name=model_name,
 288                 machine_id=machine_id,
 289                 db_dict=db_dict,
 290                 progress_timeout=progress_timeout,
 291                 total_timeout=total_timeout,
 292             )
 293             # id for the execution environment
 294             ee_id = N2VCJujuConnector._build_ee_id(
 295                 model_name=model_name,
 296                 application_name=application_name,
 297                 machine_id=str(machine.entity_id),
 298             )
 299             self.log.debug("ee_id: {}".format(ee_id))
 300
 301             if new:
 302                 # write ee_id in database
 303                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 304
 305         except Exception as e:
 306             message = "Error creating machine on juju: {}".format(e)
 307             self.log.error(message)
 308             raise N2VCException(message=message)
 309
 310         # new machine credentials
 311         credentials = {
 312             "hostname": machine.dns_name,
 313         }
 314
 315         self.log.info(
 316             "Execution environment created. ee_id: {}, credentials: {}".format(
 317                 ee_id, credentials
 318             )
 319         )
 320
 321         return ee_id, credentials
 322
 323     async def register_execution_environment(
 324         self,
 325         namespace: str,
 326         credentials: dict,
 327         db_dict: dict,
 328         progress_timeout: float = None,
 329         total_timeout: float = None,
 330     ) -> str:
 331
 332         self.log.info(
 333             "Registering execution environment. namespace={}, credentials={}".format(
 334                 namespace, credentials
 335             )
 336         )
 337
 338         if credentials is None:
 339             raise N2VCBadArgumentsException(
 340                 message="credentials are mandatory", bad_args=["credentials"]
 341             )
 342         if credentials.get("hostname"):
 343             hostname = credentials["hostname"]
 344         else:
 345             raise N2VCBadArgumentsException(
 346                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 347             )
 348         if credentials.get("username"):
 349             username = credentials["username"]
 350         else:
 351             raise N2VCBadArgumentsException(
 352                 message="username is mandatory", bad_args=["credentials.username"]
 353             )
 354         if "private_key_path" in credentials:
 355             private_key_path = credentials["private_key_path"]
 356         else:
 357             # if not passed as argument, use generated private key path
 358             private_key_path = self.private_key_path
 359
 360         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 361             namespace=namespace
 362         )
 363
 364         # model name
 365         model_name = ns_id
 366         # application name
 367         application_name = self._get_application_name(namespace=namespace)
 368
 369         # register machine on juju
 370         try:
 371             if not self.api_proxy:
 372                 msg = "Cannot provision machine: api_proxy is not defined"
 373                 self.log.error(msg=msg)
 374                 raise N2VCException(message=msg)
 375             if not await self.libjuju.model_exists(model_name):
 376                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 377             machine_id = await self.libjuju.provision_machine(
 378                 model_name=model_name,
 379                 hostname=hostname,
 380                 username=username,
 381                 private_key_path=private_key_path,
 382                 db_dict=db_dict,
 383                 progress_timeout=progress_timeout,
 384                 total_timeout=total_timeout,
 385             )
 386         except Exception as e:
 387             self.log.error("Error registering machine: {}".format(e))
 388             raise N2VCException(
 389                 message="Error registering machine on juju: {}".format(e)
 390             )
 391
 392         self.log.info("Machine registered: {}".format(machine_id))
 393
 394         # id for the execution environment
 395         ee_id = N2VCJujuConnector._build_ee_id(
 396             model_name=model_name,
 397             application_name=application_name,
 398             machine_id=str(machine_id),
 399         )
 400
 401         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 402
 403         return ee_id
 404
 405     async def install_configuration_sw(
 406         self,
 407         ee_id: str,
 408         artifact_path: str,
 409         db_dict: dict,
 410         progress_timeout: float = None,
 411         total_timeout: float = None,
 412         config: dict = None,
 413     ):
 414
 415         self.log.info(
 416             (
 417                 "Installing configuration sw on ee_id: {}, "
 418                 "artifact path: {}, db_dict: {}"
 419             ).format(ee_id, artifact_path, db_dict)
 420         )
 421
 422         # check arguments
 423         if ee_id is None or len(ee_id) == 0:
 424             raise N2VCBadArgumentsException(
 425                 message="ee_id is mandatory", bad_args=["ee_id"]
 426             )
 427         if artifact_path is None or len(artifact_path) == 0:
 428             raise N2VCBadArgumentsException(
 429                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 430             )
 431         if db_dict is None:
 432             raise N2VCBadArgumentsException(
 433                 message="db_dict is mandatory", bad_args=["db_dict"]
 434             )
 435
 436         try:
 437             (
 438                 model_name,
 439                 application_name,
 440                 machine_id,
 441             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 442             self.log.debug(
 443                 "model: {}, application: {}, machine: {}".format(
 444                     model_name, application_name, machine_id
 445                 )
 446             )
 447         except Exception:
 448             raise N2VCBadArgumentsException(
 449                 message="ee_id={} is not a valid execution environment id".format(
 450                     ee_id
 451                 ),
 452                 bad_args=["ee_id"],
 453             )
 454
 455         # remove // in charm path
 456         while artifact_path.find("//") >= 0:
 457             artifact_path = artifact_path.replace("//", "/")
 458
 459         # check charm path
 460         if not self.fs.file_exists(artifact_path, mode="dir"):
 461             msg = "artifact path does not exist: {}".format(artifact_path)
 462             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 463
 464         if artifact_path.startswith("/"):
 465             full_path = self.fs.path + artifact_path
 466         else:
 467             full_path = self.fs.path + "/" + artifact_path
 468
 469         try:
 470             await self.libjuju.deploy_charm(
 471                 model_name=model_name,
 472                 application_name=application_name,
 473                 path=full_path,
 474                 machine_id=machine_id,
 475                 db_dict=db_dict,
 476                 progress_timeout=progress_timeout,
 477                 total_timeout=total_timeout,
 478                 config=config,
 479             )
 480         except Exception as e:
 481             raise N2VCException(
 482                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 483             )
 484
 485         self.log.info("Configuration sw installed")
 486
 487     async def get_ee_ssh_public__key(
 488         self,
 489         ee_id: str,
 490         db_dict: dict,
 491         progress_timeout: float = None,
 492         total_timeout: float = None,
 493     ) -> str:
 494
 495         self.log.info(
 496             (
 497                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 498             ).format(ee_id, db_dict)
 499         )
 500
 501         # check arguments
 502         if ee_id is None or len(ee_id) == 0:
 503             raise N2VCBadArgumentsException(
 504                 message="ee_id is mandatory", bad_args=["ee_id"]
 505             )
 506         if db_dict is None:
 507             raise N2VCBadArgumentsException(
 508                 message="db_dict is mandatory", bad_args=["db_dict"]
 509             )
 510
 511         try:
 512             (
 513                 model_name,
 514                 application_name,
 515                 machine_id,
 516             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 517             self.log.debug(
 518                 "model: {}, application: {}, machine: {}".format(
 519                     model_name, application_name, machine_id
 520                 )
 521             )
 522         except Exception:
 523             raise N2VCBadArgumentsException(
 524                 message="ee_id={} is not a valid execution environment id".format(
 525                     ee_id
 526                 ),
 527                 bad_args=["ee_id"],
 528             )
 529
 530         # try to execute ssh layer primitives (if exist):
 531         #       generate-ssh-key
 532         #       get-ssh-public-key
 533
 534         output = None
 535
 536         application_name = N2VCJujuConnector._format_app_name(application_name)
 537
 538         # execute action: generate-ssh-key
 539         try:
 540             output, _status = await self.libjuju.execute_action(
 541                 model_name=model_name,
 542                 application_name=application_name,
 543                 action_name="generate-ssh-key",
 544                 db_dict=db_dict,
 545                 progress_timeout=progress_timeout,
 546                 total_timeout=total_timeout,
 547             )
 548         except Exception as e:
 549             self.log.info(
 550                 "Skipping exception while executing action generate-ssh-key: {}".format(
 551                     e
 552                 )
 553             )
 554
 555         # execute action: get-ssh-public-key
 556         try:
 557             output, _status = await self.libjuju.execute_action(
 558                 model_name=model_name,
 559                 application_name=application_name,
 560                 action_name="get-ssh-public-key",
 561                 db_dict=db_dict,
 562                 progress_timeout=progress_timeout,
 563                 total_timeout=total_timeout,
 564             )
 565         except Exception as e:
 566             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 567             self.log.info(msg)
 568             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 569
 570         # return public key if exists
 571         return output["pubkey"] if "pubkey" in output else output
 572
 573     async def add_relation(
 574         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 575     ):
 576
 577         self.log.debug(
 578             "adding new relation between {} and {}, endpoints: {}, {}".format(
 579                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 580             )
 581         )
 582
 583         # check arguments
 584         if not ee_id_1:
 585             message = "EE 1 is mandatory"
 586             self.log.error(message)
 587             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 588         if not ee_id_2:
 589             message = "EE 2 is mandatory"
 590             self.log.error(message)
 591             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 592         if not endpoint_1:
 593             message = "endpoint 1 is mandatory"
 594             self.log.error(message)
 595             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 596         if not endpoint_2:
 597             message = "endpoint 2 is mandatory"
 598             self.log.error(message)
 599             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 600
 601         # get the model, the applications and the machines from the ee_id's
 602         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 603         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 604
 605         # model must be the same
 606         if model_1 != model_2:
 607             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 608             self.log.error(message)
 609             raise N2VCBadArgumentsException(
 610                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 611             )
 612
 613         # add juju relations between two applications
 614         try:
 615             await self.libjuju.add_relation(
 616                 model_name=model_1,
 617                 application_name_1=app_1,
 618                 application_name_2=app_2,
 619                 relation_1=endpoint_1,
 620                 relation_2=endpoint_2,
 621             )
 622         except Exception as e:
 623             message = "Error adding relation between {} and {}: {}".format(
 624                 ee_id_1, ee_id_2, e
 625             )
 626             self.log.error(message)
 627             raise N2VCException(message=message)
 628
 629     async def remove_relation(self):
 630         # TODO
 631         self.log.info("Method not implemented yet")
 632         raise MethodNotImplemented()
 633
 634     async def deregister_execution_environments(self):
 635         self.log.info("Method not implemented yet")
 636         raise MethodNotImplemented()
 637
 638     async def delete_namespace(
 639         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 640     ):
 641         self.log.info("Deleting namespace={}".format(namespace))
 642
 643         # check arguments
 644         if namespace is None:
 645             raise N2VCBadArgumentsException(
 646                 message="namespace is mandatory", bad_args=["namespace"]
 647             )
 648
 649         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 650             namespace=namespace
 651         )
 652         if ns_id is not None:
 653             try:
 654                 if not await self.libjuju.model_exists(ns_id):
 655                     raise N2VCNotFound(message="Model {} does not exist".format(ns_id))
 656                 await self.libjuju.destroy_model(
 657                     model_name=ns_id, total_timeout=total_timeout
 658                 )
 659             except N2VCNotFound:
 660                 raise
 661             except Exception as e:
 662                 raise N2VCException(
 663                     message="Error deleting namespace {} : {}".format(namespace, e)
 664                 )
 665         else:
 666             raise N2VCBadArgumentsException(
 667                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 668             )
 669
 670         self.log.info("Namespace {} deleted".format(namespace))
 671
 672     async def delete_execution_environment(
 673         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 674     ):
 675         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 676
 677         # check arguments
 678         if ee_id is None:
 679             raise N2VCBadArgumentsException(
 680                 message="ee_id is mandatory", bad_args=["ee_id"]
 681             )
 682
 683         model_name, application_name, _machine_id = self._get_ee_id_components(
 684             ee_id=ee_id
 685         )
 686
 687         # destroy the application
 688         try:
 689             await self.libjuju.destroy_model(
 690                 model_name=model_name, total_timeout=total_timeout
 691             )
 692         except Exception as e:
 693             raise N2VCException(
 694                 message=(
 695                     "Error deleting execution environment {} (application {}) : {}"
 696                 ).format(ee_id, application_name, e)
 697             )
 698
 699         # destroy the machine
 700         # try:
 701         #     await self._juju_destroy_machine(
 702         #         model_name=model_name,
 703         #         machine_id=machine_id,
 704         #         total_timeout=total_timeout
 705         #     )
 706         # except Exception as e:
 707         #     raise N2VCException(
 708         #        message='Error deleting execution environment {} (machine {}) : {}'
 709         #                .format(ee_id, machine_id, e))
 710
 711         self.log.info("Execution environment {} deleted".format(ee_id))
 712
 713     async def exec_primitive(
 714         self,
 715         ee_id: str,
 716         primitive_name: str,
 717         params_dict: dict,
 718         db_dict: dict = None,
 719         progress_timeout: float = None,
 720         total_timeout: float = None,
 721     ) -> str:
 722
 723         self.log.info(
 724             "Executing primitive: {} on ee: {}, params: {}".format(
 725                 primitive_name, ee_id, params_dict
 726             )
 727         )
 728
 729         # check arguments
 730         if ee_id is None or len(ee_id) == 0:
 731             raise N2VCBadArgumentsException(
 732                 message="ee_id is mandatory", bad_args=["ee_id"]
 733             )
 734         if primitive_name is None or len(primitive_name) == 0:
 735             raise N2VCBadArgumentsException(
 736                 message="action_name is mandatory", bad_args=["action_name"]
 737             )
 738         if params_dict is None:
 739             params_dict = dict()
 740
 741         try:
 742             (
 743                 model_name,
 744                 application_name,
 745                 _machine_id,
 746             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 747         except Exception:
 748             raise N2VCBadArgumentsException(
 749                 message="ee_id={} is not a valid execution environment id".format(
 750                     ee_id
 751                 ),
 752                 bad_args=["ee_id"],
 753             )
 754
 755         if primitive_name == "config":
 756             # Special case: config primitive
 757             try:
 758                 await self.libjuju.configure_application(
 759                     model_name=model_name,
 760                     application_name=application_name,
 761                     config=params_dict,
 762                 )
 763                 actions = await self.libjuju.get_actions(
 764                     application_name=application_name, model_name=model_name,
 765                 )
 766                 self.log.debug(
 767                     "Application {} has these actions: {}".format(
 768                         application_name, actions
 769                     )
 770                 )
 771                 if "verify-ssh-credentials" in actions:
 772                     # execute verify-credentials
 773                     num_retries = 20
 774                     retry_timeout = 15.0
 775                     for _ in range(num_retries):
 776                         try:
 777                             self.log.debug("Executing action verify-ssh-credentials...")
 778                             output, ok = await self.libjuju.execute_action(
 779                                 model_name=model_name,
 780                                 application_name=application_name,
 781                                 action_name="verify-ssh-credentials",
 782                                 db_dict=db_dict,
 783                                 progress_timeout=progress_timeout,
 784                                 total_timeout=total_timeout,
 785                             )
 786                             self.log.debug("Result: {}, output: {}".format(ok, output))
 787                             break
 788                         except asyncio.CancelledError:
 789                             raise
 790                         except Exception as e:
 791                             self.log.debug(
 792                                 "Error executing verify-ssh-credentials: {}. Retrying...".format(
 793                                     e
 794                                 )
 795                             )
 796                             await asyncio.sleep(retry_timeout)
 797                     else:
 798                         self.log.error(
 799                             "Error executing verify-ssh-credentials after {} retries. ".format(
 800                                 num_retries
 801                             )
 802                         )
 803                 else:
 804                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 805                         application_name
 806                     )
 807                     self.log.debug(msg=msg)
 808             except Exception as e:
 809                 self.log.error("Error configuring juju application: {}".format(e))
 810                 raise N2VCExecutionException(
 811                     message="Error configuring application into ee={} : {}".format(
 812                         ee_id, e
 813                     ),
 814                     primitive_name=primitive_name,
 815                 )
 816             return "CONFIG OK"
 817         else:
 818             try:
 819                 output, status = await self.libjuju.execute_action(
 820                     model_name=model_name,
 821                     application_name=application_name,
 822                     action_name=primitive_name,
 823                     db_dict=db_dict,
 824                     progress_timeout=progress_timeout,
 825                     total_timeout=total_timeout,
 826                     **params_dict
 827                 )
 828                 if status == "completed":
 829                     return output
 830                 else:
 831                     raise Exception("status is not completed: {}".format(status))
 832             except Exception as e:
 833                 self.log.error(
 834                     "Error executing primitive {}: {}".format(primitive_name, e)
 835                 )
 836                 raise N2VCExecutionException(
 837                     message="Error executing primitive {} into ee={} : {}".format(
 838                         primitive_name, ee_id, e
 839                     ),
 840                     primitive_name=primitive_name,
 841                 )
 842
 843     async def disconnect(self):
 844         self.log.info("closing juju N2VC...")
 845         try:
 846             await self.libjuju.disconnect()
 847         except Exception as e:
 848             raise N2VCConnectionException(
 849                 message="Error disconnecting controller: {}".format(e), url=self.url
 850             )
 851
 852     """
 853 ####################################################################################
 854 ################################### P R I V A T E ##################################
 855 ####################################################################################
 856     """
 857
 858     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 859
 860         # write ee_id to database: _admin.deployed.VCA.x
 861         try:
 862             the_table = db_dict["collection"]
 863             the_filter = db_dict["filter"]
 864             the_path = db_dict["path"]
 865             if not the_path[-1] == ".":
 866                 the_path = the_path + "."
 867             update_dict = {the_path + "ee_id": ee_id}
 868             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 869             self.db.set_one(
 870                 table=the_table,
 871                 q_filter=the_filter,
 872                 update_dict=update_dict,
 873                 fail_on_empty=True,
 874             )
 875         except asyncio.CancelledError:
 876             raise
 877         except Exception as e:
 878             self.log.error("Error writing ee_id to database: {}".format(e))
 879
 880     @staticmethod
 881     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 882         """
 883         Build an execution environment id form model, application and machine
 884         :param model_name:
 885         :param application_name:
 886         :param machine_id:
 887         :return:
 888         """
 889         # id for the execution environment
 890         return "{}.{}.{}".format(model_name, application_name, machine_id)
 891
 892     @staticmethod
 893     def _get_ee_id_components(ee_id: str) -> (str, str, str):
 894         """
 895         Get model, application and machine components from an execution environment id
 896         :param ee_id:
 897         :return: model_name, application_name, machine_id
 898         """
 899
 900         if ee_id is None:
 901             return None, None, None
 902
 903         # split components of id
 904         parts = ee_id.split(".")
 905         model_name = parts[0]
 906         application_name = parts[1]
 907         machine_id = parts[2]
 908         return model_name, application_name, machine_id
 909
 910     def _get_application_name(self, namespace: str) -> str:
 911         """
 912         Build application name from namespace
 913         :param namespace:
 914         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
 915         """
 916
 917         # TODO: Enforce the Juju 50-character application limit
 918
 919         # split namespace components
 920         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
 921             namespace=namespace
 922         )
 923
 924         if vnf_id is None or len(vnf_id) == 0:
 925             vnf_id = ""
 926         else:
 927             # Shorten the vnf_id to its last twelve characters
 928             vnf_id = "vnf-" + vnf_id[-12:]
 929
 930         if vdu_id is None or len(vdu_id) == 0:
 931             vdu_id = ""
 932         else:
 933             # Shorten the vdu_id to its last twelve characters
 934             vdu_id = "-vdu-" + vdu_id[-12:]
 935
 936         if vdu_count is None or len(vdu_count) == 0:
 937             vdu_count = ""
 938         else:
 939             vdu_count = "-cnt-" + vdu_count
 940
 941         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
 942
 943         return N2VCJujuConnector._format_app_name(application_name)
 944
 945     async def _juju_create_machine(
 946         self,
 947         model_name: str,
 948         application_name: str,
 949         machine_id: str = None,
 950         db_dict: dict = None,
 951         progress_timeout: float = None,
 952         total_timeout: float = None,
 953     ) -> Machine:
 954
 955         self.log.debug(
 956             "creating machine in model: {}, existing machine id: {}".format(
 957                 model_name, machine_id
 958             )
 959         )
 960
 961         # get juju model and observer (create model if needed)
 962         model = await self._juju_get_model(model_name=model_name)
 963         observer = self.juju_observers[model_name]
 964
 965         # find machine id in model
 966         machine = None
 967         if machine_id is not None:
 968             self.log.debug("Finding existing machine id {} in model".format(machine_id))
 969             # get juju existing machines in the model
 970             existing_machines = await model.get_machines()
 971             if machine_id in existing_machines:
 972                 self.log.debug(
 973                     "Machine id {} found in model (reusing it)".format(machine_id)
 974                 )
 975                 machine = model.machines[machine_id]
 976
 977         if machine is None:
 978             self.log.debug("Creating a new machine in juju...")
 979             # machine does not exist, create it and wait for it
 980             machine = await model.add_machine(
 981                 spec=None, constraints=None, disks=None, series="xenial"
 982             )
 983
 984             # register machine with observer
 985             observer.register_machine(machine=machine, db_dict=db_dict)
 986
 987             # id for the execution environment
 988             ee_id = N2VCJujuConnector._build_ee_id(
 989                 model_name=model_name,
 990                 application_name=application_name,
 991                 machine_id=str(machine.entity_id),
 992             )
 993
 994             # write ee_id in database
 995             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 996
 997             # wait for machine creation
 998             await observer.wait_for_machine(
 999                 machine_id=str(machine.entity_id),
1000                 progress_timeout=progress_timeout,
1001                 total_timeout=total_timeout,
1002             )
1003
1004         else:
1005
1006             self.log.debug("Reusing old machine pending")
1007
1008             # register machine with observer
1009             observer.register_machine(machine=machine, db_dict=db_dict)
1010
1011             # machine does exist, but it is in creation process (pending), wait for
1012             # create finalisation
1013             await observer.wait_for_machine(
1014                 machine_id=machine.entity_id,
1015                 progress_timeout=progress_timeout,
1016                 total_timeout=total_timeout,
1017             )
1018
1019         self.log.debug("Machine ready at " + str(machine.dns_name))
1020         return machine
1021
1022     async def _juju_provision_machine(
1023         self,
1024         model_name: str,
1025         hostname: str,
1026         username: str,
1027         private_key_path: str,
1028         db_dict: dict = None,
1029         progress_timeout: float = None,
1030         total_timeout: float = None,
1031     ) -> str:
1032
1033         if not self.api_proxy:
1034             msg = "Cannot provision machine: api_proxy is not defined"
1035             self.log.error(msg=msg)
1036             raise N2VCException(message=msg)
1037
1038         self.log.debug(
1039             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1040                 model_name, hostname, username
1041             )
1042         )
1043
1044         if not self._authenticated:
1045             await self._juju_login()
1046
1047         # get juju model and observer
1048         model = await self._juju_get_model(model_name=model_name)
1049         observer = self.juju_observers[model_name]
1050
1051         # TODO check if machine is already provisioned
1052         machine_list = await model.get_machines()
1053
1054         provisioner = AsyncSSHProvisioner(
1055             host=hostname,
1056             user=username,
1057             private_key_path=private_key_path,
1058             log=self.log,
1059         )
1060
1061         params = None
1062         try:
1063             params = await provisioner.provision_machine()
1064         except Exception as ex:
1065             msg = "Exception provisioning machine: {}".format(ex)
1066             self.log.error(msg)
1067             raise N2VCException(message=msg)
1068
1069         params.jobs = ["JobHostUnits"]
1070
1071         connection = model.connection()
1072
1073         # Submit the request.
1074         self.log.debug("Adding machine to model")
1075         client_facade = client.ClientFacade.from_connection(connection)
1076         results = await client_facade.AddMachines(params=[params])
1077         error = results.machines[0].error
1078         if error:
1079             msg = "Error adding machine: {}".format(error.message)
1080             self.log.error(msg=msg)
1081             raise ValueError(msg)
1082
1083         machine_id = results.machines[0].machine
1084
1085         # Need to run this after AddMachines has been called,
1086         # as we need the machine_id
1087         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1088         asyncio.ensure_future(
1089             provisioner.install_agent(
1090                 connection=connection,
1091                 nonce=params.nonce,
1092                 machine_id=machine_id,
1093                 api=self.api_proxy,
1094             )
1095         )
1096
1097         # wait for machine in model (now, machine is not yet in model, so we must
1098         # wait for it)
1099         machine = None
1100         for _ in range(10):
1101             machine_list = await model.get_machines()
1102             if machine_id in machine_list:
1103                 self.log.debug("Machine {} found in model!".format(machine_id))
1104                 machine = model.machines.get(machine_id)
1105                 break
1106             await asyncio.sleep(2)
1107
1108         if machine is None:
1109             msg = "Machine {} not found in model".format(machine_id)
1110             self.log.error(msg=msg)
1111             raise Exception(msg)
1112
1113         # register machine with observer
1114         observer.register_machine(machine=machine, db_dict=db_dict)
1115
1116         # wait for machine creation
1117         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1118         await observer.wait_for_machine(
1119             machine_id=machine_id,
1120             progress_timeout=progress_timeout,
1121             total_timeout=total_timeout,
1122         )
1123
1124         self.log.debug("Machine provisioned {}".format(machine_id))
1125
1126         return machine_id
1127
1128     async def _juju_deploy_charm(
1129         self,
1130         model_name: str,
1131         application_name: str,
1132         charm_path: str,
1133         machine_id: str,
1134         db_dict: dict,
1135         progress_timeout: float = None,
1136         total_timeout: float = None,
1137         config: dict = None,
1138     ) -> (Application, int):
1139
1140         # get juju model and observer
1141         model = await self._juju_get_model(model_name=model_name)
1142         observer = self.juju_observers[model_name]
1143
1144         # check if application already exists
1145         application = None
1146         if application_name in model.applications:
1147             application = model.applications[application_name]
1148
1149         if application is None:
1150
1151             # application does not exist, create it and wait for it
1152             self.log.debug(
1153                 "deploying application {} to machine {}, model {}".format(
1154                     application_name, machine_id, model_name
1155                 )
1156             )
1157             self.log.debug("charm: {}".format(charm_path))
1158             machine = model.machines[machine_id]
1159             # series = None
1160             application = await model.deploy(
1161                 entity_url=charm_path,
1162                 application_name=application_name,
1163                 channel="stable",
1164                 num_units=1,
1165                 series=machine.series,
1166                 to=machine_id,
1167                 config=config,
1168             )
1169
1170             # register application with observer
1171             observer.register_application(application=application, db_dict=db_dict)
1172
1173             self.log.debug(
1174                 "waiting for application deployed... {}".format(application.entity_id)
1175             )
1176             retries = await observer.wait_for_application(
1177                 application_id=application.entity_id,
1178                 progress_timeout=progress_timeout,
1179                 total_timeout=total_timeout,
1180             )
1181             self.log.debug("application deployed")
1182
1183         else:
1184
1185             # register application with observer
1186             observer.register_application(application=application, db_dict=db_dict)
1187
1188             # application already exists, but not finalised
1189             self.log.debug("application already exists, waiting for deployed...")
1190             retries = await observer.wait_for_application(
1191                 application_id=application.entity_id,
1192                 progress_timeout=progress_timeout,
1193                 total_timeout=total_timeout,
1194             )
1195             self.log.debug("application deployed")
1196
1197         return application, retries
1198
1199     async def _juju_execute_action(
1200         self,
1201         model_name: str,
1202         application_name: str,
1203         action_name: str,
1204         db_dict: dict,
1205         progress_timeout: float = None,
1206         total_timeout: float = None,
1207         **kwargs
1208     ) -> Action:
1209
1210         # get juju model and observer
1211         model = await self._juju_get_model(model_name=model_name)
1212         observer = self.juju_observers[model_name]
1213
1214         application = await self._juju_get_application(
1215             model_name=model_name, application_name=application_name
1216         )
1217
1218         unit = None
1219         for u in application.units:
1220             if await u.is_leader_from_status():
1221                 unit = u
1222         if unit is not None:
1223             actions = await application.get_actions()
1224             if action_name in actions:
1225                 self.log.debug(
1226                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1227                 )
1228                 action = await unit.run_action(action_name, **kwargs)
1229
1230                 # register action with observer
1231                 observer.register_action(action=action, db_dict=db_dict)
1232
1233                 await observer.wait_for_action(
1234                     action_id=action.entity_id,
1235                     progress_timeout=progress_timeout,
1236                     total_timeout=total_timeout,
1237                 )
1238                 self.log.debug("action completed with status: {}".format(action.status))
1239                 output = await model.get_action_output(action_uuid=action.entity_id)
1240                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1241                 if action.entity_id in status:
1242                     status = status[action.entity_id]
1243                 else:
1244                     status = "failed"
1245                 return output, status
1246
1247         raise N2VCExecutionException(
1248             message="Cannot execute action on charm", primitive_name=action_name
1249         )
1250
1251     async def _juju_configure_application(
1252         self,
1253         model_name: str,
1254         application_name: str,
1255         config: dict,
1256         db_dict: dict,
1257         progress_timeout: float = None,
1258         total_timeout: float = None,
1259     ):
1260
1261         # get the application
1262         application = await self._juju_get_application(
1263             model_name=model_name, application_name=application_name
1264         )
1265
1266         self.log.debug(
1267             "configuring the application {} -> {}".format(application_name, config)
1268         )
1269         res = await application.set_config(config)
1270         self.log.debug(
1271             "application {} configured. res={}".format(application_name, res)
1272         )
1273
1274         # Verify the config is set
1275         new_conf = await application.get_config()
1276         for key in config:
1277             value = new_conf[key]["value"]
1278             self.log.debug("    {} = {}".format(key, value))
1279             if config[key] != value:
1280                 raise N2VCException(
1281                     message="key {} is not configured correctly {} != {}".format(
1282                         key, config[key], new_conf[key]
1283                     )
1284                 )
1285
1286         # check if 'verify-ssh-credentials' action exists
1287         # unit = application.units[0]
1288         actions = await application.get_actions()
1289         if "verify-ssh-credentials" not in actions:
1290             msg = (
1291                 "Action verify-ssh-credentials does not exist in application {}"
1292             ).format(application_name)
1293             self.log.debug(msg=msg)
1294             return False
1295
1296         # execute verify-credentials
1297         num_retries = 20
1298         retry_timeout = 15.0
1299         for _ in range(num_retries):
1300             try:
1301                 self.log.debug("Executing action verify-ssh-credentials...")
1302                 output, ok = await self._juju_execute_action(
1303                     model_name=model_name,
1304                     application_name=application_name,
1305                     action_name="verify-ssh-credentials",
1306                     db_dict=db_dict,
1307                     progress_timeout=progress_timeout,
1308                     total_timeout=total_timeout,
1309                 )
1310                 self.log.debug("Result: {}, output: {}".format(ok, output))
1311                 return True
1312             except asyncio.CancelledError:
1313                 raise
1314             except Exception as e:
1315                 self.log.debug(
1316                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1317                 )
1318                 await asyncio.sleep(retry_timeout)
1319         else:
1320             self.log.error(
1321                 "Error executing verify-ssh-credentials after {} retries. ".format(
1322                     num_retries
1323                 )
1324             )
1325             return False
1326
1327     async def _juju_get_application(self, model_name: str, application_name: str):
1328         """Get the deployed application."""
1329
1330         model = await self._juju_get_model(model_name=model_name)
1331
1332         application_name = N2VCJujuConnector._format_app_name(application_name)
1333
1334         if model.applications and application_name in model.applications:
1335             return model.applications[application_name]
1336         else:
1337             raise N2VCException(
1338                 message="Cannot get application {} from model {}".format(
1339                     application_name, model_name
1340                 )
1341             )
1342
1343     async def _juju_get_model(self, model_name: str) -> Model:
1344         """ Get a model object from juju controller
1345         If the model does not exits, it creates it.
1346
1347         :param str model_name: name of the model
1348         :returns Model: model obtained from juju controller or Exception
1349         """
1350
1351         # format model name
1352         model_name = N2VCJujuConnector._format_model_name(model_name)
1353
1354         if model_name in self.juju_models:
1355             return self.juju_models[model_name]
1356
1357         if self._creating_model:
1358             self.log.debug("Another coroutine is creating a model. Wait...")
1359         while self._creating_model:
1360             # another coroutine is creating a model, wait
1361             await asyncio.sleep(0.1)
1362             # retry (perhaps another coroutine has created the model meanwhile)
1363             if model_name in self.juju_models:
1364                 return self.juju_models[model_name]
1365
1366         try:
1367             self._creating_model = True
1368
1369             # get juju model names from juju
1370             model_list = await self.controller.list_models()
1371
1372             if model_name not in model_list:
1373                 self.log.info(
1374                     "Model {} does not exist. Creating new model...".format(model_name)
1375                 )
1376                 config_dict = {"authorized-keys": self.public_key}
1377                 if self.apt_mirror:
1378                     config_dict["apt-mirror"] = self.apt_mirror
1379                 if not self.enable_os_upgrade:
1380                     config_dict["enable-os-refresh-update"] = False
1381                     config_dict["enable-os-upgrade"] = False
1382                 if self.cloud in self.BUILT_IN_CLOUDS:
1383                     model = await self.controller.add_model(
1384                         model_name=model_name,
1385                         config=config_dict,
1386                         cloud_name=self.cloud,
1387                     )
1388                 else:
1389                     model = await self.controller.add_model(
1390                         model_name=model_name,
1391                         config=config_dict,
1392                         cloud_name=self.cloud,
1393                         credential_name=self.cloud,
1394                     )
1395                 self.log.info("New model created, name={}".format(model_name))
1396             else:
1397                 self.log.debug(
1398                     "Model already exists in juju. Getting model {}".format(model_name)
1399                 )
1400                 model = await self.controller.get_model(model_name)
1401                 self.log.debug("Existing model in juju, name={}".format(model_name))
1402
1403             self.juju_models[model_name] = model
1404             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1405             return model
1406
1407         except Exception as e:
1408             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1409             self.log.error(msg)
1410             raise N2VCException(msg)
1411         finally:
1412             self._creating_model = False
1413
1414     async def _juju_add_relation(
1415         self,
1416         model_name: str,
1417         application_name_1: str,
1418         application_name_2: str,
1419         relation_1: str,
1420         relation_2: str,
1421     ):
1422
1423         # get juju model and observer
1424         model = await self._juju_get_model(model_name=model_name)
1425
1426         r1 = "{}:{}".format(application_name_1, relation_1)
1427         r2 = "{}:{}".format(application_name_2, relation_2)
1428
1429         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1430         try:
1431             await model.add_relation(relation1=r1, relation2=r2)
1432         except JujuAPIError as e:
1433             # If one of the applications in the relationship doesn't exist, or the
1434             # relation has already been added,
1435             # let the operation fail silently.
1436             if "not found" in e.message:
1437                 return
1438             if "already exists" in e.message:
1439                 return
1440             # another execption, raise it
1441             raise e
1442
1443     async def _juju_destroy_application(self, model_name: str, application_name: str):
1444
1445         self.log.debug(
1446             "Destroying application {} in model {}".format(application_name, model_name)
1447         )
1448
1449         # get juju model and observer
1450         model = await self._juju_get_model(model_name=model_name)
1451         observer = self.juju_observers[model_name]
1452
1453         application = model.applications.get(application_name)
1454         if application:
1455             observer.unregister_application(application_name)
1456             await application.destroy()
1457         else:
1458             self.log.debug("Application not found: {}".format(application_name))
1459
1460     async def _juju_destroy_machine(
1461         self, model_name: str, machine_id: str, total_timeout: float = None
1462     ):
1463
1464         self.log.debug(
1465             "Destroying machine {} in model {}".format(machine_id, model_name)
1466         )
1467
1468         if total_timeout is None:
1469             total_timeout = 3600
1470
1471         # get juju model and observer
1472         model = await self._juju_get_model(model_name=model_name)
1473         observer = self.juju_observers[model_name]
1474
1475         machines = await model.get_machines()
1476         if machine_id in machines:
1477             machine = model.machines[machine_id]
1478             observer.unregister_machine(machine_id)
1479             # TODO: change this by machine.is_manual when this is upstreamed:
1480             # https://github.com/juju/python-libjuju/pull/396
1481             if "instance-id" in machine.safe_data and machine.safe_data[
1482                 "instance-id"
1483             ].startswith("manual:"):
1484                 self.log.debug("machine.destroy(force=True) started.")
1485                 await machine.destroy(force=True)
1486                 self.log.debug("machine.destroy(force=True) passed.")
1487                 # max timeout
1488                 end = time.time() + total_timeout
1489                 # wait for machine removal
1490                 machines = await model.get_machines()
1491                 while machine_id in machines and time.time() < end:
1492                     self.log.debug(
1493                         "Waiting for machine {} is destroyed".format(machine_id)
1494                     )
1495                     await asyncio.sleep(0.5)
1496                     machines = await model.get_machines()
1497                 self.log.debug("Machine destroyed: {}".format(machine_id))
1498         else:
1499             self.log.debug("Machine not found: {}".format(machine_id))
1500
1501     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1502
1503         self.log.debug("Destroying model {}".format(model_name))
1504
1505         if total_timeout is None:
1506             total_timeout = 3600
1507         end = time.time() + total_timeout
1508
1509         model = await self._juju_get_model(model_name=model_name)
1510
1511         if not model:
1512             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1513
1514         uuid = model.info.uuid
1515
1516         # destroy applications
1517         for application_name in model.applications:
1518             try:
1519                 await self._juju_destroy_application(
1520                     model_name=model_name, application_name=application_name
1521                 )
1522             except Exception as e:
1523                 self.log.error(
1524                     "Error destroying application {} in model {}: {}".format(
1525                         application_name, model_name, e
1526                     )
1527                 )
1528
1529         # destroy machines
1530         machines = await model.get_machines()
1531         for machine_id in machines:
1532             try:
1533                 await self._juju_destroy_machine(
1534                     model_name=model_name, machine_id=machine_id
1535                 )
1536             except asyncio.CancelledError:
1537                 raise
1538             except Exception:
1539                 # ignore exceptions destroying machine
1540                 pass
1541
1542         await self._juju_disconnect_model(model_name=model_name)
1543
1544         self.log.debug("destroying model {}...".format(model_name))
1545         await self.controller.destroy_model(uuid)
1546         # self.log.debug('model destroy requested {}'.format(model_name))
1547
1548         # wait for model is completely destroyed
1549         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1550         last_exception = ""
1551         while time.time() < end:
1552             try:
1553                 # await self.controller.get_model(uuid)
1554                 models = await self.controller.list_models()
1555                 if model_name not in models:
1556                     self.log.debug(
1557                         "The model {} ({}) was destroyed".format(model_name, uuid)
1558                     )
1559                     return
1560             except asyncio.CancelledError:
1561                 raise
1562             except Exception as e:
1563                 last_exception = e
1564             await asyncio.sleep(5)
1565         raise N2VCException(
1566             "Timeout waiting for model {} to be destroyed {}".format(
1567                 model_name, last_exception
1568             )
1569         )
1570
1571     async def _juju_login(self):
1572         """Connect to juju controller
1573
1574         """
1575
1576         # if already authenticated, exit function
1577         if self._authenticated:
1578             return
1579
1580         # if connecting, wait for finish
1581         # another task could be trying to connect in parallel
1582         while self._connecting:
1583             await asyncio.sleep(0.1)
1584
1585         # double check after other task has finished
1586         if self._authenticated:
1587             return
1588
1589         try:
1590             self._connecting = True
1591             self.log.info(
1592                 "connecting to juju controller: {} {}:{}{}".format(
1593                     self.url,
1594                     self.username,
1595                     self.secret[:8] + "...",
1596                     " with ca_cert" if self.ca_cert else "",
1597                 )
1598             )
1599
1600             # Create controller object
1601             self.controller = Controller(loop=self.loop)
1602             # Connect to controller
1603             await self.controller.connect(
1604                 endpoint=self.url,
1605                 username=self.username,
1606                 password=self.secret,
1607                 cacert=self.ca_cert,
1608             )
1609             self._authenticated = True
1610             self.log.info("juju controller connected")
1611         except Exception as e:
1612             message = "Exception connecting to juju: {}".format(e)
1613             self.log.error(message)
1614             raise N2VCConnectionException(message=message, url=self.url)
1615         finally:
1616             self._connecting = False
1617
1618     async def _juju_logout(self):
1619         """Logout of the Juju controller."""
1620         if not self._authenticated:
1621             return False
1622
1623         # disconnect all models
1624         for model_name in self.juju_models:
1625             try:
1626                 await self._juju_disconnect_model(model_name)
1627             except Exception as e:
1628                 self.log.error(
1629                     "Error disconnecting model {} : {}".format(model_name, e)
1630                 )
1631                 # continue with next model...
1632
1633         self.log.info("Disconnecting controller")
1634         try:
1635             await self.controller.disconnect()
1636         except Exception as e:
1637             raise N2VCConnectionException(
1638                 message="Error disconnecting controller: {}".format(e), url=self.url
1639             )
1640
1641         self.controller = None
1642         self._authenticated = False
1643         self.log.info("disconnected")
1644
1645     async def _juju_disconnect_model(self, model_name: str):
1646         self.log.debug("Disconnecting model {}".format(model_name))
1647         if model_name in self.juju_models:
1648             await self.juju_models[model_name].disconnect()
1649             self.juju_models[model_name] = None
1650             self.juju_observers[model_name] = None
1651         else:
1652             self.warning("Cannot disconnect model: {}".format(model_name))
1653
1654     def _create_juju_public_key(self):
1655         """Recreate the Juju public key on lcm container, if needed
1656         Certain libjuju commands expect to be run from the same machine as Juju
1657          is bootstrapped to. This method will write the public key to disk in
1658          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1659         """
1660
1661         # Make sure that we have a public key before writing to disk
1662         if self.public_key is None or len(self.public_key) == 0:
1663             if "OSMLCM_VCA_PUBKEY" in os.environ:
1664                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1665                 if len(self.public_key) == 0:
1666                     return
1667             else:
1668                 return
1669
1670         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1671         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1672         self.log.debug(
1673             "writing juju public key to file:\n{}\npublic key: {}".format(
1674                 file_path, self.public_key
1675             )
1676         )
1677         if not os.path.exists(pk_path):
1678             # create path and write file
1679             os.makedirs(pk_path)
1680             with open(file_path, "w") as f:
1681                 self.log.debug("Creating juju public key file: {}".format(file_path))
1682                 f.write(self.public_key)
1683         else:
1684             self.log.debug("juju public key file already exists: {}".format(file_path))
1685
1686     @staticmethod
1687     def _format_model_name(name: str) -> str:
1688         """Format the name of the model.
1689
1690         Model names may only contain lowercase letters, digits and hyphens
1691         """
1692
1693         return name.replace("_", "-").replace(" ", "-").lower()
1694
1695     @staticmethod
1696     def _format_app_name(name: str) -> str:
1697         """Format the name of the application (in order to assure valid application name).
1698
1699         Application names have restrictions (run juju deploy --help):
1700             - contains lowercase letters 'a'-'z'
1701             - contains numbers '0'-'9'
1702             - contains hyphens '-'
1703             - starts with a lowercase letter
1704             - not two or more consecutive hyphens
1705             - after a hyphen, not a group with all numbers
1706         """
1707
1708         def all_numbers(s: str) -> bool:
1709             for c in s:
1710                 if not c.isdigit():
1711                     return False
1712             return True
1713
1714         new_name = name.replace("_", "-")
1715         new_name = new_name.replace(" ", "-")
1716         new_name = new_name.lower()
1717         while new_name.find("--") >= 0:
1718             new_name = new_name.replace("--", "-")
1719         groups = new_name.split("-")
1720
1721         # find 'all numbers' groups and prefix them with a letter
1722         app_name = ""
1723         for i in range(len(groups)):
1724             group = groups[i]
1725             if all_numbers(group):
1726                 group = "z" + group
1727             if i > 0:
1728                 app_name += "-"
1729             app_name += group
1730
1731         if app_name[0].isdigit():
1732             app_name = "z" + app_name
1733
1734         return app_name