n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46     JujuK8sProxycharmNotSupported,
  47 )
  48 from n2vc.juju_observer import JujuModelObserver
  49 from n2vc.n2vc_conn import N2VCConnector
  50 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  51 from n2vc.provisioner import AsyncSSHProvisioner
  52 from n2vc.libjuju import Libjuju
  53
  54
  55 class N2VCJujuConnector(N2VCConnector):
  56
  57     """
  58 ####################################################################################
  59 ################################### P U B L I C ####################################
  60 ####################################################################################
  61     """
  62
  63     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  64
  65     def __init__(
  66         self,
  67         db: object,
  68         fs: object,
  69         log: object = None,
  70         loop: object = None,
  71         url: str = "127.0.0.1:17070",
  72         username: str = "admin",
  73         vca_config: dict = None,
  74         on_update_db=None,
  75     ):
  76         """Initialize juju N2VC connector
  77         """
  78
  79         # parent class constructor
  80         N2VCConnector.__init__(
  81             self,
  82             db=db,
  83             fs=fs,
  84             log=log,
  85             loop=loop,
  86             url=url,
  87             username=username,
  88             vca_config=vca_config,
  89             on_update_db=on_update_db,
  90         )
  91
  92         # silence websocket traffic log
  93         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  94         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  95         logging.getLogger("model").setLevel(logging.WARN)
  96
  97         self.log.info("Initializing N2VC juju connector...")
  98
  99         """
 100         ##############################################################
 101         # check arguments
 102         ##############################################################
 103         """
 104
 105         # juju URL
 106         if url is None:
 107             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 108         url_parts = url.split(":")
 109         if len(url_parts) != 2:
 110             raise N2VCBadArgumentsException(
 111                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 112             )
 113         self.hostname = url_parts[0]
 114         try:
 115             self.port = int(url_parts[1])
 116         except ValueError:
 117             raise N2VCBadArgumentsException(
 118                 "url port must be a number -> {}".format(url), ["url"]
 119             )
 120
 121         # juju USERNAME
 122         if username is None:
 123             raise N2VCBadArgumentsException(
 124                 "Argument username is mandatory", ["username"]
 125             )
 126
 127         # juju CONFIGURATION
 128         if vca_config is None:
 129             raise N2VCBadArgumentsException(
 130                 "Argument vca_config is mandatory", ["vca_config"]
 131             )
 132
 133         if "secret" in vca_config:
 134             self.secret = vca_config["secret"]
 135         else:
 136             raise N2VCBadArgumentsException(
 137                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 138             )
 139
 140         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 141         # if exists, it will be written in lcm container: _create_juju_public_key()
 142         if "public_key" in vca_config:
 143             self.public_key = vca_config["public_key"]
 144         else:
 145             self.public_key = None
 146
 147         # TODO: Verify ca_cert is valid before using. VCA will crash
 148         # if the ca_cert isn't formatted correctly.
 149         def base64_to_cacert(b64string):
 150             """Convert the base64-encoded string containing the VCA CACERT.
 151
 152             The input string....
 153
 154             """
 155             try:
 156                 cacert = base64.b64decode(b64string).decode("utf-8")
 157
 158                 cacert = re.sub(r"\\n", r"\n", cacert,)
 159             except binascii.Error as e:
 160                 self.log.debug("Caught binascii.Error: {}".format(e))
 161                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 162
 163             return cacert
 164
 165         self.ca_cert = vca_config.get("ca_cert")
 166         if self.ca_cert:
 167             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 168
 169         if "api_proxy" in vca_config:
 170             self.api_proxy = vca_config["api_proxy"]
 171             self.log.debug(
 172                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 173             )
 174         else:
 175             self.warning(
 176                 "api_proxy is not configured. Support for native charms is disabled"
 177             )
 178
 179         if "enable_os_upgrade" in vca_config:
 180             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 181         else:
 182             self.enable_os_upgrade = True
 183
 184         if "apt_mirror" in vca_config:
 185             self.apt_mirror = vca_config["apt_mirror"]
 186         else:
 187             self.apt_mirror = None
 188
 189         self.cloud = vca_config.get('cloud')
 190         self.k8s_cloud = None
 191         if "k8s_cloud" in vca_config:
 192             self.k8s_cloud = vca_config.get("k8s_cloud")
 193         self.log.debug('Arguments have been checked')
 194
 195         # juju data
 196         self.controller = None  # it will be filled when connect to juju
 197         self.juju_models = {}  # model objects for every model_name
 198         self.juju_observers = {}  # model observers for every model_name
 199         self._connecting = (
 200             False  # while connecting to juju (to avoid duplicate connections)
 201         )
 202         self._authenticated = (
 203             False  # it will be True when juju connection be stablished
 204         )
 205         self._creating_model = False  # True during model creation
 206         self.libjuju = Libjuju(
 207             endpoint=self.url,
 208             api_proxy=self.api_proxy,
 209             enable_os_upgrade=self.enable_os_upgrade,
 210             apt_mirror=self.apt_mirror,
 211             username=self.username,
 212             password=self.secret,
 213             cacert=self.ca_cert,
 214             loop=self.loop,
 215             log=self.log,
 216             db=self.db,
 217             n2vc=self,
 218         )
 219
 220         # create juju pub key file in lcm container at
 221         # ./local/share/juju/ssh/juju_id_rsa.pub
 222         self._create_juju_public_key()
 223
 224         self.log.info("N2VC juju connector initialized")
 225
 226     async def get_status(self, namespace: str, yaml_format: bool = True):
 227
 228         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 229
 230         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 231             namespace=namespace
 232         )
 233         # model name is ns_id
 234         model_name = ns_id
 235         if model_name is None:
 236             msg = "Namespace {} not valid".format(namespace)
 237             self.log.error(msg)
 238             raise N2VCBadArgumentsException(msg, ["namespace"])
 239
 240         status = {}
 241         models = await self.libjuju.list_models(contains=ns_id)
 242
 243         for m in models:
 244             status[m] = self.libjuju.get_model_status(m)
 245
 246         if yaml_format:
 247             return obj_to_yaml(status)
 248         else:
 249             return obj_to_dict(status)
 250
 251     async def create_execution_environment(
 252         self,
 253         namespace: str,
 254         db_dict: dict,
 255         reuse_ee_id: str = None,
 256         progress_timeout: float = None,
 257         total_timeout: float = None,
 258     ) -> (str, dict):
 259
 260         self.log.info(
 261             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 262                 namespace, reuse_ee_id
 263             )
 264         )
 265
 266         machine_id = None
 267         if reuse_ee_id:
 268             model_name, application_name, machine_id = self._get_ee_id_components(
 269                 ee_id=reuse_ee_id
 270             )
 271         else:
 272             (
 273                 _nsi_id,
 274                 ns_id,
 275                 _vnf_id,
 276                 _vdu_id,
 277                 _vdu_count,
 278             ) = self._get_namespace_components(namespace=namespace)
 279             # model name is ns_id
 280             model_name = ns_id
 281             # application name
 282             application_name = self._get_application_name(namespace=namespace)
 283
 284         self.log.debug(
 285             "model name: {}, application name:  {}, machine_id: {}".format(
 286                 model_name, application_name, machine_id
 287             )
 288         )
 289
 290         # create or reuse a new juju machine
 291         try:
 292             if not await self.libjuju.model_exists(model_name):
 293                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 294             machine, new = await self.libjuju.create_machine(
 295                 model_name=model_name,
 296                 machine_id=machine_id,
 297                 db_dict=db_dict,
 298                 progress_timeout=progress_timeout,
 299                 total_timeout=total_timeout,
 300             )
 301             # id for the execution environment
 302             ee_id = N2VCJujuConnector._build_ee_id(
 303                 model_name=model_name,
 304                 application_name=application_name,
 305                 machine_id=str(machine.entity_id),
 306             )
 307             self.log.debug("ee_id: {}".format(ee_id))
 308
 309             if new:
 310                 # write ee_id in database
 311                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 312
 313         except Exception as e:
 314             message = "Error creating machine on juju: {}".format(e)
 315             self.log.error(message)
 316             raise N2VCException(message=message)
 317
 318         # new machine credentials
 319         credentials = {
 320             "hostname": machine.dns_name,
 321         }
 322
 323         self.log.info(
 324             "Execution environment created. ee_id: {}, credentials: {}".format(
 325                 ee_id, credentials
 326             )
 327         )
 328
 329         return ee_id, credentials
 330
 331     async def register_execution_environment(
 332         self,
 333         namespace: str,
 334         credentials: dict,
 335         db_dict: dict,
 336         progress_timeout: float = None,
 337         total_timeout: float = None,
 338     ) -> str:
 339
 340         self.log.info(
 341             "Registering execution environment. namespace={}, credentials={}".format(
 342                 namespace, credentials
 343             )
 344         )
 345
 346         if credentials is None:
 347             raise N2VCBadArgumentsException(
 348                 message="credentials are mandatory", bad_args=["credentials"]
 349             )
 350         if credentials.get("hostname"):
 351             hostname = credentials["hostname"]
 352         else:
 353             raise N2VCBadArgumentsException(
 354                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 355             )
 356         if credentials.get("username"):
 357             username = credentials["username"]
 358         else:
 359             raise N2VCBadArgumentsException(
 360                 message="username is mandatory", bad_args=["credentials.username"]
 361             )
 362         if "private_key_path" in credentials:
 363             private_key_path = credentials["private_key_path"]
 364         else:
 365             # if not passed as argument, use generated private key path
 366             private_key_path = self.private_key_path
 367
 368         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 369             namespace=namespace
 370         )
 371
 372         # model name
 373         model_name = ns_id
 374         # application name
 375         application_name = self._get_application_name(namespace=namespace)
 376
 377         # register machine on juju
 378         try:
 379             if not self.api_proxy:
 380                 msg = "Cannot provision machine: api_proxy is not defined"
 381                 self.log.error(msg=msg)
 382                 raise N2VCException(message=msg)
 383             if not await self.libjuju.model_exists(model_name):
 384                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 385             machine_id = await self.libjuju.provision_machine(
 386                 model_name=model_name,
 387                 hostname=hostname,
 388                 username=username,
 389                 private_key_path=private_key_path,
 390                 db_dict=db_dict,
 391                 progress_timeout=progress_timeout,
 392                 total_timeout=total_timeout,
 393             )
 394         except Exception as e:
 395             self.log.error("Error registering machine: {}".format(e))
 396             raise N2VCException(
 397                 message="Error registering machine on juju: {}".format(e)
 398             )
 399
 400         self.log.info("Machine registered: {}".format(machine_id))
 401
 402         # id for the execution environment
 403         ee_id = N2VCJujuConnector._build_ee_id(
 404             model_name=model_name,
 405             application_name=application_name,
 406             machine_id=str(machine_id),
 407         )
 408
 409         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 410
 411         return ee_id
 412
 413     async def install_configuration_sw(
 414         self,
 415         ee_id: str,
 416         artifact_path: str,
 417         db_dict: dict,
 418         progress_timeout: float = None,
 419         total_timeout: float = None,
 420         config: dict = None,
 421         num_units: int = 1,
 422     ):
 423
 424         self.log.info(
 425             (
 426                 "Installing configuration sw on ee_id: {}, "
 427                 "artifact path: {}, db_dict: {}"
 428             ).format(ee_id, artifact_path, db_dict)
 429         )
 430
 431         # check arguments
 432         if ee_id is None or len(ee_id) == 0:
 433             raise N2VCBadArgumentsException(
 434                 message="ee_id is mandatory", bad_args=["ee_id"]
 435             )
 436         if artifact_path is None or len(artifact_path) == 0:
 437             raise N2VCBadArgumentsException(
 438                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 439             )
 440         if db_dict is None:
 441             raise N2VCBadArgumentsException(
 442                 message="db_dict is mandatory", bad_args=["db_dict"]
 443             )
 444
 445         try:
 446             (
 447                 model_name,
 448                 application_name,
 449                 machine_id,
 450             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 451             self.log.debug(
 452                 "model: {}, application: {}, machine: {}".format(
 453                     model_name, application_name, machine_id
 454                 )
 455             )
 456         except Exception:
 457             raise N2VCBadArgumentsException(
 458                 message="ee_id={} is not a valid execution environment id".format(
 459                     ee_id
 460                 ),
 461                 bad_args=["ee_id"],
 462             )
 463
 464         # remove // in charm path
 465         while artifact_path.find("//") >= 0:
 466             artifact_path = artifact_path.replace("//", "/")
 467
 468         # check charm path
 469         if not self.fs.file_exists(artifact_path, mode="dir"):
 470             msg = "artifact path does not exist: {}".format(artifact_path)
 471             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 472
 473         if artifact_path.startswith("/"):
 474             full_path = self.fs.path + artifact_path
 475         else:
 476             full_path = self.fs.path + "/" + artifact_path
 477
 478         try:
 479             await self.libjuju.deploy_charm(
 480                 model_name=model_name,
 481                 application_name=application_name,
 482                 path=full_path,
 483                 machine_id=machine_id,
 484                 db_dict=db_dict,
 485                 progress_timeout=progress_timeout,
 486                 total_timeout=total_timeout,
 487                 config=config,
 488                 num_units=num_units,
 489             )
 490         except Exception as e:
 491             raise N2VCException(
 492                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 493             )
 494
 495         self.log.info("Configuration sw installed")
 496
 497     async def install_k8s_proxy_charm(
 498         self,
 499         charm_name: str,
 500         namespace: str,
 501         artifact_path: str,
 502         db_dict: dict,
 503         progress_timeout: float = None,
 504         total_timeout: float = None,
 505         config: dict = None,
 506     ) -> str:
 507         """
 508         Install a k8s proxy charm
 509
 510         :param charm_name: Name of the charm being deployed
 511         :param namespace: collection of all the uuids related to the charm.
 512         :param str artifact_path: where to locate the artifacts (parent folder) using
 513             the self.fs
 514             the final artifact path will be a combination of this artifact_path and
 515             additional string from the config_dict (e.g. charm name)
 516         :param dict db_dict: where to write into database when the status changes.
 517                         It contains a dict with
 518                             {collection: <str>, filter: {},  path: <str>},
 519                             e.g. {collection: "nsrs", filter:
 520                                 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
 521         :param float progress_timeout:
 522         :param float total_timeout:
 523         :param config: Dictionary with additional configuration
 524
 525         :returns ee_id: execution environment id.
 526         """
 527         self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
 528                       .format(charm_name, artifact_path, db_dict))
 529
 530         if not self.k8s_cloud:
 531             raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
 532
 533         if artifact_path is None or len(artifact_path) == 0:
 534             raise N2VCBadArgumentsException(
 535                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 536             )
 537         if db_dict is None:
 538             raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict'])
 539
 540         # remove // in charm path
 541         while artifact_path.find('//') >= 0:
 542             artifact_path = artifact_path.replace('//', '/')
 543
 544         # check charm path
 545         if not self.fs.file_exists(artifact_path, mode="dir"):
 546             msg = 'artifact path does not exist: {}'.format(artifact_path)
 547             raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path'])
 548
 549         if artifact_path.startswith('/'):
 550             full_path = self.fs.path + artifact_path
 551         else:
 552             full_path = self.fs.path + '/' + artifact_path
 553
 554         _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace)
 555         model_name = '{}-k8s'.format(ns_id)
 556
 557         await self.libjuju.add_model(model_name, self.k8s_cloud)
 558         application_name = self._get_application_name(namespace)
 559
 560         try:
 561             await self.libjuju.deploy_charm(
 562                 model_name=model_name,
 563                 application_name=application_name,
 564                 path=full_path,
 565                 machine_id=None,
 566                 db_dict=db_dict,
 567                 progress_timeout=progress_timeout,
 568                 total_timeout=total_timeout,
 569                 config=config
 570             )
 571         except Exception as e:
 572             raise N2VCException(message='Error deploying charm: {}'.format(e))
 573
 574         self.log.info('K8s proxy charm installed')
 575         ee_id = N2VCJujuConnector._build_ee_id(
 576             model_name=model_name,
 577             application_name=application_name,
 578             machine_id="k8s",
 579         )
 580
 581         self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 582
 583         return ee_id
 584
 585     async def get_ee_ssh_public__key(
 586         self,
 587         ee_id: str,
 588         db_dict: dict,
 589         progress_timeout: float = None,
 590         total_timeout: float = None,
 591     ) -> str:
 592
 593         self.log.info(
 594             (
 595                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 596             ).format(ee_id, db_dict)
 597         )
 598
 599         # check arguments
 600         if ee_id is None or len(ee_id) == 0:
 601             raise N2VCBadArgumentsException(
 602                 message="ee_id is mandatory", bad_args=["ee_id"]
 603             )
 604         if db_dict is None:
 605             raise N2VCBadArgumentsException(
 606                 message="db_dict is mandatory", bad_args=["db_dict"]
 607             )
 608
 609         try:
 610             (
 611                 model_name,
 612                 application_name,
 613                 machine_id,
 614             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 615             self.log.debug(
 616                 "model: {}, application: {}, machine: {}".format(
 617                     model_name, application_name, machine_id
 618                 )
 619             )
 620         except Exception:
 621             raise N2VCBadArgumentsException(
 622                 message="ee_id={} is not a valid execution environment id".format(
 623                     ee_id
 624                 ),
 625                 bad_args=["ee_id"],
 626             )
 627
 628         # try to execute ssh layer primitives (if exist):
 629         #       generate-ssh-key
 630         #       get-ssh-public-key
 631
 632         output = None
 633
 634         application_name = N2VCJujuConnector._format_app_name(application_name)
 635
 636         # execute action: generate-ssh-key
 637         try:
 638             output, _status = await self.libjuju.execute_action(
 639                 model_name=model_name,
 640                 application_name=application_name,
 641                 action_name="generate-ssh-key",
 642                 db_dict=db_dict,
 643                 progress_timeout=progress_timeout,
 644                 total_timeout=total_timeout,
 645             )
 646         except Exception as e:
 647             self.log.info(
 648                 "Skipping exception while executing action generate-ssh-key: {}".format(
 649                     e
 650                 )
 651             )
 652
 653         # execute action: get-ssh-public-key
 654         try:
 655             output, _status = await self.libjuju.execute_action(
 656                 model_name=model_name,
 657                 application_name=application_name,
 658                 action_name="get-ssh-public-key",
 659                 db_dict=db_dict,
 660                 progress_timeout=progress_timeout,
 661                 total_timeout=total_timeout,
 662             )
 663         except Exception as e:
 664             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 665             self.log.info(msg)
 666             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 667
 668         # return public key if exists
 669         return output["pubkey"] if "pubkey" in output else output
 670
 671     async def add_relation(
 672         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 673     ):
 674
 675         self.log.debug(
 676             "adding new relation between {} and {}, endpoints: {}, {}".format(
 677                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 678             )
 679         )
 680
 681         # check arguments
 682         if not ee_id_1:
 683             message = "EE 1 is mandatory"
 684             self.log.error(message)
 685             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 686         if not ee_id_2:
 687             message = "EE 2 is mandatory"
 688             self.log.error(message)
 689             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 690         if not endpoint_1:
 691             message = "endpoint 1 is mandatory"
 692             self.log.error(message)
 693             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 694         if not endpoint_2:
 695             message = "endpoint 2 is mandatory"
 696             self.log.error(message)
 697             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 698
 699         # get the model, the applications and the machines from the ee_id's
 700         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 701         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 702
 703         # model must be the same
 704         if model_1 != model_2:
 705             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 706             self.log.error(message)
 707             raise N2VCBadArgumentsException(
 708                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 709             )
 710
 711         # add juju relations between two applications
 712         try:
 713             await self.libjuju.add_relation(
 714                 model_name=model_1,
 715                 application_name_1=app_1,
 716                 application_name_2=app_2,
 717                 relation_1=endpoint_1,
 718                 relation_2=endpoint_2,
 719             )
 720         except Exception as e:
 721             message = "Error adding relation between {} and {}: {}".format(
 722                 ee_id_1, ee_id_2, e
 723             )
 724             self.log.error(message)
 725             raise N2VCException(message=message)
 726
 727     async def remove_relation(self):
 728         # TODO
 729         self.log.info("Method not implemented yet")
 730         raise MethodNotImplemented()
 731
 732     async def deregister_execution_environments(self):
 733         self.log.info("Method not implemented yet")
 734         raise MethodNotImplemented()
 735
 736     async def delete_namespace(
 737         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 738     ):
 739         self.log.info("Deleting namespace={}".format(namespace))
 740
 741         # check arguments
 742         if namespace is None:
 743             raise N2VCBadArgumentsException(
 744                 message="namespace is mandatory", bad_args=["namespace"]
 745             )
 746
 747         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 748             namespace=namespace
 749         )
 750         if ns_id is not None:
 751             try:
 752                 models = await self.libjuju.list_models(contains=ns_id)
 753                 for model in models:
 754                     await self.libjuju.destroy_model(
 755                         model_name=model, total_timeout=total_timeout
 756                     )
 757             except Exception as e:
 758                 raise N2VCException(
 759                     message="Error deleting namespace {} : {}".format(namespace, e)
 760                 )
 761         else:
 762             raise N2VCBadArgumentsException(
 763                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 764             )
 765
 766         self.log.info("Namespace {} deleted".format(namespace))
 767
 768     async def delete_execution_environment(
 769         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 770     ):
 771         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 772
 773         # check arguments
 774         if ee_id is None:
 775             raise N2VCBadArgumentsException(
 776                 message="ee_id is mandatory", bad_args=["ee_id"]
 777             )
 778
 779         model_name, application_name, _machine_id = self._get_ee_id_components(
 780             ee_id=ee_id
 781         )
 782
 783         # destroy the application
 784         try:
 785             await self.libjuju.destroy_model(
 786                 model_name=model_name, total_timeout=total_timeout
 787             )
 788         except Exception as e:
 789             raise N2VCException(
 790                 message=(
 791                     "Error deleting execution environment {} (application {}) : {}"
 792                 ).format(ee_id, application_name, e)
 793             )
 794
 795         # destroy the machine
 796         # try:
 797         #     await self._juju_destroy_machine(
 798         #         model_name=model_name,
 799         #         machine_id=machine_id,
 800         #         total_timeout=total_timeout
 801         #     )
 802         # except Exception as e:
 803         #     raise N2VCException(
 804         #        message='Error deleting execution environment {} (machine {}) : {}'
 805         #                .format(ee_id, machine_id, e))
 806
 807         self.log.info("Execution environment {} deleted".format(ee_id))
 808
 809     async def exec_primitive(
 810         self,
 811         ee_id: str,
 812         primitive_name: str,
 813         params_dict: dict,
 814         db_dict: dict = None,
 815         progress_timeout: float = None,
 816         total_timeout: float = None,
 817     ) -> str:
 818
 819         self.log.info(
 820             "Executing primitive: {} on ee: {}, params: {}".format(
 821                 primitive_name, ee_id, params_dict
 822             )
 823         )
 824
 825         # check arguments
 826         if ee_id is None or len(ee_id) == 0:
 827             raise N2VCBadArgumentsException(
 828                 message="ee_id is mandatory", bad_args=["ee_id"]
 829             )
 830         if primitive_name is None or len(primitive_name) == 0:
 831             raise N2VCBadArgumentsException(
 832                 message="action_name is mandatory", bad_args=["action_name"]
 833             )
 834         if params_dict is None:
 835             params_dict = dict()
 836
 837         try:
 838             (
 839                 model_name,
 840                 application_name,
 841                 _machine_id,
 842             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 843         except Exception:
 844             raise N2VCBadArgumentsException(
 845                 message="ee_id={} is not a valid execution environment id".format(
 846                     ee_id
 847                 ),
 848                 bad_args=["ee_id"],
 849             )
 850
 851         if primitive_name == "config":
 852             # Special case: config primitive
 853             try:
 854                 await self.libjuju.configure_application(
 855                     model_name=model_name,
 856                     application_name=application_name,
 857                     config=params_dict,
 858                 )
 859                 actions = await self.libjuju.get_actions(
 860                     application_name=application_name, model_name=model_name,
 861                 )
 862                 self.log.debug(
 863                     "Application {} has these actions: {}".format(
 864                         application_name, actions
 865                     )
 866                 )
 867                 if "verify-ssh-credentials" in actions:
 868                     # execute verify-credentials
 869                     num_retries = 20
 870                     retry_timeout = 15.0
 871                     for _ in range(num_retries):
 872                         try:
 873                             self.log.debug("Executing action verify-ssh-credentials...")
 874                             output, ok = await self.libjuju.execute_action(
 875                                 model_name=model_name,
 876                                 application_name=application_name,
 877                                 action_name="verify-ssh-credentials",
 878                                 db_dict=db_dict,
 879                                 progress_timeout=progress_timeout,
 880                                 total_timeout=total_timeout,
 881                             )
 882
 883                             if ok == "failed":
 884                                 self.log.debug(
 885                                     "Error executing verify-ssh-credentials: {}. Retrying..."
 886                                 )
 887                                 await asyncio.sleep(retry_timeout)
 888
 889                                 continue
 890                             self.log.debug("Result: {}, output: {}".format(ok, output))
 891                             break
 892                         except asyncio.CancelledError:
 893                             raise
 894                     else:
 895                         self.log.error(
 896                             "Error executing verify-ssh-credentials after {} retries. ".format(
 897                                 num_retries
 898                             )
 899                         )
 900                 else:
 901                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 902                         application_name
 903                     )
 904                     self.log.debug(msg=msg)
 905             except Exception as e:
 906                 self.log.error("Error configuring juju application: {}".format(e))
 907                 raise N2VCExecutionException(
 908                     message="Error configuring application into ee={} : {}".format(
 909                         ee_id, e
 910                     ),
 911                     primitive_name=primitive_name,
 912                 )
 913             return "CONFIG OK"
 914         else:
 915             try:
 916                 output, status = await self.libjuju.execute_action(
 917                     model_name=model_name,
 918                     application_name=application_name,
 919                     action_name=primitive_name,
 920                     db_dict=db_dict,
 921                     progress_timeout=progress_timeout,
 922                     total_timeout=total_timeout,
 923                     **params_dict
 924                 )
 925                 if status == "completed":
 926                     return output
 927                 else:
 928                     raise Exception("status is not completed: {}".format(status))
 929             except Exception as e:
 930                 self.log.error(
 931                     "Error executing primitive {}: {}".format(primitive_name, e)
 932                 )
 933                 raise N2VCExecutionException(
 934                     message="Error executing primitive {} into ee={} : {}".format(
 935                         primitive_name, ee_id, e
 936                     ),
 937                     primitive_name=primitive_name,
 938                 )
 939
 940     async def disconnect(self):
 941         self.log.info("closing juju N2VC...")
 942         try:
 943             await self.libjuju.disconnect()
 944         except Exception as e:
 945             raise N2VCConnectionException(
 946                 message="Error disconnecting controller: {}".format(e), url=self.url
 947             )
 948
 949     """
 950 ####################################################################################
 951 ################################### P R I V A T E ##################################
 952 ####################################################################################
 953     """
 954
 955     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 956
 957         # write ee_id to database: _admin.deployed.VCA.x
 958         try:
 959             the_table = db_dict["collection"]
 960             the_filter = db_dict["filter"]
 961             the_path = db_dict["path"]
 962             if not the_path[-1] == ".":
 963                 the_path = the_path + "."
 964             update_dict = {the_path + "ee_id": ee_id}
 965             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 966             self.db.set_one(
 967                 table=the_table,
 968                 q_filter=the_filter,
 969                 update_dict=update_dict,
 970                 fail_on_empty=True,
 971             )
 972         except asyncio.CancelledError:
 973             raise
 974         except Exception as e:
 975             self.log.error("Error writing ee_id to database: {}".format(e))
 976
 977     @staticmethod
 978     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 979         """
 980         Build an execution environment id form model, application and machine
 981         :param model_name:
 982         :param application_name:
 983         :param machine_id:
 984         :return:
 985         """
 986         # id for the execution environment
 987         return "{}.{}.{}".format(model_name, application_name, machine_id)
 988
 989     @staticmethod
 990     def _get_ee_id_components(ee_id: str) -> (str, str, str):
 991         """
 992         Get model, application and machine components from an execution environment id
 993         :param ee_id:
 994         :return: model_name, application_name, machine_id
 995         """
 996
 997         if ee_id is None:
 998             return None, None, None
 999
1000         # split components of id
1001         parts = ee_id.split(".")
1002         model_name = parts[0]
1003         application_name = parts[1]
1004         machine_id = parts[2]
1005         return model_name, application_name, machine_id
1006
1007     def _get_application_name(self, namespace: str) -> str:
1008         """
1009         Build application name from namespace
1010         :param namespace:
1011         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1012         """
1013
1014         # TODO: Enforce the Juju 50-character application limit
1015
1016         # split namespace components
1017         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
1018             namespace=namespace
1019         )
1020
1021         if vnf_id is None or len(vnf_id) == 0:
1022             vnf_id = ""
1023         else:
1024             # Shorten the vnf_id to its last twelve characters
1025             vnf_id = "vnf-" + vnf_id[-12:]
1026
1027         if vdu_id is None or len(vdu_id) == 0:
1028             vdu_id = ""
1029         else:
1030             # Shorten the vdu_id to its last twelve characters
1031             vdu_id = "-vdu-" + vdu_id[-12:]
1032
1033         if vdu_count is None or len(vdu_count) == 0:
1034             vdu_count = ""
1035         else:
1036             vdu_count = "-cnt-" + vdu_count
1037
1038         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
1039
1040         return N2VCJujuConnector._format_app_name(application_name)
1041
1042     async def _juju_create_machine(
1043         self,
1044         model_name: str,
1045         application_name: str,
1046         machine_id: str = None,
1047         db_dict: dict = None,
1048         progress_timeout: float = None,
1049         total_timeout: float = None,
1050     ) -> Machine:
1051
1052         self.log.debug(
1053             "creating machine in model: {}, existing machine id: {}".format(
1054                 model_name, machine_id
1055             )
1056         )
1057
1058         # get juju model and observer (create model if needed)
1059         model = await self._juju_get_model(model_name=model_name)
1060         observer = self.juju_observers[model_name]
1061
1062         # find machine id in model
1063         machine = None
1064         if machine_id is not None:
1065             self.log.debug("Finding existing machine id {} in model".format(machine_id))
1066             # get juju existing machines in the model
1067             existing_machines = await model.get_machines()
1068             if machine_id in existing_machines:
1069                 self.log.debug(
1070                     "Machine id {} found in model (reusing it)".format(machine_id)
1071                 )
1072                 machine = model.machines[machine_id]
1073
1074         if machine is None:
1075             self.log.debug("Creating a new machine in juju...")
1076             # machine does not exist, create it and wait for it
1077             machine = await model.add_machine(
1078                 spec=None, constraints=None, disks=None, series="xenial"
1079             )
1080
1081             # register machine with observer
1082             observer.register_machine(machine=machine, db_dict=db_dict)
1083
1084             # id for the execution environment
1085             ee_id = N2VCJujuConnector._build_ee_id(
1086                 model_name=model_name,
1087                 application_name=application_name,
1088                 machine_id=str(machine.entity_id),
1089             )
1090
1091             # write ee_id in database
1092             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
1093
1094             # wait for machine creation
1095             await observer.wait_for_machine(
1096                 machine_id=str(machine.entity_id),
1097                 progress_timeout=progress_timeout,
1098                 total_timeout=total_timeout,
1099             )
1100
1101         else:
1102
1103             self.log.debug("Reusing old machine pending")
1104
1105             # register machine with observer
1106             observer.register_machine(machine=machine, db_dict=db_dict)
1107
1108             # machine does exist, but it is in creation process (pending), wait for
1109             # create finalisation
1110             await observer.wait_for_machine(
1111                 machine_id=machine.entity_id,
1112                 progress_timeout=progress_timeout,
1113                 total_timeout=total_timeout,
1114             )
1115
1116         self.log.debug("Machine ready at " + str(machine.dns_name))
1117         return machine
1118
1119     async def _juju_provision_machine(
1120         self,
1121         model_name: str,
1122         hostname: str,
1123         username: str,
1124         private_key_path: str,
1125         db_dict: dict = None,
1126         progress_timeout: float = None,
1127         total_timeout: float = None,
1128     ) -> str:
1129
1130         if not self.api_proxy:
1131             msg = "Cannot provision machine: api_proxy is not defined"
1132             self.log.error(msg=msg)
1133             raise N2VCException(message=msg)
1134
1135         self.log.debug(
1136             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1137                 model_name, hostname, username
1138             )
1139         )
1140
1141         if not self._authenticated:
1142             await self._juju_login()
1143
1144         # get juju model and observer
1145         model = await self._juju_get_model(model_name=model_name)
1146         observer = self.juju_observers[model_name]
1147
1148         # TODO check if machine is already provisioned
1149         machine_list = await model.get_machines()
1150
1151         provisioner = AsyncSSHProvisioner(
1152             host=hostname,
1153             user=username,
1154             private_key_path=private_key_path,
1155             log=self.log,
1156         )
1157
1158         params = None
1159         try:
1160             params = await provisioner.provision_machine()
1161         except Exception as ex:
1162             msg = "Exception provisioning machine: {}".format(ex)
1163             self.log.error(msg)
1164             raise N2VCException(message=msg)
1165
1166         params.jobs = ["JobHostUnits"]
1167
1168         connection = model.connection()
1169
1170         # Submit the request.
1171         self.log.debug("Adding machine to model")
1172         client_facade = client.ClientFacade.from_connection(connection)
1173         results = await client_facade.AddMachines(params=[params])
1174         error = results.machines[0].error
1175         if error:
1176             msg = "Error adding machine: {}".format(error.message)
1177             self.log.error(msg=msg)
1178             raise ValueError(msg)
1179
1180         machine_id = results.machines[0].machine
1181
1182         # Need to run this after AddMachines has been called,
1183         # as we need the machine_id
1184         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1185         asyncio.ensure_future(
1186             provisioner.install_agent(
1187                 connection=connection,
1188                 nonce=params.nonce,
1189                 machine_id=machine_id,
1190                 api=self.api_proxy,
1191             )
1192         )
1193
1194         # wait for machine in model (now, machine is not yet in model, so we must
1195         # wait for it)
1196         machine = None
1197         for _ in range(10):
1198             machine_list = await model.get_machines()
1199             if machine_id in machine_list:
1200                 self.log.debug("Machine {} found in model!".format(machine_id))
1201                 machine = model.machines.get(machine_id)
1202                 break
1203             await asyncio.sleep(2)
1204
1205         if machine is None:
1206             msg = "Machine {} not found in model".format(machine_id)
1207             self.log.error(msg=msg)
1208             raise Exception(msg)
1209
1210         # register machine with observer
1211         observer.register_machine(machine=machine, db_dict=db_dict)
1212
1213         # wait for machine creation
1214         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1215         await observer.wait_for_machine(
1216             machine_id=machine_id,
1217             progress_timeout=progress_timeout,
1218             total_timeout=total_timeout,
1219         )
1220
1221         self.log.debug("Machine provisioned {}".format(machine_id))
1222
1223         return machine_id
1224
1225     async def _juju_deploy_charm(
1226         self,
1227         model_name: str,
1228         application_name: str,
1229         charm_path: str,
1230         machine_id: str,
1231         db_dict: dict,
1232         progress_timeout: float = None,
1233         total_timeout: float = None,
1234         config: dict = None,
1235     ) -> (Application, int):
1236
1237         # get juju model and observer
1238         model = await self._juju_get_model(model_name=model_name)
1239         observer = self.juju_observers[model_name]
1240
1241         # check if application already exists
1242         application = None
1243         if application_name in model.applications:
1244             application = model.applications[application_name]
1245
1246         if application is None:
1247
1248             # application does not exist, create it and wait for it
1249             self.log.debug(
1250                 "deploying application {} to machine {}, model {}".format(
1251                     application_name, machine_id, model_name
1252                 )
1253             )
1254             self.log.debug("charm: {}".format(charm_path))
1255             machine = model.machines[machine_id]
1256             # series = None
1257             application = await model.deploy(
1258                 entity_url=charm_path,
1259                 application_name=application_name,
1260                 channel="stable",
1261                 num_units=1,
1262                 series=machine.series,
1263                 to=machine_id,
1264                 config=config,
1265             )
1266
1267             # register application with observer
1268             observer.register_application(application=application, db_dict=db_dict)
1269
1270             self.log.debug(
1271                 "waiting for application deployed... {}".format(application.entity_id)
1272             )
1273             retries = await observer.wait_for_application(
1274                 application_id=application.entity_id,
1275                 progress_timeout=progress_timeout,
1276                 total_timeout=total_timeout,
1277             )
1278             self.log.debug("application deployed")
1279
1280         else:
1281
1282             # register application with observer
1283             observer.register_application(application=application, db_dict=db_dict)
1284
1285             # application already exists, but not finalised
1286             self.log.debug("application already exists, waiting for deployed...")
1287             retries = await observer.wait_for_application(
1288                 application_id=application.entity_id,
1289                 progress_timeout=progress_timeout,
1290                 total_timeout=total_timeout,
1291             )
1292             self.log.debug("application deployed")
1293
1294         return application, retries
1295
1296     async def _juju_execute_action(
1297         self,
1298         model_name: str,
1299         application_name: str,
1300         action_name: str,
1301         db_dict: dict,
1302         progress_timeout: float = None,
1303         total_timeout: float = None,
1304         **kwargs
1305     ) -> Action:
1306
1307         # get juju model and observer
1308         model = await self._juju_get_model(model_name=model_name)
1309         observer = self.juju_observers[model_name]
1310
1311         application = await self._juju_get_application(
1312             model_name=model_name, application_name=application_name
1313         )
1314
1315         unit = None
1316         for u in application.units:
1317             if await u.is_leader_from_status():
1318                 unit = u
1319         if unit is not None:
1320             actions = await application.get_actions()
1321             if action_name in actions:
1322                 self.log.debug(
1323                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1324                 )
1325                 action = await unit.run_action(action_name, **kwargs)
1326
1327                 # register action with observer
1328                 observer.register_action(action=action, db_dict=db_dict)
1329
1330                 await observer.wait_for_action(
1331                     action_id=action.entity_id,
1332                     progress_timeout=progress_timeout,
1333                     total_timeout=total_timeout,
1334                 )
1335                 self.log.debug("action completed with status: {}".format(action.status))
1336                 output = await model.get_action_output(action_uuid=action.entity_id)
1337                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1338                 if action.entity_id in status:
1339                     status = status[action.entity_id]
1340                 else:
1341                     status = "failed"
1342                 return output, status
1343
1344         raise N2VCExecutionException(
1345             message="Cannot execute action on charm", primitive_name=action_name
1346         )
1347
1348     async def _juju_configure_application(
1349         self,
1350         model_name: str,
1351         application_name: str,
1352         config: dict,
1353         db_dict: dict,
1354         progress_timeout: float = None,
1355         total_timeout: float = None,
1356     ):
1357
1358         # get the application
1359         application = await self._juju_get_application(
1360             model_name=model_name, application_name=application_name
1361         )
1362
1363         self.log.debug(
1364             "configuring the application {} -> {}".format(application_name, config)
1365         )
1366         res = await application.set_config(config)
1367         self.log.debug(
1368             "application {} configured. res={}".format(application_name, res)
1369         )
1370
1371         # Verify the config is set
1372         new_conf = await application.get_config()
1373         for key in config:
1374             value = new_conf[key]["value"]
1375             self.log.debug("    {} = {}".format(key, value))
1376             if config[key] != value:
1377                 raise N2VCException(
1378                     message="key {} is not configured correctly {} != {}".format(
1379                         key, config[key], new_conf[key]
1380                     )
1381                 )
1382
1383         # check if 'verify-ssh-credentials' action exists
1384         # unit = application.units[0]
1385         actions = await application.get_actions()
1386         if "verify-ssh-credentials" not in actions:
1387             msg = (
1388                 "Action verify-ssh-credentials does not exist in application {}"
1389             ).format(application_name)
1390             self.log.debug(msg=msg)
1391             return False
1392
1393         # execute verify-credentials
1394         num_retries = 20
1395         retry_timeout = 15.0
1396         for _ in range(num_retries):
1397             try:
1398                 self.log.debug("Executing action verify-ssh-credentials...")
1399                 output, ok = await self._juju_execute_action(
1400                     model_name=model_name,
1401                     application_name=application_name,
1402                     action_name="verify-ssh-credentials",
1403                     db_dict=db_dict,
1404                     progress_timeout=progress_timeout,
1405                     total_timeout=total_timeout,
1406                 )
1407                 self.log.debug("Result: {}, output: {}".format(ok, output))
1408                 return True
1409             except asyncio.CancelledError:
1410                 raise
1411             except Exception as e:
1412                 self.log.debug(
1413                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1414                 )
1415                 await asyncio.sleep(retry_timeout)
1416         else:
1417             self.log.error(
1418                 "Error executing verify-ssh-credentials after {} retries. ".format(
1419                     num_retries
1420                 )
1421             )
1422             return False
1423
1424     async def _juju_get_application(self, model_name: str, application_name: str):
1425         """Get the deployed application."""
1426
1427         model = await self._juju_get_model(model_name=model_name)
1428
1429         application_name = N2VCJujuConnector._format_app_name(application_name)
1430
1431         if model.applications and application_name in model.applications:
1432             return model.applications[application_name]
1433         else:
1434             raise N2VCException(
1435                 message="Cannot get application {} from model {}".format(
1436                     application_name, model_name
1437                 )
1438             )
1439
1440     async def _juju_get_model(self, model_name: str) -> Model:
1441         """ Get a model object from juju controller
1442         If the model does not exits, it creates it.
1443
1444         :param str model_name: name of the model
1445         :returns Model: model obtained from juju controller or Exception
1446         """
1447
1448         # format model name
1449         model_name = N2VCJujuConnector._format_model_name(model_name)
1450
1451         if model_name in self.juju_models:
1452             return self.juju_models[model_name]
1453
1454         if self._creating_model:
1455             self.log.debug("Another coroutine is creating a model. Wait...")
1456         while self._creating_model:
1457             # another coroutine is creating a model, wait
1458             await asyncio.sleep(0.1)
1459             # retry (perhaps another coroutine has created the model meanwhile)
1460             if model_name in self.juju_models:
1461                 return self.juju_models[model_name]
1462
1463         try:
1464             self._creating_model = True
1465
1466             # get juju model names from juju
1467             model_list = await self.controller.list_models()
1468             if model_name not in model_list:
1469                 self.log.info(
1470                     "Model {} does not exist. Creating new model...".format(model_name)
1471                 )
1472                 config_dict = {"authorized-keys": self.public_key}
1473                 if self.apt_mirror:
1474                     config_dict["apt-mirror"] = self.apt_mirror
1475                 if not self.enable_os_upgrade:
1476                     config_dict["enable-os-refresh-update"] = False
1477                     config_dict["enable-os-upgrade"] = False
1478                 if self.cloud in self.BUILT_IN_CLOUDS:
1479                     model = await self.controller.add_model(
1480                         model_name=model_name,
1481                         config=config_dict,
1482                         cloud_name=self.cloud,
1483                     )
1484                 else:
1485                     model = await self.controller.add_model(
1486                         model_name=model_name,
1487                         config=config_dict,
1488                         cloud_name=self.cloud,
1489                         credential_name=self.cloud,
1490                     )
1491                 self.log.info("New model created, name={}".format(model_name))
1492             else:
1493                 self.log.debug(
1494                     "Model already exists in juju. Getting model {}".format(model_name)
1495                 )
1496                 model = await self.controller.get_model(model_name)
1497                 self.log.debug("Existing model in juju, name={}".format(model_name))
1498
1499             self.juju_models[model_name] = model
1500             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1501             return model
1502
1503         except Exception as e:
1504             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1505             self.log.error(msg)
1506             raise N2VCException(msg)
1507         finally:
1508             self._creating_model = False
1509
1510     async def _juju_add_relation(
1511         self,
1512         model_name: str,
1513         application_name_1: str,
1514         application_name_2: str,
1515         relation_1: str,
1516         relation_2: str,
1517     ):
1518
1519         # get juju model and observer
1520         model = await self._juju_get_model(model_name=model_name)
1521
1522         r1 = "{}:{}".format(application_name_1, relation_1)
1523         r2 = "{}:{}".format(application_name_2, relation_2)
1524
1525         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1526         try:
1527             await model.add_relation(relation1=r1, relation2=r2)
1528         except JujuAPIError as e:
1529             # If one of the applications in the relationship doesn't exist, or the
1530             # relation has already been added,
1531             # let the operation fail silently.
1532             if "not found" in e.message:
1533                 return
1534             if "already exists" in e.message:
1535                 return
1536             # another execption, raise it
1537             raise e
1538
1539     async def _juju_destroy_application(self, model_name: str, application_name: str):
1540
1541         self.log.debug(
1542             "Destroying application {} in model {}".format(application_name, model_name)
1543         )
1544
1545         # get juju model and observer
1546         model = await self._juju_get_model(model_name=model_name)
1547         observer = self.juju_observers[model_name]
1548
1549         application = model.applications.get(application_name)
1550         if application:
1551             observer.unregister_application(application_name)
1552             await application.destroy()
1553         else:
1554             self.log.debug("Application not found: {}".format(application_name))
1555
1556     async def _juju_destroy_machine(
1557         self, model_name: str, machine_id: str, total_timeout: float = None
1558     ):
1559
1560         self.log.debug(
1561             "Destroying machine {} in model {}".format(machine_id, model_name)
1562         )
1563
1564         if total_timeout is None:
1565             total_timeout = 3600
1566
1567         # get juju model and observer
1568         model = await self._juju_get_model(model_name=model_name)
1569         observer = self.juju_observers[model_name]
1570
1571         machines = await model.get_machines()
1572         if machine_id in machines:
1573             machine = model.machines[machine_id]
1574             observer.unregister_machine(machine_id)
1575             # TODO: change this by machine.is_manual when this is upstreamed:
1576             # https://github.com/juju/python-libjuju/pull/396
1577             if "instance-id" in machine.safe_data and machine.safe_data[
1578                 "instance-id"
1579             ].startswith("manual:"):
1580                 self.log.debug("machine.destroy(force=True) started.")
1581                 await machine.destroy(force=True)
1582                 self.log.debug("machine.destroy(force=True) passed.")
1583                 # max timeout
1584                 end = time.time() + total_timeout
1585                 # wait for machine removal
1586                 machines = await model.get_machines()
1587                 while machine_id in machines and time.time() < end:
1588                     self.log.debug(
1589                         "Waiting for machine {} is destroyed".format(machine_id)
1590                     )
1591                     await asyncio.sleep(0.5)
1592                     machines = await model.get_machines()
1593                 self.log.debug("Machine destroyed: {}".format(machine_id))
1594         else:
1595             self.log.debug("Machine not found: {}".format(machine_id))
1596
1597     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1598
1599         self.log.debug("Destroying model {}".format(model_name))
1600
1601         if total_timeout is None:
1602             total_timeout = 3600
1603         end = time.time() + total_timeout
1604
1605         model = await self._juju_get_model(model_name=model_name)
1606
1607         if not model:
1608             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1609
1610         uuid = model.info.uuid
1611
1612         # destroy applications
1613         for application_name in model.applications:
1614             try:
1615                 await self._juju_destroy_application(
1616                     model_name=model_name, application_name=application_name
1617                 )
1618             except Exception as e:
1619                 self.log.error(
1620                     "Error destroying application {} in model {}: {}".format(
1621                         application_name, model_name, e
1622                     )
1623                 )
1624
1625         # destroy machines
1626         machines = await model.get_machines()
1627         for machine_id in machines:
1628             try:
1629                 await self._juju_destroy_machine(
1630                     model_name=model_name, machine_id=machine_id
1631                 )
1632             except asyncio.CancelledError:
1633                 raise
1634             except Exception:
1635                 # ignore exceptions destroying machine
1636                 pass
1637
1638         await self._juju_disconnect_model(model_name=model_name)
1639
1640         self.log.debug("destroying model {}...".format(model_name))
1641         await self.controller.destroy_model(uuid)
1642         # self.log.debug('model destroy requested {}'.format(model_name))
1643
1644         # wait for model is completely destroyed
1645         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1646         last_exception = ""
1647         while time.time() < end:
1648             try:
1649                 # await self.controller.get_model(uuid)
1650                 models = await self.controller.list_models()
1651                 if model_name not in models:
1652                     self.log.debug(
1653                         "The model {} ({}) was destroyed".format(model_name, uuid)
1654                     )
1655                     return
1656             except asyncio.CancelledError:
1657                 raise
1658             except Exception as e:
1659                 last_exception = e
1660             await asyncio.sleep(5)
1661         raise N2VCException(
1662             "Timeout waiting for model {} to be destroyed {}".format(
1663                 model_name, last_exception
1664             )
1665         )
1666
1667     async def _juju_login(self):
1668         """Connect to juju controller
1669
1670         """
1671
1672         # if already authenticated, exit function
1673         if self._authenticated:
1674             return
1675
1676         # if connecting, wait for finish
1677         # another task could be trying to connect in parallel
1678         while self._connecting:
1679             await asyncio.sleep(0.1)
1680
1681         # double check after other task has finished
1682         if self._authenticated:
1683             return
1684
1685         try:
1686             self._connecting = True
1687             self.log.info(
1688                 "connecting to juju controller: {} {}:{}{}".format(
1689                     self.url,
1690                     self.username,
1691                     self.secret[:8] + "...",
1692                     " with ca_cert" if self.ca_cert else "",
1693                 )
1694             )
1695
1696             # Create controller object
1697             self.controller = Controller(loop=self.loop)
1698             # Connect to controller
1699             await self.controller.connect(
1700                 endpoint=self.url,
1701                 username=self.username,
1702                 password=self.secret,
1703                 cacert=self.ca_cert,
1704             )
1705             self._authenticated = True
1706             self.log.info("juju controller connected")
1707         except Exception as e:
1708             message = "Exception connecting to juju: {}".format(e)
1709             self.log.error(message)
1710             raise N2VCConnectionException(message=message, url=self.url)
1711         finally:
1712             self._connecting = False
1713
1714     async def _juju_logout(self):
1715         """Logout of the Juju controller."""
1716         if not self._authenticated:
1717             return False
1718
1719         # disconnect all models
1720         for model_name in self.juju_models:
1721             try:
1722                 await self._juju_disconnect_model(model_name)
1723             except Exception as e:
1724                 self.log.error(
1725                     "Error disconnecting model {} : {}".format(model_name, e)
1726                 )
1727                 # continue with next model...
1728
1729         self.log.info("Disconnecting controller")
1730         try:
1731             await self.controller.disconnect()
1732         except Exception as e:
1733             raise N2VCConnectionException(
1734                 message="Error disconnecting controller: {}".format(e), url=self.url
1735             )
1736
1737         self.controller = None
1738         self._authenticated = False
1739         self.log.info("disconnected")
1740
1741     async def _juju_disconnect_model(self, model_name: str):
1742         self.log.debug("Disconnecting model {}".format(model_name))
1743         if model_name in self.juju_models:
1744             await self.juju_models[model_name].disconnect()
1745             self.juju_models[model_name] = None
1746             self.juju_observers[model_name] = None
1747         else:
1748             self.warning("Cannot disconnect model: {}".format(model_name))
1749
1750     def _create_juju_public_key(self):
1751         """Recreate the Juju public key on lcm container, if needed
1752         Certain libjuju commands expect to be run from the same machine as Juju
1753          is bootstrapped to. This method will write the public key to disk in
1754          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1755         """
1756
1757         # Make sure that we have a public key before writing to disk
1758         if self.public_key is None or len(self.public_key) == 0:
1759             if "OSMLCM_VCA_PUBKEY" in os.environ:
1760                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1761                 if len(self.public_key) == 0:
1762                     return
1763             else:
1764                 return
1765
1766         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1767         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1768         self.log.debug(
1769             "writing juju public key to file:\n{}\npublic key: {}".format(
1770                 file_path, self.public_key
1771             )
1772         )
1773         if not os.path.exists(pk_path):
1774             # create path and write file
1775             os.makedirs(pk_path)
1776             with open(file_path, "w") as f:
1777                 self.log.debug("Creating juju public key file: {}".format(file_path))
1778                 f.write(self.public_key)
1779         else:
1780             self.log.debug("juju public key file already exists: {}".format(file_path))
1781
1782     @staticmethod
1783     def _format_model_name(name: str) -> str:
1784         """Format the name of the model.
1785
1786         Model names may only contain lowercase letters, digits and hyphens
1787         """
1788
1789         return name.replace("_", "-").replace(" ", "-").lower()
1790
1791     @staticmethod
1792     def _format_app_name(name: str) -> str:
1793         """Format the name of the application (in order to assure valid application name).
1794
1795         Application names have restrictions (run juju deploy --help):
1796             - contains lowercase letters 'a'-'z'
1797             - contains numbers '0'-'9'
1798             - contains hyphens '-'
1799             - starts with a lowercase letter
1800             - not two or more consecutive hyphens
1801             - after a hyphen, not a group with all numbers
1802         """
1803
1804         def all_numbers(s: str) -> bool:
1805             for c in s:
1806                 if not c.isdigit():
1807                     return False
1808             return True
1809
1810         new_name = name.replace("_", "-")
1811         new_name = new_name.replace(" ", "-")
1812         new_name = new_name.lower()
1813         while new_name.find("--") >= 0:
1814             new_name = new_name.replace("--", "-")
1815         groups = new_name.split("-")
1816
1817         # find 'all numbers' groups and prefix them with a letter
1818         app_name = ""
1819         for i in range(len(groups)):
1820             group = groups[i]
1821             if all_numbers(group):
1822                 group = "z" + group
1823             if i > 0:
1824                 app_name += "-"
1825             app_name += group
1826
1827         if app_name[0].isdigit():
1828             app_name = "z" + app_name
1829
1830         return app_name