n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46     JujuK8sProxycharmNotSupported,
  47 )
  48 from n2vc.juju_observer import JujuModelObserver
  49 from n2vc.n2vc_conn import N2VCConnector
  50 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  51 from n2vc.provisioner import AsyncSSHProvisioner
  52 from n2vc.libjuju import Libjuju
  53
  54
  55 class N2VCJujuConnector(N2VCConnector):
  56
  57     """
  58 ####################################################################################
  59 ################################### P U B L I C ####################################
  60 ####################################################################################
  61     """
  62
  63     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  64
  65     def __init__(
  66         self,
  67         db: object,
  68         fs: object,
  69         log: object = None,
  70         loop: object = None,
  71         url: str = "127.0.0.1:17070",
  72         username: str = "admin",
  73         vca_config: dict = None,
  74         on_update_db=None,
  75     ):
  76         """Initialize juju N2VC connector
  77         """
  78
  79         # parent class constructor
  80         N2VCConnector.__init__(
  81             self,
  82             db=db,
  83             fs=fs,
  84             log=log,
  85             loop=loop,
  86             url=url,
  87             username=username,
  88             vca_config=vca_config,
  89             on_update_db=on_update_db,
  90         )
  91
  92         # silence websocket traffic log
  93         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  94         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  95         logging.getLogger("model").setLevel(logging.WARN)
  96
  97         self.log.info("Initializing N2VC juju connector...")
  98
  99         """
 100         ##############################################################
 101         # check arguments
 102         ##############################################################
 103         """
 104
 105         # juju URL
 106         if url is None:
 107             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 108         url_parts = url.split(":")
 109         if len(url_parts) != 2:
 110             raise N2VCBadArgumentsException(
 111                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 112             )
 113         self.hostname = url_parts[0]
 114         try:
 115             self.port = int(url_parts[1])
 116         except ValueError:
 117             raise N2VCBadArgumentsException(
 118                 "url port must be a number -> {}".format(url), ["url"]
 119             )
 120
 121         # juju USERNAME
 122         if username is None:
 123             raise N2VCBadArgumentsException(
 124                 "Argument username is mandatory", ["username"]
 125             )
 126
 127         # juju CONFIGURATION
 128         if vca_config is None:
 129             raise N2VCBadArgumentsException(
 130                 "Argument vca_config is mandatory", ["vca_config"]
 131             )
 132
 133         if "secret" in vca_config:
 134             self.secret = vca_config["secret"]
 135         else:
 136             raise N2VCBadArgumentsException(
 137                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 138             )
 139
 140         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 141         # if exists, it will be written in lcm container: _create_juju_public_key()
 142         if "public_key" in vca_config:
 143             self.public_key = vca_config["public_key"]
 144         else:
 145             self.public_key = None
 146
 147         # TODO: Verify ca_cert is valid before using. VCA will crash
 148         # if the ca_cert isn't formatted correctly.
 149         def base64_to_cacert(b64string):
 150             """Convert the base64-encoded string containing the VCA CACERT.
 151
 152             The input string....
 153
 154             """
 155             try:
 156                 cacert = base64.b64decode(b64string).decode("utf-8")
 157
 158                 cacert = re.sub(r"\\n", r"\n", cacert,)
 159             except binascii.Error as e:
 160                 self.log.debug("Caught binascii.Error: {}".format(e))
 161                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 162
 163             return cacert
 164
 165         self.ca_cert = vca_config.get("ca_cert")
 166         if self.ca_cert:
 167             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 168
 169         if "api_proxy" in vca_config and vca_config["api_proxy"] != "":
 170             self.api_proxy = vca_config["api_proxy"]
 171             self.log.debug(
 172                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 173             )
 174         else:
 175             self.warning(
 176                 "api_proxy is not configured"
 177             )
 178             self.api_proxy = None
 179
 180         if "enable_os_upgrade" in vca_config:
 181             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 182         else:
 183             self.enable_os_upgrade = True
 184
 185         if "apt_mirror" in vca_config:
 186             self.apt_mirror = vca_config["apt_mirror"]
 187         else:
 188             self.apt_mirror = None
 189
 190         self.cloud = vca_config.get('cloud')
 191         self.k8s_cloud = None
 192         if "k8s_cloud" in vca_config:
 193             self.k8s_cloud = vca_config.get("k8s_cloud")
 194         self.log.debug('Arguments have been checked')
 195
 196         # juju data
 197         self.controller = None  # it will be filled when connect to juju
 198         self.juju_models = {}  # model objects for every model_name
 199         self.juju_observers = {}  # model observers for every model_name
 200         self._connecting = (
 201             False  # while connecting to juju (to avoid duplicate connections)
 202         )
 203         self._authenticated = (
 204             False  # it will be True when juju connection be stablished
 205         )
 206         self._creating_model = False  # True during model creation
 207         self.libjuju = Libjuju(
 208             endpoint=self.url,
 209             api_proxy=self.api_proxy,
 210             enable_os_upgrade=self.enable_os_upgrade,
 211             apt_mirror=self.apt_mirror,
 212             username=self.username,
 213             password=self.secret,
 214             cacert=self.ca_cert,
 215             loop=self.loop,
 216             log=self.log,
 217             db=self.db,
 218             n2vc=self,
 219         )
 220
 221         # create juju pub key file in lcm container at
 222         # ./local/share/juju/ssh/juju_id_rsa.pub
 223         self._create_juju_public_key()
 224
 225         self.log.info("N2VC juju connector initialized")
 226
 227     async def get_status(self, namespace: str, yaml_format: bool = True):
 228
 229         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 230
 231         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 232             namespace=namespace
 233         )
 234         # model name is ns_id
 235         model_name = ns_id
 236         if model_name is None:
 237             msg = "Namespace {} not valid".format(namespace)
 238             self.log.error(msg)
 239             raise N2VCBadArgumentsException(msg, ["namespace"])
 240
 241         status = {}
 242         models = await self.libjuju.list_models(contains=ns_id)
 243
 244         for m in models:
 245             status[m] = await self.libjuju.get_model_status(m)
 246         if yaml_format:
 247             return obj_to_yaml(status)
 248         else:
 249             return obj_to_dict(status)
 250
 251     async def update_vca_status(self, vcastatus: dict):
 252         """
 253         Add all configs, actions, executed actions of all applications in a model to vcastatus dict.
 254         :param vcastatus: dict containing vcaStatus
 255         :return: None
 256         """
 257         try:
 258             for model_name in vcastatus:
 259                 # Adding executed actions
 260                 vcastatus[model_name]["executedActions"] = \
 261                     await self.libjuju.get_executed_actions(model_name)
 262                 for application in vcastatus[model_name]["applications"]:
 263                     # Adding application actions
 264                     vcastatus[model_name]["applications"][application]["actions"] = \
 265                         await self.libjuju.get_actions(application, model_name)
 266                     # Adding application configs
 267                     vcastatus[model_name]["applications"][application]["configs"] = \
 268                         await self.libjuju.get_application_configs(model_name, application)
 269         except Exception as e:
 270             self.log.debug("Error in updating vca status: {}".format(str(e)))
 271
 272     async def create_execution_environment(
 273         self,
 274         namespace: str,
 275         db_dict: dict,
 276         reuse_ee_id: str = None,
 277         progress_timeout: float = None,
 278         total_timeout: float = None,
 279     ) -> (str, dict):
 280
 281         self.log.info(
 282             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 283                 namespace, reuse_ee_id
 284             )
 285         )
 286
 287         machine_id = None
 288         if reuse_ee_id:
 289             model_name, application_name, machine_id = self._get_ee_id_components(
 290                 ee_id=reuse_ee_id
 291             )
 292         else:
 293             (
 294                 _nsi_id,
 295                 ns_id,
 296                 _vnf_id,
 297                 _vdu_id,
 298                 _vdu_count,
 299             ) = self._get_namespace_components(namespace=namespace)
 300             # model name is ns_id
 301             model_name = ns_id
 302             # application name
 303             application_name = self._get_application_name(namespace=namespace)
 304
 305         self.log.debug(
 306             "model name: {}, application name:  {}, machine_id: {}".format(
 307                 model_name, application_name, machine_id
 308             )
 309         )
 310
 311         # create or reuse a new juju machine
 312         try:
 313             if not await self.libjuju.model_exists(model_name):
 314                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 315             machine, new = await self.libjuju.create_machine(
 316                 model_name=model_name,
 317                 machine_id=machine_id,
 318                 db_dict=db_dict,
 319                 progress_timeout=progress_timeout,
 320                 total_timeout=total_timeout,
 321             )
 322             # id for the execution environment
 323             ee_id = N2VCJujuConnector._build_ee_id(
 324                 model_name=model_name,
 325                 application_name=application_name,
 326                 machine_id=str(machine.entity_id),
 327             )
 328             self.log.debug("ee_id: {}".format(ee_id))
 329
 330             if new:
 331                 # write ee_id in database
 332                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 333
 334         except Exception as e:
 335             message = "Error creating machine on juju: {}".format(e)
 336             self.log.error(message)
 337             raise N2VCException(message=message)
 338
 339         # new machine credentials
 340         credentials = {
 341             "hostname": machine.dns_name,
 342         }
 343
 344         self.log.info(
 345             "Execution environment created. ee_id: {}, credentials: {}".format(
 346                 ee_id, credentials
 347             )
 348         )
 349
 350         return ee_id, credentials
 351
 352     async def register_execution_environment(
 353         self,
 354         namespace: str,
 355         credentials: dict,
 356         db_dict: dict,
 357         progress_timeout: float = None,
 358         total_timeout: float = None,
 359     ) -> str:
 360
 361         self.log.info(
 362             "Registering execution environment. namespace={}, credentials={}".format(
 363                 namespace, credentials
 364             )
 365         )
 366
 367         if credentials is None:
 368             raise N2VCBadArgumentsException(
 369                 message="credentials are mandatory", bad_args=["credentials"]
 370             )
 371         if credentials.get("hostname"):
 372             hostname = credentials["hostname"]
 373         else:
 374             raise N2VCBadArgumentsException(
 375                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 376             )
 377         if credentials.get("username"):
 378             username = credentials["username"]
 379         else:
 380             raise N2VCBadArgumentsException(
 381                 message="username is mandatory", bad_args=["credentials.username"]
 382             )
 383         if "private_key_path" in credentials:
 384             private_key_path = credentials["private_key_path"]
 385         else:
 386             # if not passed as argument, use generated private key path
 387             private_key_path = self.private_key_path
 388
 389         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 390             namespace=namespace
 391         )
 392
 393         # model name
 394         model_name = ns_id
 395         # application name
 396         application_name = self._get_application_name(namespace=namespace)
 397
 398         # register machine on juju
 399         try:
 400             if not await self.libjuju.model_exists(model_name):
 401                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 402             machine_id = await self.libjuju.provision_machine(
 403                 model_name=model_name,
 404                 hostname=hostname,
 405                 username=username,
 406                 private_key_path=private_key_path,
 407                 db_dict=db_dict,
 408                 progress_timeout=progress_timeout,
 409                 total_timeout=total_timeout,
 410             )
 411         except Exception as e:
 412             self.log.error("Error registering machine: {}".format(e))
 413             raise N2VCException(
 414                 message="Error registering machine on juju: {}".format(e)
 415             )
 416
 417         self.log.info("Machine registered: {}".format(machine_id))
 418
 419         # id for the execution environment
 420         ee_id = N2VCJujuConnector._build_ee_id(
 421             model_name=model_name,
 422             application_name=application_name,
 423             machine_id=str(machine_id),
 424         )
 425
 426         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 427
 428         return ee_id
 429
 430     async def install_configuration_sw(
 431         self,
 432         ee_id: str,
 433         artifact_path: str,
 434         db_dict: dict,
 435         progress_timeout: float = None,
 436         total_timeout: float = None,
 437         config: dict = None,
 438         num_units: int = 1,
 439     ):
 440
 441         self.log.info(
 442             (
 443                 "Installing configuration sw on ee_id: {}, "
 444                 "artifact path: {}, db_dict: {}"
 445             ).format(ee_id, artifact_path, db_dict)
 446         )
 447
 448         # check arguments
 449         if ee_id is None or len(ee_id) == 0:
 450             raise N2VCBadArgumentsException(
 451                 message="ee_id is mandatory", bad_args=["ee_id"]
 452             )
 453         if artifact_path is None or len(artifact_path) == 0:
 454             raise N2VCBadArgumentsException(
 455                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 456             )
 457         if db_dict is None:
 458             raise N2VCBadArgumentsException(
 459                 message="db_dict is mandatory", bad_args=["db_dict"]
 460             )
 461
 462         try:
 463             (
 464                 model_name,
 465                 application_name,
 466                 machine_id,
 467             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 468             self.log.debug(
 469                 "model: {}, application: {}, machine: {}".format(
 470                     model_name, application_name, machine_id
 471                 )
 472             )
 473         except Exception:
 474             raise N2VCBadArgumentsException(
 475                 message="ee_id={} is not a valid execution environment id".format(
 476                     ee_id
 477                 ),
 478                 bad_args=["ee_id"],
 479             )
 480
 481         # remove // in charm path
 482         while artifact_path.find("//") >= 0:
 483             artifact_path = artifact_path.replace("//", "/")
 484
 485         # check charm path
 486         if not self.fs.file_exists(artifact_path, mode="dir"):
 487             msg = "artifact path does not exist: {}".format(artifact_path)
 488             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 489
 490         if artifact_path.startswith("/"):
 491             full_path = self.fs.path + artifact_path
 492         else:
 493             full_path = self.fs.path + "/" + artifact_path
 494
 495         try:
 496             await self.libjuju.deploy_charm(
 497                 model_name=model_name,
 498                 application_name=application_name,
 499                 path=full_path,
 500                 machine_id=machine_id,
 501                 db_dict=db_dict,
 502                 progress_timeout=progress_timeout,
 503                 total_timeout=total_timeout,
 504                 config=config,
 505                 num_units=num_units,
 506             )
 507         except Exception as e:
 508             raise N2VCException(
 509                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 510             )
 511
 512         self.log.info("Configuration sw installed")
 513
 514     async def install_k8s_proxy_charm(
 515         self,
 516         charm_name: str,
 517         namespace: str,
 518         artifact_path: str,
 519         db_dict: dict,
 520         progress_timeout: float = None,
 521         total_timeout: float = None,
 522         config: dict = None,
 523     ) -> str:
 524         """
 525         Install a k8s proxy charm
 526
 527         :param charm_name: Name of the charm being deployed
 528         :param namespace: collection of all the uuids related to the charm.
 529         :param str artifact_path: where to locate the artifacts (parent folder) using
 530             the self.fs
 531             the final artifact path will be a combination of this artifact_path and
 532             additional string from the config_dict (e.g. charm name)
 533         :param dict db_dict: where to write into database when the status changes.
 534                         It contains a dict with
 535                             {collection: <str>, filter: {},  path: <str>},
 536                             e.g. {collection: "nsrs", filter:
 537                                 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
 538         :param float progress_timeout:
 539         :param float total_timeout:
 540         :param config: Dictionary with additional configuration
 541
 542         :returns ee_id: execution environment id.
 543         """
 544         self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
 545                       .format(charm_name, artifact_path, db_dict))
 546
 547         if not self.k8s_cloud:
 548             raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
 549
 550         if artifact_path is None or len(artifact_path) == 0:
 551             raise N2VCBadArgumentsException(
 552                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 553             )
 554         if db_dict is None:
 555             raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict'])
 556
 557         # remove // in charm path
 558         while artifact_path.find('//') >= 0:
 559             artifact_path = artifact_path.replace('//', '/')
 560
 561         # check charm path
 562         if not self.fs.file_exists(artifact_path, mode="dir"):
 563             msg = 'artifact path does not exist: {}'.format(artifact_path)
 564             raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path'])
 565
 566         if artifact_path.startswith('/'):
 567             full_path = self.fs.path + artifact_path
 568         else:
 569             full_path = self.fs.path + '/' + artifact_path
 570
 571         _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace)
 572         model_name = '{}-k8s'.format(ns_id)
 573
 574         await self.libjuju.add_model(model_name, self.k8s_cloud)
 575         application_name = self._get_application_name(namespace)
 576
 577         try:
 578             await self.libjuju.deploy_charm(
 579                 model_name=model_name,
 580                 application_name=application_name,
 581                 path=full_path,
 582                 machine_id=None,
 583                 db_dict=db_dict,
 584                 progress_timeout=progress_timeout,
 585                 total_timeout=total_timeout,
 586                 config=config
 587             )
 588         except Exception as e:
 589             raise N2VCException(message='Error deploying charm: {}'.format(e))
 590
 591         self.log.info('K8s proxy charm installed')
 592         ee_id = N2VCJujuConnector._build_ee_id(
 593             model_name=model_name,
 594             application_name=application_name,
 595             machine_id="k8s",
 596         )
 597
 598         self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 599
 600         return ee_id
 601
 602     async def get_ee_ssh_public__key(
 603         self,
 604         ee_id: str,
 605         db_dict: dict,
 606         progress_timeout: float = None,
 607         total_timeout: float = None,
 608     ) -> str:
 609
 610         self.log.info(
 611             (
 612                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 613             ).format(ee_id, db_dict)
 614         )
 615
 616         # check arguments
 617         if ee_id is None or len(ee_id) == 0:
 618             raise N2VCBadArgumentsException(
 619                 message="ee_id is mandatory", bad_args=["ee_id"]
 620             )
 621         if db_dict is None:
 622             raise N2VCBadArgumentsException(
 623                 message="db_dict is mandatory", bad_args=["db_dict"]
 624             )
 625
 626         try:
 627             (
 628                 model_name,
 629                 application_name,
 630                 machine_id,
 631             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 632             self.log.debug(
 633                 "model: {}, application: {}, machine: {}".format(
 634                     model_name, application_name, machine_id
 635                 )
 636             )
 637         except Exception:
 638             raise N2VCBadArgumentsException(
 639                 message="ee_id={} is not a valid execution environment id".format(
 640                     ee_id
 641                 ),
 642                 bad_args=["ee_id"],
 643             )
 644
 645         # try to execute ssh layer primitives (if exist):
 646         #       generate-ssh-key
 647         #       get-ssh-public-key
 648
 649         output = None
 650
 651         application_name = N2VCJujuConnector._format_app_name(application_name)
 652
 653         # execute action: generate-ssh-key
 654         try:
 655             output, _status = await self.libjuju.execute_action(
 656                 model_name=model_name,
 657                 application_name=application_name,
 658                 action_name="generate-ssh-key",
 659                 db_dict=db_dict,
 660                 progress_timeout=progress_timeout,
 661                 total_timeout=total_timeout,
 662             )
 663         except Exception as e:
 664             self.log.info(
 665                 "Skipping exception while executing action generate-ssh-key: {}".format(
 666                     e
 667                 )
 668             )
 669
 670         # execute action: get-ssh-public-key
 671         try:
 672             output, _status = await self.libjuju.execute_action(
 673                 model_name=model_name,
 674                 application_name=application_name,
 675                 action_name="get-ssh-public-key",
 676                 db_dict=db_dict,
 677                 progress_timeout=progress_timeout,
 678                 total_timeout=total_timeout,
 679             )
 680         except Exception as e:
 681             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 682             self.log.info(msg)
 683             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 684
 685         # return public key if exists
 686         return output["pubkey"] if "pubkey" in output else output
 687
 688     async def add_relation(
 689         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 690     ):
 691
 692         self.log.debug(
 693             "adding new relation between {} and {}, endpoints: {}, {}".format(
 694                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 695             )
 696         )
 697
 698         # check arguments
 699         if not ee_id_1:
 700             message = "EE 1 is mandatory"
 701             self.log.error(message)
 702             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 703         if not ee_id_2:
 704             message = "EE 2 is mandatory"
 705             self.log.error(message)
 706             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 707         if not endpoint_1:
 708             message = "endpoint 1 is mandatory"
 709             self.log.error(message)
 710             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 711         if not endpoint_2:
 712             message = "endpoint 2 is mandatory"
 713             self.log.error(message)
 714             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 715
 716         # get the model, the applications and the machines from the ee_id's
 717         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 718         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 719
 720         # model must be the same
 721         if model_1 != model_2:
 722             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 723             self.log.error(message)
 724             raise N2VCBadArgumentsException(
 725                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 726             )
 727
 728         # add juju relations between two applications
 729         try:
 730             await self.libjuju.add_relation(
 731                 model_name=model_1,
 732                 application_name_1=app_1,
 733                 application_name_2=app_2,
 734                 relation_1=endpoint_1,
 735                 relation_2=endpoint_2,
 736             )
 737         except Exception as e:
 738             message = "Error adding relation between {} and {}: {}".format(
 739                 ee_id_1, ee_id_2, e
 740             )
 741             self.log.error(message)
 742             raise N2VCException(message=message)
 743
 744     async def remove_relation(self):
 745         # TODO
 746         self.log.info("Method not implemented yet")
 747         raise MethodNotImplemented()
 748
 749     async def deregister_execution_environments(self):
 750         self.log.info("Method not implemented yet")
 751         raise MethodNotImplemented()
 752
 753     async def delete_namespace(
 754         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 755     ):
 756         self.log.info("Deleting namespace={}".format(namespace))
 757
 758         # check arguments
 759         if namespace is None:
 760             raise N2VCBadArgumentsException(
 761                 message="namespace is mandatory", bad_args=["namespace"]
 762             )
 763
 764         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 765             namespace=namespace
 766         )
 767         if ns_id is not None:
 768             try:
 769                 models = await self.libjuju.list_models(contains=ns_id)
 770                 for model in models:
 771                     await self.libjuju.destroy_model(
 772                         model_name=model, total_timeout=total_timeout
 773                     )
 774             except Exception as e:
 775                 raise N2VCException(
 776                     message="Error deleting namespace {} : {}".format(namespace, e)
 777                 )
 778         else:
 779             raise N2VCBadArgumentsException(
 780                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 781             )
 782
 783         self.log.info("Namespace {} deleted".format(namespace))
 784
 785     async def delete_execution_environment(
 786         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 787     ):
 788         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 789
 790         # check arguments
 791         if ee_id is None:
 792             raise N2VCBadArgumentsException(
 793                 message="ee_id is mandatory", bad_args=["ee_id"]
 794             )
 795
 796         model_name, application_name, _machine_id = self._get_ee_id_components(
 797             ee_id=ee_id
 798         )
 799
 800         # destroy the application
 801         try:
 802             await self.libjuju.destroy_model(
 803                 model_name=model_name, total_timeout=total_timeout
 804             )
 805         except Exception as e:
 806             raise N2VCException(
 807                 message=(
 808                     "Error deleting execution environment {} (application {}) : {}"
 809                 ).format(ee_id, application_name, e)
 810             )
 811
 812         # destroy the machine
 813         # try:
 814         #     await self._juju_destroy_machine(
 815         #         model_name=model_name,
 816         #         machine_id=machine_id,
 817         #         total_timeout=total_timeout
 818         #     )
 819         # except Exception as e:
 820         #     raise N2VCException(
 821         #        message='Error deleting execution environment {} (machine {}) : {}'
 822         #                .format(ee_id, machine_id, e))
 823
 824         self.log.info("Execution environment {} deleted".format(ee_id))
 825
 826     async def exec_primitive(
 827         self,
 828         ee_id: str,
 829         primitive_name: str,
 830         params_dict: dict,
 831         db_dict: dict = None,
 832         progress_timeout: float = None,
 833         total_timeout: float = None,
 834     ) -> str:
 835
 836         self.log.info(
 837             "Executing primitive: {} on ee: {}, params: {}".format(
 838                 primitive_name, ee_id, params_dict
 839             )
 840         )
 841
 842         # check arguments
 843         if ee_id is None or len(ee_id) == 0:
 844             raise N2VCBadArgumentsException(
 845                 message="ee_id is mandatory", bad_args=["ee_id"]
 846             )
 847         if primitive_name is None or len(primitive_name) == 0:
 848             raise N2VCBadArgumentsException(
 849                 message="action_name is mandatory", bad_args=["action_name"]
 850             )
 851         if params_dict is None:
 852             params_dict = dict()
 853
 854         try:
 855             (
 856                 model_name,
 857                 application_name,
 858                 _machine_id,
 859             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 860         except Exception:
 861             raise N2VCBadArgumentsException(
 862                 message="ee_id={} is not a valid execution environment id".format(
 863                     ee_id
 864                 ),
 865                 bad_args=["ee_id"],
 866             )
 867
 868         if primitive_name == "config":
 869             # Special case: config primitive
 870             try:
 871                 await self.libjuju.configure_application(
 872                     model_name=model_name,
 873                     application_name=application_name,
 874                     config=params_dict,
 875                 )
 876                 actions = await self.libjuju.get_actions(
 877                     application_name=application_name, model_name=model_name,
 878                 )
 879                 self.log.debug(
 880                     "Application {} has these actions: {}".format(
 881                         application_name, actions
 882                     )
 883                 )
 884                 if "verify-ssh-credentials" in actions:
 885                     # execute verify-credentials
 886                     num_retries = 20
 887                     retry_timeout = 15.0
 888                     for _ in range(num_retries):
 889                         try:
 890                             self.log.debug("Executing action verify-ssh-credentials...")
 891                             output, ok = await self.libjuju.execute_action(
 892                                 model_name=model_name,
 893                                 application_name=application_name,
 894                                 action_name="verify-ssh-credentials",
 895                                 db_dict=db_dict,
 896                                 progress_timeout=progress_timeout,
 897                                 total_timeout=total_timeout,
 898                             )
 899
 900                             if ok == "failed":
 901                                 self.log.debug(
 902                                     "Error executing verify-ssh-credentials: {}. Retrying..."
 903                                 )
 904                                 await asyncio.sleep(retry_timeout)
 905
 906                                 continue
 907                             self.log.debug("Result: {}, output: {}".format(ok, output))
 908                             break
 909                         except asyncio.CancelledError:
 910                             raise
 911                     else:
 912                         self.log.error(
 913                             "Error executing verify-ssh-credentials after {} retries. ".format(
 914                                 num_retries
 915                             )
 916                         )
 917                 else:
 918                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 919                         application_name
 920                     )
 921                     self.log.debug(msg=msg)
 922             except Exception as e:
 923                 self.log.error("Error configuring juju application: {}".format(e))
 924                 raise N2VCExecutionException(
 925                     message="Error configuring application into ee={} : {}".format(
 926                         ee_id, e
 927                     ),
 928                     primitive_name=primitive_name,
 929                 )
 930             return "CONFIG OK"
 931         else:
 932             try:
 933                 output, status = await self.libjuju.execute_action(
 934                     model_name=model_name,
 935                     application_name=application_name,
 936                     action_name=primitive_name,
 937                     db_dict=db_dict,
 938                     progress_timeout=progress_timeout,
 939                     total_timeout=total_timeout,
 940                     **params_dict
 941                 )
 942                 if status == "completed":
 943                     return output
 944                 else:
 945                     raise Exception("status: {}, output: {}".format(status, output))
 946             except Exception as e:
 947                 self.log.error(
 948                     "Error executing primitive {}: {}".format(primitive_name, e)
 949                 )
 950                 raise N2VCExecutionException(
 951                     message="Error executing primitive {} into ee={} : {}".format(
 952                         primitive_name, ee_id, e
 953                     ),
 954                     primitive_name=primitive_name,
 955                 )
 956
 957     async def disconnect(self):
 958         self.log.info("closing juju N2VC...")
 959         try:
 960             await self.libjuju.disconnect()
 961         except Exception as e:
 962             raise N2VCConnectionException(
 963                 message="Error disconnecting controller: {}".format(e), url=self.url
 964             )
 965
 966     """
 967 ####################################################################################
 968 ################################### P R I V A T E ##################################
 969 ####################################################################################
 970     """
 971
 972     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 973
 974         # write ee_id to database: _admin.deployed.VCA.x
 975         try:
 976             the_table = db_dict["collection"]
 977             the_filter = db_dict["filter"]
 978             the_path = db_dict["path"]
 979             if not the_path[-1] == ".":
 980                 the_path = the_path + "."
 981             update_dict = {the_path + "ee_id": ee_id}
 982             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 983             self.db.set_one(
 984                 table=the_table,
 985                 q_filter=the_filter,
 986                 update_dict=update_dict,
 987                 fail_on_empty=True,
 988             )
 989         except asyncio.CancelledError:
 990             raise
 991         except Exception as e:
 992             self.log.error("Error writing ee_id to database: {}".format(e))
 993
 994     @staticmethod
 995     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 996         """
 997         Build an execution environment id form model, application and machine
 998         :param model_name:
 999         :param application_name:
1000         :param machine_id:
1001         :return:
1002         """
1003         # id for the execution environment
1004         return "{}.{}.{}".format(model_name, application_name, machine_id)
1005
1006     @staticmethod
1007     def _get_ee_id_components(ee_id: str) -> (str, str, str):
1008         """
1009         Get model, application and machine components from an execution environment id
1010         :param ee_id:
1011         :return: model_name, application_name, machine_id
1012         """
1013
1014         if ee_id is None:
1015             return None, None, None
1016
1017         # split components of id
1018         parts = ee_id.split(".")
1019         model_name = parts[0]
1020         application_name = parts[1]
1021         machine_id = parts[2]
1022         return model_name, application_name, machine_id
1023
1024     def _get_application_name(self, namespace: str) -> str:
1025         """
1026         Build application name from namespace
1027         :param namespace:
1028         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1029         """
1030
1031         # TODO: Enforce the Juju 50-character application limit
1032
1033         # split namespace components
1034         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
1035             namespace=namespace
1036         )
1037
1038         if vnf_id is None or len(vnf_id) == 0:
1039             vnf_id = ""
1040         else:
1041             # Shorten the vnf_id to its last twelve characters
1042             vnf_id = "vnf-" + vnf_id[-12:]
1043
1044         if vdu_id is None or len(vdu_id) == 0:
1045             vdu_id = ""
1046         else:
1047             # Shorten the vdu_id to its last twelve characters
1048             vdu_id = "-vdu-" + vdu_id[-12:]
1049
1050         if vdu_count is None or len(vdu_count) == 0:
1051             vdu_count = ""
1052         else:
1053             vdu_count = "-cnt-" + vdu_count
1054
1055         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
1056
1057         return N2VCJujuConnector._format_app_name(application_name)
1058
1059     async def _juju_create_machine(
1060         self,
1061         model_name: str,
1062         application_name: str,
1063         machine_id: str = None,
1064         db_dict: dict = None,
1065         progress_timeout: float = None,
1066         total_timeout: float = None,
1067     ) -> Machine:
1068
1069         self.log.debug(
1070             "creating machine in model: {}, existing machine id: {}".format(
1071                 model_name, machine_id
1072             )
1073         )
1074
1075         # get juju model and observer (create model if needed)
1076         model = await self._juju_get_model(model_name=model_name)
1077         observer = self.juju_observers[model_name]
1078
1079         # find machine id in model
1080         machine = None
1081         if machine_id is not None:
1082             self.log.debug("Finding existing machine id {} in model".format(machine_id))
1083             # get juju existing machines in the model
1084             existing_machines = await model.get_machines()
1085             if machine_id in existing_machines:
1086                 self.log.debug(
1087                     "Machine id {} found in model (reusing it)".format(machine_id)
1088                 )
1089                 machine = model.machines[machine_id]
1090
1091         if machine is None:
1092             self.log.debug("Creating a new machine in juju...")
1093             # machine does not exist, create it and wait for it
1094             machine = await model.add_machine(
1095                 spec=None, constraints=None, disks=None, series="xenial"
1096             )
1097
1098             # register machine with observer
1099             observer.register_machine(machine=machine, db_dict=db_dict)
1100
1101             # id for the execution environment
1102             ee_id = N2VCJujuConnector._build_ee_id(
1103                 model_name=model_name,
1104                 application_name=application_name,
1105                 machine_id=str(machine.entity_id),
1106             )
1107
1108             # write ee_id in database
1109             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
1110
1111             # wait for machine creation
1112             await observer.wait_for_machine(
1113                 machine_id=str(machine.entity_id),
1114                 progress_timeout=progress_timeout,
1115                 total_timeout=total_timeout,
1116             )
1117
1118         else:
1119
1120             self.log.debug("Reusing old machine pending")
1121
1122             # register machine with observer
1123             observer.register_machine(machine=machine, db_dict=db_dict)
1124
1125             # machine does exist, but it is in creation process (pending), wait for
1126             # create finalisation
1127             await observer.wait_for_machine(
1128                 machine_id=machine.entity_id,
1129                 progress_timeout=progress_timeout,
1130                 total_timeout=total_timeout,
1131             )
1132
1133         self.log.debug("Machine ready at " + str(machine.dns_name))
1134         return machine
1135
1136     async def _juju_provision_machine(
1137         self,
1138         model_name: str,
1139         hostname: str,
1140         username: str,
1141         private_key_path: str,
1142         db_dict: dict = None,
1143         progress_timeout: float = None,
1144         total_timeout: float = None,
1145     ) -> str:
1146
1147         if not self.api_proxy:
1148             msg = "Cannot provision machine: api_proxy is not defined"
1149             self.log.error(msg=msg)
1150             raise N2VCException(message=msg)
1151
1152         self.log.debug(
1153             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1154                 model_name, hostname, username
1155             )
1156         )
1157
1158         if not self._authenticated:
1159             await self._juju_login()
1160
1161         # get juju model and observer
1162         model = await self._juju_get_model(model_name=model_name)
1163         observer = self.juju_observers[model_name]
1164
1165         # TODO check if machine is already provisioned
1166         machine_list = await model.get_machines()
1167
1168         provisioner = AsyncSSHProvisioner(
1169             host=hostname,
1170             user=username,
1171             private_key_path=private_key_path,
1172             log=self.log,
1173         )
1174
1175         params = None
1176         try:
1177             params = await provisioner.provision_machine()
1178         except Exception as ex:
1179             msg = "Exception provisioning machine: {}".format(ex)
1180             self.log.error(msg)
1181             raise N2VCException(message=msg)
1182
1183         params.jobs = ["JobHostUnits"]
1184
1185         connection = model.connection()
1186
1187         # Submit the request.
1188         self.log.debug("Adding machine to model")
1189         client_facade = client.ClientFacade.from_connection(connection)
1190         results = await client_facade.AddMachines(params=[params])
1191         error = results.machines[0].error
1192         if error:
1193             msg = "Error adding machine: {}".format(error.message)
1194             self.log.error(msg=msg)
1195             raise ValueError(msg)
1196
1197         machine_id = results.machines[0].machine
1198
1199         # Need to run this after AddMachines has been called,
1200         # as we need the machine_id
1201         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1202         asyncio.ensure_future(
1203             provisioner.install_agent(
1204                 connection=connection,
1205                 nonce=params.nonce,
1206                 machine_id=machine_id,
1207                 proxy=self.api_proxy,
1208             )
1209         )
1210
1211         # wait for machine in model (now, machine is not yet in model, so we must
1212         # wait for it)
1213         machine = None
1214         for _ in range(10):
1215             machine_list = await model.get_machines()
1216             if machine_id in machine_list:
1217                 self.log.debug("Machine {} found in model!".format(machine_id))
1218                 machine = model.machines.get(machine_id)
1219                 break
1220             await asyncio.sleep(2)
1221
1222         if machine is None:
1223             msg = "Machine {} not found in model".format(machine_id)
1224             self.log.error(msg=msg)
1225             raise Exception(msg)
1226
1227         # register machine with observer
1228         observer.register_machine(machine=machine, db_dict=db_dict)
1229
1230         # wait for machine creation
1231         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1232         await observer.wait_for_machine(
1233             machine_id=machine_id,
1234             progress_timeout=progress_timeout,
1235             total_timeout=total_timeout,
1236         )
1237
1238         self.log.debug("Machine provisioned {}".format(machine_id))
1239
1240         return machine_id
1241
1242     async def _juju_deploy_charm(
1243         self,
1244         model_name: str,
1245         application_name: str,
1246         charm_path: str,
1247         machine_id: str,
1248         db_dict: dict,
1249         progress_timeout: float = None,
1250         total_timeout: float = None,
1251         config: dict = None,
1252     ) -> (Application, int):
1253
1254         # get juju model and observer
1255         model = await self._juju_get_model(model_name=model_name)
1256         observer = self.juju_observers[model_name]
1257
1258         # check if application already exists
1259         application = None
1260         if application_name in model.applications:
1261             application = model.applications[application_name]
1262
1263         if application is None:
1264
1265             # application does not exist, create it and wait for it
1266             self.log.debug(
1267                 "deploying application {} to machine {}, model {}".format(
1268                     application_name, machine_id, model_name
1269                 )
1270             )
1271             self.log.debug("charm: {}".format(charm_path))
1272             machine = model.machines[machine_id]
1273             # series = None
1274             application = await model.deploy(
1275                 entity_url=charm_path,
1276                 application_name=application_name,
1277                 channel="stable",
1278                 num_units=1,
1279                 series=machine.series,
1280                 to=machine_id,
1281                 config=config,
1282             )
1283
1284             # register application with observer
1285             observer.register_application(application=application, db_dict=db_dict)
1286
1287             self.log.debug(
1288                 "waiting for application deployed... {}".format(application.entity_id)
1289             )
1290             retries = await observer.wait_for_application(
1291                 application_id=application.entity_id,
1292                 progress_timeout=progress_timeout,
1293                 total_timeout=total_timeout,
1294             )
1295             self.log.debug("application deployed")
1296
1297         else:
1298
1299             # register application with observer
1300             observer.register_application(application=application, db_dict=db_dict)
1301
1302             # application already exists, but not finalised
1303             self.log.debug("application already exists, waiting for deployed...")
1304             retries = await observer.wait_for_application(
1305                 application_id=application.entity_id,
1306                 progress_timeout=progress_timeout,
1307                 total_timeout=total_timeout,
1308             )
1309             self.log.debug("application deployed")
1310
1311         return application, retries
1312
1313     async def _juju_execute_action(
1314         self,
1315         model_name: str,
1316         application_name: str,
1317         action_name: str,
1318         db_dict: dict,
1319         progress_timeout: float = None,
1320         total_timeout: float = None,
1321         **kwargs
1322     ) -> Action:
1323
1324         # get juju model and observer
1325         model = await self._juju_get_model(model_name=model_name)
1326         observer = self.juju_observers[model_name]
1327
1328         application = await self._juju_get_application(
1329             model_name=model_name, application_name=application_name
1330         )
1331
1332         unit = None
1333         for u in application.units:
1334             if await u.is_leader_from_status():
1335                 unit = u
1336         if unit is not None:
1337             actions = await application.get_actions()
1338             if action_name in actions:
1339                 self.log.debug(
1340                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1341                 )
1342                 action = await unit.run_action(action_name, **kwargs)
1343
1344                 # register action with observer
1345                 observer.register_action(action=action, db_dict=db_dict)
1346
1347                 await observer.wait_for_action(
1348                     action_id=action.entity_id,
1349                     progress_timeout=progress_timeout,
1350                     total_timeout=total_timeout,
1351                 )
1352                 self.log.debug("action completed with status: {}".format(action.status))
1353                 output = await model.get_action_output(action_uuid=action.entity_id)
1354                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1355                 if action.entity_id in status:
1356                     status = status[action.entity_id]
1357                 else:
1358                     status = "failed"
1359                 return output, status
1360
1361         raise N2VCExecutionException(
1362             message="Cannot execute action on charm", primitive_name=action_name
1363         )
1364
1365     async def _juju_configure_application(
1366         self,
1367         model_name: str,
1368         application_name: str,
1369         config: dict,
1370         db_dict: dict,
1371         progress_timeout: float = None,
1372         total_timeout: float = None,
1373     ):
1374
1375         # get the application
1376         application = await self._juju_get_application(
1377             model_name=model_name, application_name=application_name
1378         )
1379
1380         self.log.debug(
1381             "configuring the application {} -> {}".format(application_name, config)
1382         )
1383         res = await application.set_config(config)
1384         self.log.debug(
1385             "application {} configured. res={}".format(application_name, res)
1386         )
1387
1388         # Verify the config is set
1389         new_conf = await application.get_config()
1390         for key in config:
1391             value = new_conf[key]["value"]
1392             self.log.debug("    {} = {}".format(key, value))
1393             if config[key] != value:
1394                 raise N2VCException(
1395                     message="key {} is not configured correctly {} != {}".format(
1396                         key, config[key], new_conf[key]
1397                     )
1398                 )
1399
1400         # check if 'verify-ssh-credentials' action exists
1401         # unit = application.units[0]
1402         actions = await application.get_actions()
1403         if "verify-ssh-credentials" not in actions:
1404             msg = (
1405                 "Action verify-ssh-credentials does not exist in application {}"
1406             ).format(application_name)
1407             self.log.debug(msg=msg)
1408             return False
1409
1410         # execute verify-credentials
1411         num_retries = 20
1412         retry_timeout = 15.0
1413         for _ in range(num_retries):
1414             try:
1415                 self.log.debug("Executing action verify-ssh-credentials...")
1416                 output, ok = await self._juju_execute_action(
1417                     model_name=model_name,
1418                     application_name=application_name,
1419                     action_name="verify-ssh-credentials",
1420                     db_dict=db_dict,
1421                     progress_timeout=progress_timeout,
1422                     total_timeout=total_timeout,
1423                 )
1424                 self.log.debug("Result: {}, output: {}".format(ok, output))
1425                 return True
1426             except asyncio.CancelledError:
1427                 raise
1428             except Exception as e:
1429                 self.log.debug(
1430                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1431                 )
1432                 await asyncio.sleep(retry_timeout)
1433         else:
1434             self.log.error(
1435                 "Error executing verify-ssh-credentials after {} retries. ".format(
1436                     num_retries
1437                 )
1438             )
1439             return False
1440
1441     async def _juju_get_application(self, model_name: str, application_name: str):
1442         """Get the deployed application."""
1443
1444         model = await self._juju_get_model(model_name=model_name)
1445
1446         application_name = N2VCJujuConnector._format_app_name(application_name)
1447
1448         if model.applications and application_name in model.applications:
1449             return model.applications[application_name]
1450         else:
1451             raise N2VCException(
1452                 message="Cannot get application {} from model {}".format(
1453                     application_name, model_name
1454                 )
1455             )
1456
1457     async def _juju_get_model(self, model_name: str) -> Model:
1458         """ Get a model object from juju controller
1459         If the model does not exits, it creates it.
1460
1461         :param str model_name: name of the model
1462         :returns Model: model obtained from juju controller or Exception
1463         """
1464
1465         # format model name
1466         model_name = N2VCJujuConnector._format_model_name(model_name)
1467
1468         if model_name in self.juju_models:
1469             return self.juju_models[model_name]
1470
1471         if self._creating_model:
1472             self.log.debug("Another coroutine is creating a model. Wait...")
1473         while self._creating_model:
1474             # another coroutine is creating a model, wait
1475             await asyncio.sleep(0.1)
1476             # retry (perhaps another coroutine has created the model meanwhile)
1477             if model_name in self.juju_models:
1478                 return self.juju_models[model_name]
1479
1480         try:
1481             self._creating_model = True
1482
1483             # get juju model names from juju
1484             model_list = await self.controller.list_models()
1485             if model_name not in model_list:
1486                 self.log.info(
1487                     "Model {} does not exist. Creating new model...".format(model_name)
1488                 )
1489                 config_dict = {"authorized-keys": self.public_key}
1490                 if self.apt_mirror:
1491                     config_dict["apt-mirror"] = self.apt_mirror
1492                 if not self.enable_os_upgrade:
1493                     config_dict["enable-os-refresh-update"] = False
1494                     config_dict["enable-os-upgrade"] = False
1495                 if self.cloud in self.BUILT_IN_CLOUDS:
1496                     model = await self.controller.add_model(
1497                         model_name=model_name,
1498                         config=config_dict,
1499                         cloud_name=self.cloud,
1500                     )
1501                 else:
1502                     model = await self.controller.add_model(
1503                         model_name=model_name,
1504                         config=config_dict,
1505                         cloud_name=self.cloud,
1506                         credential_name=self.cloud,
1507                     )
1508                 self.log.info("New model created, name={}".format(model_name))
1509             else:
1510                 self.log.debug(
1511                     "Model already exists in juju. Getting model {}".format(model_name)
1512                 )
1513                 model = await self.controller.get_model(model_name)
1514                 self.log.debug("Existing model in juju, name={}".format(model_name))
1515
1516             self.juju_models[model_name] = model
1517             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1518             return model
1519
1520         except Exception as e:
1521             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1522             self.log.error(msg)
1523             raise N2VCException(msg)
1524         finally:
1525             self._creating_model = False
1526
1527     async def _juju_add_relation(
1528         self,
1529         model_name: str,
1530         application_name_1: str,
1531         application_name_2: str,
1532         relation_1: str,
1533         relation_2: str,
1534     ):
1535
1536         # get juju model and observer
1537         model = await self._juju_get_model(model_name=model_name)
1538
1539         r1 = "{}:{}".format(application_name_1, relation_1)
1540         r2 = "{}:{}".format(application_name_2, relation_2)
1541
1542         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1543         try:
1544             await model.add_relation(relation1=r1, relation2=r2)
1545         except JujuAPIError as e:
1546             # If one of the applications in the relationship doesn't exist, or the
1547             # relation has already been added,
1548             # let the operation fail silently.
1549             if "not found" in e.message:
1550                 return
1551             if "already exists" in e.message:
1552                 return
1553             # another execption, raise it
1554             raise e
1555
1556     async def _juju_destroy_application(self, model_name: str, application_name: str):
1557
1558         self.log.debug(
1559             "Destroying application {} in model {}".format(application_name, model_name)
1560         )
1561
1562         # get juju model and observer
1563         model = await self._juju_get_model(model_name=model_name)
1564         observer = self.juju_observers[model_name]
1565
1566         application = model.applications.get(application_name)
1567         if application:
1568             observer.unregister_application(application_name)
1569             await application.destroy()
1570         else:
1571             self.log.debug("Application not found: {}".format(application_name))
1572
1573     async def _juju_destroy_machine(
1574         self, model_name: str, machine_id: str, total_timeout: float = None
1575     ):
1576
1577         self.log.debug(
1578             "Destroying machine {} in model {}".format(machine_id, model_name)
1579         )
1580
1581         if total_timeout is None:
1582             total_timeout = 3600
1583
1584         # get juju model and observer
1585         model = await self._juju_get_model(model_name=model_name)
1586         observer = self.juju_observers[model_name]
1587
1588         machines = await model.get_machines()
1589         if machine_id in machines:
1590             machine = model.machines[machine_id]
1591             observer.unregister_machine(machine_id)
1592             # TODO: change this by machine.is_manual when this is upstreamed:
1593             # https://github.com/juju/python-libjuju/pull/396
1594             if "instance-id" in machine.safe_data and machine.safe_data[
1595                 "instance-id"
1596             ].startswith("manual:"):
1597                 self.log.debug("machine.destroy(force=True) started.")
1598                 await machine.destroy(force=True)
1599                 self.log.debug("machine.destroy(force=True) passed.")
1600                 # max timeout
1601                 end = time.time() + total_timeout
1602                 # wait for machine removal
1603                 machines = await model.get_machines()
1604                 while machine_id in machines and time.time() < end:
1605                     self.log.debug(
1606                         "Waiting for machine {} is destroyed".format(machine_id)
1607                     )
1608                     await asyncio.sleep(0.5)
1609                     machines = await model.get_machines()
1610                 self.log.debug("Machine destroyed: {}".format(machine_id))
1611         else:
1612             self.log.debug("Machine not found: {}".format(machine_id))
1613
1614     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1615
1616         self.log.debug("Destroying model {}".format(model_name))
1617
1618         if total_timeout is None:
1619             total_timeout = 3600
1620         end = time.time() + total_timeout
1621
1622         model = await self._juju_get_model(model_name=model_name)
1623
1624         if not model:
1625             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1626
1627         uuid = model.info.uuid
1628
1629         # destroy applications
1630         for application_name in model.applications:
1631             try:
1632                 await self._juju_destroy_application(
1633                     model_name=model_name, application_name=application_name
1634                 )
1635             except Exception as e:
1636                 self.log.error(
1637                     "Error destroying application {} in model {}: {}".format(
1638                         application_name, model_name, e
1639                     )
1640                 )
1641
1642         # destroy machines
1643         machines = await model.get_machines()
1644         for machine_id in machines:
1645             try:
1646                 await self._juju_destroy_machine(
1647                     model_name=model_name, machine_id=machine_id
1648                 )
1649             except asyncio.CancelledError:
1650                 raise
1651             except Exception:
1652                 # ignore exceptions destroying machine
1653                 pass
1654
1655         await self._juju_disconnect_model(model_name=model_name)
1656
1657         self.log.debug("destroying model {}...".format(model_name))
1658         await self.controller.destroy_model(uuid)
1659         # self.log.debug('model destroy requested {}'.format(model_name))
1660
1661         # wait for model is completely destroyed
1662         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1663         last_exception = ""
1664         while time.time() < end:
1665             try:
1666                 # await self.controller.get_model(uuid)
1667                 models = await self.controller.list_models()
1668                 if model_name not in models:
1669                     self.log.debug(
1670                         "The model {} ({}) was destroyed".format(model_name, uuid)
1671                     )
1672                     return
1673             except asyncio.CancelledError:
1674                 raise
1675             except Exception as e:
1676                 last_exception = e
1677             await asyncio.sleep(5)
1678         raise N2VCException(
1679             "Timeout waiting for model {} to be destroyed {}".format(
1680                 model_name, last_exception
1681             )
1682         )
1683
1684     async def _juju_login(self):
1685         """Connect to juju controller
1686
1687         """
1688
1689         # if already authenticated, exit function
1690         if self._authenticated:
1691             return
1692
1693         # if connecting, wait for finish
1694         # another task could be trying to connect in parallel
1695         while self._connecting:
1696             await asyncio.sleep(0.1)
1697
1698         # double check after other task has finished
1699         if self._authenticated:
1700             return
1701
1702         try:
1703             self._connecting = True
1704             self.log.info(
1705                 "connecting to juju controller: {} {}:{}{}".format(
1706                     self.url,
1707                     self.username,
1708                     self.secret[:8] + "...",
1709                     " with ca_cert" if self.ca_cert else "",
1710                 )
1711             )
1712
1713             # Create controller object
1714             self.controller = Controller(loop=self.loop)
1715             # Connect to controller
1716             await self.controller.connect(
1717                 endpoint=self.url,
1718                 username=self.username,
1719                 password=self.secret,
1720                 cacert=self.ca_cert,
1721             )
1722             self._authenticated = True
1723             self.log.info("juju controller connected")
1724         except Exception as e:
1725             message = "Exception connecting to juju: {}".format(e)
1726             self.log.error(message)
1727             raise N2VCConnectionException(message=message, url=self.url)
1728         finally:
1729             self._connecting = False
1730
1731     async def _juju_logout(self):
1732         """Logout of the Juju controller."""
1733         if not self._authenticated:
1734             return False
1735
1736         # disconnect all models
1737         for model_name in self.juju_models:
1738             try:
1739                 await self._juju_disconnect_model(model_name)
1740             except Exception as e:
1741                 self.log.error(
1742                     "Error disconnecting model {} : {}".format(model_name, e)
1743                 )
1744                 # continue with next model...
1745
1746         self.log.info("Disconnecting controller")
1747         try:
1748             await self.controller.disconnect()
1749         except Exception as e:
1750             raise N2VCConnectionException(
1751                 message="Error disconnecting controller: {}".format(e), url=self.url
1752             )
1753
1754         self.controller = None
1755         self._authenticated = False
1756         self.log.info("disconnected")
1757
1758     async def _juju_disconnect_model(self, model_name: str):
1759         self.log.debug("Disconnecting model {}".format(model_name))
1760         if model_name in self.juju_models:
1761             await self.juju_models[model_name].disconnect()
1762             self.juju_models[model_name] = None
1763             self.juju_observers[model_name] = None
1764         else:
1765             self.warning("Cannot disconnect model: {}".format(model_name))
1766
1767     def _create_juju_public_key(self):
1768         """Recreate the Juju public key on lcm container, if needed
1769         Certain libjuju commands expect to be run from the same machine as Juju
1770          is bootstrapped to. This method will write the public key to disk in
1771          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1772         """
1773
1774         # Make sure that we have a public key before writing to disk
1775         if self.public_key is None or len(self.public_key) == 0:
1776             if "OSMLCM_VCA_PUBKEY" in os.environ:
1777                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1778                 if len(self.public_key) == 0:
1779                     return
1780             else:
1781                 return
1782
1783         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1784         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1785         self.log.debug(
1786             "writing juju public key to file:\n{}\npublic key: {}".format(
1787                 file_path, self.public_key
1788             )
1789         )
1790         if not os.path.exists(pk_path):
1791             # create path and write file
1792             os.makedirs(pk_path)
1793             with open(file_path, "w") as f:
1794                 self.log.debug("Creating juju public key file: {}".format(file_path))
1795                 f.write(self.public_key)
1796         else:
1797             self.log.debug("juju public key file already exists: {}".format(file_path))
1798
1799     @staticmethod
1800     def _format_model_name(name: str) -> str:
1801         """Format the name of the model.
1802
1803         Model names may only contain lowercase letters, digits and hyphens
1804         """
1805
1806         return name.replace("_", "-").replace(" ", "-").lower()
1807
1808     @staticmethod
1809     def _format_app_name(name: str) -> str:
1810         """Format the name of the application (in order to assure valid application name).
1811
1812         Application names have restrictions (run juju deploy --help):
1813             - contains lowercase letters 'a'-'z'
1814             - contains numbers '0'-'9'
1815             - contains hyphens '-'
1816             - starts with a lowercase letter
1817             - not two or more consecutive hyphens
1818             - after a hyphen, not a group with all numbers
1819         """
1820
1821         def all_numbers(s: str) -> bool:
1822             for c in s:
1823                 if not c.isdigit():
1824                     return False
1825             return True
1826
1827         new_name = name.replace("_", "-")
1828         new_name = new_name.replace(" ", "-")
1829         new_name = new_name.lower()
1830         while new_name.find("--") >= 0:
1831             new_name = new_name.replace("--", "-")
1832         groups = new_name.split("-")
1833
1834         # find 'all numbers' groups and prefix them with a letter
1835         app_name = ""
1836         for i in range(len(groups)):
1837             group = groups[i]
1838             if all_numbers(group):
1839                 group = "z" + group
1840             if i > 0:
1841                 app_name += "-"
1842             app_name += group
1843
1844         if app_name[0].isdigit():
1845             app_name = "z" + app_name
1846
1847         return app_name