n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46     JujuK8sProxycharmNotSupported,
  47 )
  48 from n2vc.juju_observer import JujuModelObserver
  49 from n2vc.n2vc_conn import N2VCConnector
  50 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  51 from n2vc.provisioner import AsyncSSHProvisioner
  52 from n2vc.libjuju import Libjuju
  53
  54
  55 class N2VCJujuConnector(N2VCConnector):
  56
  57     """
  58 ####################################################################################
  59 ################################### P U B L I C ####################################
  60 ####################################################################################
  61     """
  62
  63     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  64
  65     def __init__(
  66         self,
  67         db: object,
  68         fs: object,
  69         log: object = None,
  70         loop: object = None,
  71         url: str = "127.0.0.1:17070",
  72         username: str = "admin",
  73         vca_config: dict = None,
  74         on_update_db=None,
  75     ):
  76         """Initialize juju N2VC connector
  77         """
  78
  79         # parent class constructor
  80         N2VCConnector.__init__(
  81             self,
  82             db=db,
  83             fs=fs,
  84             log=log,
  85             loop=loop,
  86             url=url,
  87             username=username,
  88             vca_config=vca_config,
  89             on_update_db=on_update_db,
  90         )
  91
  92         # silence websocket traffic log
  93         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  94         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  95         logging.getLogger("model").setLevel(logging.WARN)
  96
  97         self.log.info("Initializing N2VC juju connector...")
  98
  99         """
 100         ##############################################################
 101         # check arguments
 102         ##############################################################
 103         """
 104
 105         # juju URL
 106         if url is None:
 107             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 108         url_parts = url.split(":")
 109         if len(url_parts) != 2:
 110             raise N2VCBadArgumentsException(
 111                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 112             )
 113         self.hostname = url_parts[0]
 114         try:
 115             self.port = int(url_parts[1])
 116         except ValueError:
 117             raise N2VCBadArgumentsException(
 118                 "url port must be a number -> {}".format(url), ["url"]
 119             )
 120
 121         # juju USERNAME
 122         if username is None:
 123             raise N2VCBadArgumentsException(
 124                 "Argument username is mandatory", ["username"]
 125             )
 126
 127         # juju CONFIGURATION
 128         if vca_config is None:
 129             raise N2VCBadArgumentsException(
 130                 "Argument vca_config is mandatory", ["vca_config"]
 131             )
 132
 133         if "secret" in vca_config:
 134             self.secret = vca_config["secret"]
 135         else:
 136             raise N2VCBadArgumentsException(
 137                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 138             )
 139
 140         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 141         # if exists, it will be written in lcm container: _create_juju_public_key()
 142         if "public_key" in vca_config:
 143             self.public_key = vca_config["public_key"]
 144         else:
 145             self.public_key = None
 146
 147         # TODO: Verify ca_cert is valid before using. VCA will crash
 148         # if the ca_cert isn't formatted correctly.
 149         def base64_to_cacert(b64string):
 150             """Convert the base64-encoded string containing the VCA CACERT.
 151
 152             The input string....
 153
 154             """
 155             try:
 156                 cacert = base64.b64decode(b64string).decode("utf-8")
 157
 158                 cacert = re.sub(r"\\n", r"\n", cacert,)
 159             except binascii.Error as e:
 160                 self.log.debug("Caught binascii.Error: {}".format(e))
 161                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 162
 163             return cacert
 164
 165         self.ca_cert = vca_config.get("ca_cert")
 166         if self.ca_cert:
 167             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 168
 169         if "api_proxy" in vca_config:
 170             self.api_proxy = vca_config["api_proxy"]
 171             self.log.debug(
 172                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 173             )
 174         else:
 175             self.warning(
 176                 "api_proxy is not configured. Support for native charms is disabled"
 177             )
 178             self.api_proxy = None
 179
 180         if "enable_os_upgrade" in vca_config:
 181             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 182         else:
 183             self.enable_os_upgrade = True
 184
 185         if "apt_mirror" in vca_config:
 186             self.apt_mirror = vca_config["apt_mirror"]
 187         else:
 188             self.apt_mirror = None
 189
 190         self.cloud = vca_config.get('cloud')
 191         self.k8s_cloud = None
 192         if "k8s_cloud" in vca_config:
 193             self.k8s_cloud = vca_config.get("k8s_cloud")
 194         self.log.debug('Arguments have been checked')
 195
 196         # juju data
 197         self.controller = None  # it will be filled when connect to juju
 198         self.juju_models = {}  # model objects for every model_name
 199         self.juju_observers = {}  # model observers for every model_name
 200         self._connecting = (
 201             False  # while connecting to juju (to avoid duplicate connections)
 202         )
 203         self._authenticated = (
 204             False  # it will be True when juju connection be stablished
 205         )
 206         self._creating_model = False  # True during model creation
 207         self.libjuju = Libjuju(
 208             endpoint=self.url,
 209             api_proxy=self.api_proxy,
 210             enable_os_upgrade=self.enable_os_upgrade,
 211             apt_mirror=self.apt_mirror,
 212             username=self.username,
 213             password=self.secret,
 214             cacert=self.ca_cert,
 215             loop=self.loop,
 216             log=self.log,
 217             db=self.db,
 218             n2vc=self,
 219         )
 220
 221         # create juju pub key file in lcm container at
 222         # ./local/share/juju/ssh/juju_id_rsa.pub
 223         self._create_juju_public_key()
 224
 225         self.log.info("N2VC juju connector initialized")
 226
 227     async def get_status(self, namespace: str, yaml_format: bool = True):
 228
 229         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 230
 231         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 232             namespace=namespace
 233         )
 234         # model name is ns_id
 235         model_name = ns_id
 236         if model_name is None:
 237             msg = "Namespace {} not valid".format(namespace)
 238             self.log.error(msg)
 239             raise N2VCBadArgumentsException(msg, ["namespace"])
 240
 241         status = {}
 242         models = await self.libjuju.list_models(contains=ns_id)
 243
 244         for m in models:
 245             status[m] = await self.libjuju.get_model_status(m)
 246
 247         if yaml_format:
 248             return obj_to_yaml(status)
 249         else:
 250             return obj_to_dict(status)
 251
 252     async def create_execution_environment(
 253         self,
 254         namespace: str,
 255         db_dict: dict,
 256         reuse_ee_id: str = None,
 257         progress_timeout: float = None,
 258         total_timeout: float = None,
 259     ) -> (str, dict):
 260
 261         self.log.info(
 262             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 263                 namespace, reuse_ee_id
 264             )
 265         )
 266
 267         machine_id = None
 268         if reuse_ee_id:
 269             model_name, application_name, machine_id = self._get_ee_id_components(
 270                 ee_id=reuse_ee_id
 271             )
 272         else:
 273             (
 274                 _nsi_id,
 275                 ns_id,
 276                 _vnf_id,
 277                 _vdu_id,
 278                 _vdu_count,
 279             ) = self._get_namespace_components(namespace=namespace)
 280             # model name is ns_id
 281             model_name = ns_id
 282             # application name
 283             application_name = self._get_application_name(namespace=namespace)
 284
 285         self.log.debug(
 286             "model name: {}, application name:  {}, machine_id: {}".format(
 287                 model_name, application_name, machine_id
 288             )
 289         )
 290
 291         # create or reuse a new juju machine
 292         try:
 293             if not await self.libjuju.model_exists(model_name):
 294                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 295             machine, new = await self.libjuju.create_machine(
 296                 model_name=model_name,
 297                 machine_id=machine_id,
 298                 db_dict=db_dict,
 299                 progress_timeout=progress_timeout,
 300                 total_timeout=total_timeout,
 301             )
 302             # id for the execution environment
 303             ee_id = N2VCJujuConnector._build_ee_id(
 304                 model_name=model_name,
 305                 application_name=application_name,
 306                 machine_id=str(machine.entity_id),
 307             )
 308             self.log.debug("ee_id: {}".format(ee_id))
 309
 310             if new:
 311                 # write ee_id in database
 312                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 313
 314         except Exception as e:
 315             message = "Error creating machine on juju: {}".format(e)
 316             self.log.error(message)
 317             raise N2VCException(message=message)
 318
 319         # new machine credentials
 320         credentials = {
 321             "hostname": machine.dns_name,
 322         }
 323
 324         self.log.info(
 325             "Execution environment created. ee_id: {}, credentials: {}".format(
 326                 ee_id, credentials
 327             )
 328         )
 329
 330         return ee_id, credentials
 331
 332     async def register_execution_environment(
 333         self,
 334         namespace: str,
 335         credentials: dict,
 336         db_dict: dict,
 337         progress_timeout: float = None,
 338         total_timeout: float = None,
 339     ) -> str:
 340
 341         self.log.info(
 342             "Registering execution environment. namespace={}, credentials={}".format(
 343                 namespace, credentials
 344             )
 345         )
 346
 347         if credentials is None:
 348             raise N2VCBadArgumentsException(
 349                 message="credentials are mandatory", bad_args=["credentials"]
 350             )
 351         if credentials.get("hostname"):
 352             hostname = credentials["hostname"]
 353         else:
 354             raise N2VCBadArgumentsException(
 355                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 356             )
 357         if credentials.get("username"):
 358             username = credentials["username"]
 359         else:
 360             raise N2VCBadArgumentsException(
 361                 message="username is mandatory", bad_args=["credentials.username"]
 362             )
 363         if "private_key_path" in credentials:
 364             private_key_path = credentials["private_key_path"]
 365         else:
 366             # if not passed as argument, use generated private key path
 367             private_key_path = self.private_key_path
 368
 369         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 370             namespace=namespace
 371         )
 372
 373         # model name
 374         model_name = ns_id
 375         # application name
 376         application_name = self._get_application_name(namespace=namespace)
 377
 378         # register machine on juju
 379         try:
 380             if not self.api_proxy:
 381                 msg = "Cannot provision machine: api_proxy is not defined"
 382                 self.log.error(msg=msg)
 383                 raise N2VCException(message=msg)
 384             if not await self.libjuju.model_exists(model_name):
 385                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 386             machine_id = await self.libjuju.provision_machine(
 387                 model_name=model_name,
 388                 hostname=hostname,
 389                 username=username,
 390                 private_key_path=private_key_path,
 391                 db_dict=db_dict,
 392                 progress_timeout=progress_timeout,
 393                 total_timeout=total_timeout,
 394             )
 395         except Exception as e:
 396             self.log.error("Error registering machine: {}".format(e))
 397             raise N2VCException(
 398                 message="Error registering machine on juju: {}".format(e)
 399             )
 400
 401         self.log.info("Machine registered: {}".format(machine_id))
 402
 403         # id for the execution environment
 404         ee_id = N2VCJujuConnector._build_ee_id(
 405             model_name=model_name,
 406             application_name=application_name,
 407             machine_id=str(machine_id),
 408         )
 409
 410         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 411
 412         return ee_id
 413
 414     async def install_configuration_sw(
 415         self,
 416         ee_id: str,
 417         artifact_path: str,
 418         db_dict: dict,
 419         progress_timeout: float = None,
 420         total_timeout: float = None,
 421         config: dict = None,
 422         num_units: int = 1,
 423     ):
 424
 425         self.log.info(
 426             (
 427                 "Installing configuration sw on ee_id: {}, "
 428                 "artifact path: {}, db_dict: {}"
 429             ).format(ee_id, artifact_path, db_dict)
 430         )
 431
 432         # check arguments
 433         if ee_id is None or len(ee_id) == 0:
 434             raise N2VCBadArgumentsException(
 435                 message="ee_id is mandatory", bad_args=["ee_id"]
 436             )
 437         if artifact_path is None or len(artifact_path) == 0:
 438             raise N2VCBadArgumentsException(
 439                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 440             )
 441         if db_dict is None:
 442             raise N2VCBadArgumentsException(
 443                 message="db_dict is mandatory", bad_args=["db_dict"]
 444             )
 445
 446         try:
 447             (
 448                 model_name,
 449                 application_name,
 450                 machine_id,
 451             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 452             self.log.debug(
 453                 "model: {}, application: {}, machine: {}".format(
 454                     model_name, application_name, machine_id
 455                 )
 456             )
 457         except Exception:
 458             raise N2VCBadArgumentsException(
 459                 message="ee_id={} is not a valid execution environment id".format(
 460                     ee_id
 461                 ),
 462                 bad_args=["ee_id"],
 463             )
 464
 465         # remove // in charm path
 466         while artifact_path.find("//") >= 0:
 467             artifact_path = artifact_path.replace("//", "/")
 468
 469         # check charm path
 470         if not self.fs.file_exists(artifact_path, mode="dir"):
 471             msg = "artifact path does not exist: {}".format(artifact_path)
 472             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 473
 474         if artifact_path.startswith("/"):
 475             full_path = self.fs.path + artifact_path
 476         else:
 477             full_path = self.fs.path + "/" + artifact_path
 478
 479         try:
 480             await self.libjuju.deploy_charm(
 481                 model_name=model_name,
 482                 application_name=application_name,
 483                 path=full_path,
 484                 machine_id=machine_id,
 485                 db_dict=db_dict,
 486                 progress_timeout=progress_timeout,
 487                 total_timeout=total_timeout,
 488                 config=config,
 489                 num_units=num_units,
 490             )
 491         except Exception as e:
 492             raise N2VCException(
 493                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 494             )
 495
 496         self.log.info("Configuration sw installed")
 497
 498     async def install_k8s_proxy_charm(
 499         self,
 500         charm_name: str,
 501         namespace: str,
 502         artifact_path: str,
 503         db_dict: dict,
 504         progress_timeout: float = None,
 505         total_timeout: float = None,
 506         config: dict = None,
 507     ) -> str:
 508         """
 509         Install a k8s proxy charm
 510
 511         :param charm_name: Name of the charm being deployed
 512         :param namespace: collection of all the uuids related to the charm.
 513         :param str artifact_path: where to locate the artifacts (parent folder) using
 514             the self.fs
 515             the final artifact path will be a combination of this artifact_path and
 516             additional string from the config_dict (e.g. charm name)
 517         :param dict db_dict: where to write into database when the status changes.
 518                         It contains a dict with
 519                             {collection: <str>, filter: {},  path: <str>},
 520                             e.g. {collection: "nsrs", filter:
 521                                 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
 522         :param float progress_timeout:
 523         :param float total_timeout:
 524         :param config: Dictionary with additional configuration
 525
 526         :returns ee_id: execution environment id.
 527         """
 528         self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
 529                       .format(charm_name, artifact_path, db_dict))
 530
 531         if not self.k8s_cloud:
 532             raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
 533
 534         if artifact_path is None or len(artifact_path) == 0:
 535             raise N2VCBadArgumentsException(
 536                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 537             )
 538         if db_dict is None:
 539             raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict'])
 540
 541         # remove // in charm path
 542         while artifact_path.find('//') >= 0:
 543             artifact_path = artifact_path.replace('//', '/')
 544
 545         # check charm path
 546         if not self.fs.file_exists(artifact_path, mode="dir"):
 547             msg = 'artifact path does not exist: {}'.format(artifact_path)
 548             raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path'])
 549
 550         if artifact_path.startswith('/'):
 551             full_path = self.fs.path + artifact_path
 552         else:
 553             full_path = self.fs.path + '/' + artifact_path
 554
 555         _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace)
 556         model_name = '{}-k8s'.format(ns_id)
 557
 558         await self.libjuju.add_model(model_name, self.k8s_cloud)
 559         application_name = self._get_application_name(namespace)
 560
 561         try:
 562             await self.libjuju.deploy_charm(
 563                 model_name=model_name,
 564                 application_name=application_name,
 565                 path=full_path,
 566                 machine_id=None,
 567                 db_dict=db_dict,
 568                 progress_timeout=progress_timeout,
 569                 total_timeout=total_timeout,
 570                 config=config
 571             )
 572         except Exception as e:
 573             raise N2VCException(message='Error deploying charm: {}'.format(e))
 574
 575         self.log.info('K8s proxy charm installed')
 576         ee_id = N2VCJujuConnector._build_ee_id(
 577             model_name=model_name,
 578             application_name=application_name,
 579             machine_id="k8s",
 580         )
 581
 582         self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 583
 584         return ee_id
 585
 586     async def get_ee_ssh_public__key(
 587         self,
 588         ee_id: str,
 589         db_dict: dict,
 590         progress_timeout: float = None,
 591         total_timeout: float = None,
 592     ) -> str:
 593
 594         self.log.info(
 595             (
 596                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 597             ).format(ee_id, db_dict)
 598         )
 599
 600         # check arguments
 601         if ee_id is None or len(ee_id) == 0:
 602             raise N2VCBadArgumentsException(
 603                 message="ee_id is mandatory", bad_args=["ee_id"]
 604             )
 605         if db_dict is None:
 606             raise N2VCBadArgumentsException(
 607                 message="db_dict is mandatory", bad_args=["db_dict"]
 608             )
 609
 610         try:
 611             (
 612                 model_name,
 613                 application_name,
 614                 machine_id,
 615             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 616             self.log.debug(
 617                 "model: {}, application: {}, machine: {}".format(
 618                     model_name, application_name, machine_id
 619                 )
 620             )
 621         except Exception:
 622             raise N2VCBadArgumentsException(
 623                 message="ee_id={} is not a valid execution environment id".format(
 624                     ee_id
 625                 ),
 626                 bad_args=["ee_id"],
 627             )
 628
 629         # try to execute ssh layer primitives (if exist):
 630         #       generate-ssh-key
 631         #       get-ssh-public-key
 632
 633         output = None
 634
 635         application_name = N2VCJujuConnector._format_app_name(application_name)
 636
 637         # execute action: generate-ssh-key
 638         try:
 639             output, _status = await self.libjuju.execute_action(
 640                 model_name=model_name,
 641                 application_name=application_name,
 642                 action_name="generate-ssh-key",
 643                 db_dict=db_dict,
 644                 progress_timeout=progress_timeout,
 645                 total_timeout=total_timeout,
 646             )
 647         except Exception as e:
 648             self.log.info(
 649                 "Skipping exception while executing action generate-ssh-key: {}".format(
 650                     e
 651                 )
 652             )
 653
 654         # execute action: get-ssh-public-key
 655         try:
 656             output, _status = await self.libjuju.execute_action(
 657                 model_name=model_name,
 658                 application_name=application_name,
 659                 action_name="get-ssh-public-key",
 660                 db_dict=db_dict,
 661                 progress_timeout=progress_timeout,
 662                 total_timeout=total_timeout,
 663             )
 664         except Exception as e:
 665             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 666             self.log.info(msg)
 667             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 668
 669         # return public key if exists
 670         return output["pubkey"] if "pubkey" in output else output
 671
 672     async def add_relation(
 673         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 674     ):
 675
 676         self.log.debug(
 677             "adding new relation between {} and {}, endpoints: {}, {}".format(
 678                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 679             )
 680         )
 681
 682         # check arguments
 683         if not ee_id_1:
 684             message = "EE 1 is mandatory"
 685             self.log.error(message)
 686             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 687         if not ee_id_2:
 688             message = "EE 2 is mandatory"
 689             self.log.error(message)
 690             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 691         if not endpoint_1:
 692             message = "endpoint 1 is mandatory"
 693             self.log.error(message)
 694             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 695         if not endpoint_2:
 696             message = "endpoint 2 is mandatory"
 697             self.log.error(message)
 698             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 699
 700         # get the model, the applications and the machines from the ee_id's
 701         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 702         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 703
 704         # model must be the same
 705         if model_1 != model_2:
 706             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 707             self.log.error(message)
 708             raise N2VCBadArgumentsException(
 709                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 710             )
 711
 712         # add juju relations between two applications
 713         try:
 714             await self.libjuju.add_relation(
 715                 model_name=model_1,
 716                 application_name_1=app_1,
 717                 application_name_2=app_2,
 718                 relation_1=endpoint_1,
 719                 relation_2=endpoint_2,
 720             )
 721         except Exception as e:
 722             message = "Error adding relation between {} and {}: {}".format(
 723                 ee_id_1, ee_id_2, e
 724             )
 725             self.log.error(message)
 726             raise N2VCException(message=message)
 727
 728     async def remove_relation(self):
 729         # TODO
 730         self.log.info("Method not implemented yet")
 731         raise MethodNotImplemented()
 732
 733     async def deregister_execution_environments(self):
 734         self.log.info("Method not implemented yet")
 735         raise MethodNotImplemented()
 736
 737     async def delete_namespace(
 738         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 739     ):
 740         self.log.info("Deleting namespace={}".format(namespace))
 741
 742         # check arguments
 743         if namespace is None:
 744             raise N2VCBadArgumentsException(
 745                 message="namespace is mandatory", bad_args=["namespace"]
 746             )
 747
 748         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 749             namespace=namespace
 750         )
 751         if ns_id is not None:
 752             try:
 753                 models = await self.libjuju.list_models(contains=ns_id)
 754                 for model in models:
 755                     await self.libjuju.destroy_model(
 756                         model_name=model, total_timeout=total_timeout
 757                     )
 758             except Exception as e:
 759                 raise N2VCException(
 760                     message="Error deleting namespace {} : {}".format(namespace, e)
 761                 )
 762         else:
 763             raise N2VCBadArgumentsException(
 764                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 765             )
 766
 767         self.log.info("Namespace {} deleted".format(namespace))
 768
 769     async def delete_execution_environment(
 770         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 771     ):
 772         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 773
 774         # check arguments
 775         if ee_id is None:
 776             raise N2VCBadArgumentsException(
 777                 message="ee_id is mandatory", bad_args=["ee_id"]
 778             )
 779
 780         model_name, application_name, _machine_id = self._get_ee_id_components(
 781             ee_id=ee_id
 782         )
 783
 784         # destroy the application
 785         try:
 786             await self.libjuju.destroy_model(
 787                 model_name=model_name, total_timeout=total_timeout
 788             )
 789         except Exception as e:
 790             raise N2VCException(
 791                 message=(
 792                     "Error deleting execution environment {} (application {}) : {}"
 793                 ).format(ee_id, application_name, e)
 794             )
 795
 796         # destroy the machine
 797         # try:
 798         #     await self._juju_destroy_machine(
 799         #         model_name=model_name,
 800         #         machine_id=machine_id,
 801         #         total_timeout=total_timeout
 802         #     )
 803         # except Exception as e:
 804         #     raise N2VCException(
 805         #        message='Error deleting execution environment {} (machine {}) : {}'
 806         #                .format(ee_id, machine_id, e))
 807
 808         self.log.info("Execution environment {} deleted".format(ee_id))
 809
 810     async def exec_primitive(
 811         self,
 812         ee_id: str,
 813         primitive_name: str,
 814         params_dict: dict,
 815         db_dict: dict = None,
 816         progress_timeout: float = None,
 817         total_timeout: float = None,
 818     ) -> str:
 819
 820         self.log.info(
 821             "Executing primitive: {} on ee: {}, params: {}".format(
 822                 primitive_name, ee_id, params_dict
 823             )
 824         )
 825
 826         # check arguments
 827         if ee_id is None or len(ee_id) == 0:
 828             raise N2VCBadArgumentsException(
 829                 message="ee_id is mandatory", bad_args=["ee_id"]
 830             )
 831         if primitive_name is None or len(primitive_name) == 0:
 832             raise N2VCBadArgumentsException(
 833                 message="action_name is mandatory", bad_args=["action_name"]
 834             )
 835         if params_dict is None:
 836             params_dict = dict()
 837
 838         try:
 839             (
 840                 model_name,
 841                 application_name,
 842                 _machine_id,
 843             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 844         except Exception:
 845             raise N2VCBadArgumentsException(
 846                 message="ee_id={} is not a valid execution environment id".format(
 847                     ee_id
 848                 ),
 849                 bad_args=["ee_id"],
 850             )
 851
 852         if primitive_name == "config":
 853             # Special case: config primitive
 854             try:
 855                 await self.libjuju.configure_application(
 856                     model_name=model_name,
 857                     application_name=application_name,
 858                     config=params_dict,
 859                 )
 860                 actions = await self.libjuju.get_actions(
 861                     application_name=application_name, model_name=model_name,
 862                 )
 863                 self.log.debug(
 864                     "Application {} has these actions: {}".format(
 865                         application_name, actions
 866                     )
 867                 )
 868                 if "verify-ssh-credentials" in actions:
 869                     # execute verify-credentials
 870                     num_retries = 20
 871                     retry_timeout = 15.0
 872                     for _ in range(num_retries):
 873                         try:
 874                             self.log.debug("Executing action verify-ssh-credentials...")
 875                             output, ok = await self.libjuju.execute_action(
 876                                 model_name=model_name,
 877                                 application_name=application_name,
 878                                 action_name="verify-ssh-credentials",
 879                                 db_dict=db_dict,
 880                                 progress_timeout=progress_timeout,
 881                                 total_timeout=total_timeout,
 882                             )
 883
 884                             if ok == "failed":
 885                                 self.log.debug(
 886                                     "Error executing verify-ssh-credentials: {}. Retrying..."
 887                                 )
 888                                 await asyncio.sleep(retry_timeout)
 889
 890                                 continue
 891                             self.log.debug("Result: {}, output: {}".format(ok, output))
 892                             break
 893                         except asyncio.CancelledError:
 894                             raise
 895                     else:
 896                         self.log.error(
 897                             "Error executing verify-ssh-credentials after {} retries. ".format(
 898                                 num_retries
 899                             )
 900                         )
 901                 else:
 902                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 903                         application_name
 904                     )
 905                     self.log.debug(msg=msg)
 906             except Exception as e:
 907                 self.log.error("Error configuring juju application: {}".format(e))
 908                 raise N2VCExecutionException(
 909                     message="Error configuring application into ee={} : {}".format(
 910                         ee_id, e
 911                     ),
 912                     primitive_name=primitive_name,
 913                 )
 914             return "CONFIG OK"
 915         else:
 916             try:
 917                 output, status = await self.libjuju.execute_action(
 918                     model_name=model_name,
 919                     application_name=application_name,
 920                     action_name=primitive_name,
 921                     db_dict=db_dict,
 922                     progress_timeout=progress_timeout,
 923                     total_timeout=total_timeout,
 924                     **params_dict
 925                 )
 926                 if status == "completed":
 927                     return output
 928                 else:
 929                     raise Exception("status is not completed: {}".format(status))
 930             except Exception as e:
 931                 self.log.error(
 932                     "Error executing primitive {}: {}".format(primitive_name, e)
 933                 )
 934                 raise N2VCExecutionException(
 935                     message="Error executing primitive {} into ee={} : {}".format(
 936                         primitive_name, ee_id, e
 937                     ),
 938                     primitive_name=primitive_name,
 939                 )
 940
 941     async def disconnect(self):
 942         self.log.info("closing juju N2VC...")
 943         try:
 944             await self.libjuju.disconnect()
 945         except Exception as e:
 946             raise N2VCConnectionException(
 947                 message="Error disconnecting controller: {}".format(e), url=self.url
 948             )
 949
 950     """
 951 ####################################################################################
 952 ################################### P R I V A T E ##################################
 953 ####################################################################################
 954     """
 955
 956     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 957
 958         # write ee_id to database: _admin.deployed.VCA.x
 959         try:
 960             the_table = db_dict["collection"]
 961             the_filter = db_dict["filter"]
 962             the_path = db_dict["path"]
 963             if not the_path[-1] == ".":
 964                 the_path = the_path + "."
 965             update_dict = {the_path + "ee_id": ee_id}
 966             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 967             self.db.set_one(
 968                 table=the_table,
 969                 q_filter=the_filter,
 970                 update_dict=update_dict,
 971                 fail_on_empty=True,
 972             )
 973         except asyncio.CancelledError:
 974             raise
 975         except Exception as e:
 976             self.log.error("Error writing ee_id to database: {}".format(e))
 977
 978     @staticmethod
 979     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 980         """
 981         Build an execution environment id form model, application and machine
 982         :param model_name:
 983         :param application_name:
 984         :param machine_id:
 985         :return:
 986         """
 987         # id for the execution environment
 988         return "{}.{}.{}".format(model_name, application_name, machine_id)
 989
 990     @staticmethod
 991     def _get_ee_id_components(ee_id: str) -> (str, str, str):
 992         """
 993         Get model, application and machine components from an execution environment id
 994         :param ee_id:
 995         :return: model_name, application_name, machine_id
 996         """
 997
 998         if ee_id is None:
 999             return None, None, None
1000
1001         # split components of id
1002         parts = ee_id.split(".")
1003         model_name = parts[0]
1004         application_name = parts[1]
1005         machine_id = parts[2]
1006         return model_name, application_name, machine_id
1007
1008     def _get_application_name(self, namespace: str) -> str:
1009         """
1010         Build application name from namespace
1011         :param namespace:
1012         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1013         """
1014
1015         # TODO: Enforce the Juju 50-character application limit
1016
1017         # split namespace components
1018         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
1019             namespace=namespace
1020         )
1021
1022         if vnf_id is None or len(vnf_id) == 0:
1023             vnf_id = ""
1024         else:
1025             # Shorten the vnf_id to its last twelve characters
1026             vnf_id = "vnf-" + vnf_id[-12:]
1027
1028         if vdu_id is None or len(vdu_id) == 0:
1029             vdu_id = ""
1030         else:
1031             # Shorten the vdu_id to its last twelve characters
1032             vdu_id = "-vdu-" + vdu_id[-12:]
1033
1034         if vdu_count is None or len(vdu_count) == 0:
1035             vdu_count = ""
1036         else:
1037             vdu_count = "-cnt-" + vdu_count
1038
1039         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
1040
1041         return N2VCJujuConnector._format_app_name(application_name)
1042
1043     async def _juju_create_machine(
1044         self,
1045         model_name: str,
1046         application_name: str,
1047         machine_id: str = None,
1048         db_dict: dict = None,
1049         progress_timeout: float = None,
1050         total_timeout: float = None,
1051     ) -> Machine:
1052
1053         self.log.debug(
1054             "creating machine in model: {}, existing machine id: {}".format(
1055                 model_name, machine_id
1056             )
1057         )
1058
1059         # get juju model and observer (create model if needed)
1060         model = await self._juju_get_model(model_name=model_name)
1061         observer = self.juju_observers[model_name]
1062
1063         # find machine id in model
1064         machine = None
1065         if machine_id is not None:
1066             self.log.debug("Finding existing machine id {} in model".format(machine_id))
1067             # get juju existing machines in the model
1068             existing_machines = await model.get_machines()
1069             if machine_id in existing_machines:
1070                 self.log.debug(
1071                     "Machine id {} found in model (reusing it)".format(machine_id)
1072                 )
1073                 machine = model.machines[machine_id]
1074
1075         if machine is None:
1076             self.log.debug("Creating a new machine in juju...")
1077             # machine does not exist, create it and wait for it
1078             machine = await model.add_machine(
1079                 spec=None, constraints=None, disks=None, series="xenial"
1080             )
1081
1082             # register machine with observer
1083             observer.register_machine(machine=machine, db_dict=db_dict)
1084
1085             # id for the execution environment
1086             ee_id = N2VCJujuConnector._build_ee_id(
1087                 model_name=model_name,
1088                 application_name=application_name,
1089                 machine_id=str(machine.entity_id),
1090             )
1091
1092             # write ee_id in database
1093             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
1094
1095             # wait for machine creation
1096             await observer.wait_for_machine(
1097                 machine_id=str(machine.entity_id),
1098                 progress_timeout=progress_timeout,
1099                 total_timeout=total_timeout,
1100             )
1101
1102         else:
1103
1104             self.log.debug("Reusing old machine pending")
1105
1106             # register machine with observer
1107             observer.register_machine(machine=machine, db_dict=db_dict)
1108
1109             # machine does exist, but it is in creation process (pending), wait for
1110             # create finalisation
1111             await observer.wait_for_machine(
1112                 machine_id=machine.entity_id,
1113                 progress_timeout=progress_timeout,
1114                 total_timeout=total_timeout,
1115             )
1116
1117         self.log.debug("Machine ready at " + str(machine.dns_name))
1118         return machine
1119
1120     async def _juju_provision_machine(
1121         self,
1122         model_name: str,
1123         hostname: str,
1124         username: str,
1125         private_key_path: str,
1126         db_dict: dict = None,
1127         progress_timeout: float = None,
1128         total_timeout: float = None,
1129     ) -> str:
1130
1131         if not self.api_proxy:
1132             msg = "Cannot provision machine: api_proxy is not defined"
1133             self.log.error(msg=msg)
1134             raise N2VCException(message=msg)
1135
1136         self.log.debug(
1137             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1138                 model_name, hostname, username
1139             )
1140         )
1141
1142         if not self._authenticated:
1143             await self._juju_login()
1144
1145         # get juju model and observer
1146         model = await self._juju_get_model(model_name=model_name)
1147         observer = self.juju_observers[model_name]
1148
1149         # TODO check if machine is already provisioned
1150         machine_list = await model.get_machines()
1151
1152         provisioner = AsyncSSHProvisioner(
1153             host=hostname,
1154             user=username,
1155             private_key_path=private_key_path,
1156             log=self.log,
1157         )
1158
1159         params = None
1160         try:
1161             params = await provisioner.provision_machine()
1162         except Exception as ex:
1163             msg = "Exception provisioning machine: {}".format(ex)
1164             self.log.error(msg)
1165             raise N2VCException(message=msg)
1166
1167         params.jobs = ["JobHostUnits"]
1168
1169         connection = model.connection()
1170
1171         # Submit the request.
1172         self.log.debug("Adding machine to model")
1173         client_facade = client.ClientFacade.from_connection(connection)
1174         results = await client_facade.AddMachines(params=[params])
1175         error = results.machines[0].error
1176         if error:
1177             msg = "Error adding machine: {}".format(error.message)
1178             self.log.error(msg=msg)
1179             raise ValueError(msg)
1180
1181         machine_id = results.machines[0].machine
1182
1183         # Need to run this after AddMachines has been called,
1184         # as we need the machine_id
1185         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1186         asyncio.ensure_future(
1187             provisioner.install_agent(
1188                 connection=connection,
1189                 nonce=params.nonce,
1190                 machine_id=machine_id,
1191                 api=self.api_proxy,
1192             )
1193         )
1194
1195         # wait for machine in model (now, machine is not yet in model, so we must
1196         # wait for it)
1197         machine = None
1198         for _ in range(10):
1199             machine_list = await model.get_machines()
1200             if machine_id in machine_list:
1201                 self.log.debug("Machine {} found in model!".format(machine_id))
1202                 machine = model.machines.get(machine_id)
1203                 break
1204             await asyncio.sleep(2)
1205
1206         if machine is None:
1207             msg = "Machine {} not found in model".format(machine_id)
1208             self.log.error(msg=msg)
1209             raise Exception(msg)
1210
1211         # register machine with observer
1212         observer.register_machine(machine=machine, db_dict=db_dict)
1213
1214         # wait for machine creation
1215         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1216         await observer.wait_for_machine(
1217             machine_id=machine_id,
1218             progress_timeout=progress_timeout,
1219             total_timeout=total_timeout,
1220         )
1221
1222         self.log.debug("Machine provisioned {}".format(machine_id))
1223
1224         return machine_id
1225
1226     async def _juju_deploy_charm(
1227         self,
1228         model_name: str,
1229         application_name: str,
1230         charm_path: str,
1231         machine_id: str,
1232         db_dict: dict,
1233         progress_timeout: float = None,
1234         total_timeout: float = None,
1235         config: dict = None,
1236     ) -> (Application, int):
1237
1238         # get juju model and observer
1239         model = await self._juju_get_model(model_name=model_name)
1240         observer = self.juju_observers[model_name]
1241
1242         # check if application already exists
1243         application = None
1244         if application_name in model.applications:
1245             application = model.applications[application_name]
1246
1247         if application is None:
1248
1249             # application does not exist, create it and wait for it
1250             self.log.debug(
1251                 "deploying application {} to machine {}, model {}".format(
1252                     application_name, machine_id, model_name
1253                 )
1254             )
1255             self.log.debug("charm: {}".format(charm_path))
1256             machine = model.machines[machine_id]
1257             # series = None
1258             application = await model.deploy(
1259                 entity_url=charm_path,
1260                 application_name=application_name,
1261                 channel="stable",
1262                 num_units=1,
1263                 series=machine.series,
1264                 to=machine_id,
1265                 config=config,
1266             )
1267
1268             # register application with observer
1269             observer.register_application(application=application, db_dict=db_dict)
1270
1271             self.log.debug(
1272                 "waiting for application deployed... {}".format(application.entity_id)
1273             )
1274             retries = await observer.wait_for_application(
1275                 application_id=application.entity_id,
1276                 progress_timeout=progress_timeout,
1277                 total_timeout=total_timeout,
1278             )
1279             self.log.debug("application deployed")
1280
1281         else:
1282
1283             # register application with observer
1284             observer.register_application(application=application, db_dict=db_dict)
1285
1286             # application already exists, but not finalised
1287             self.log.debug("application already exists, waiting for deployed...")
1288             retries = await observer.wait_for_application(
1289                 application_id=application.entity_id,
1290                 progress_timeout=progress_timeout,
1291                 total_timeout=total_timeout,
1292             )
1293             self.log.debug("application deployed")
1294
1295         return application, retries
1296
1297     async def _juju_execute_action(
1298         self,
1299         model_name: str,
1300         application_name: str,
1301         action_name: str,
1302         db_dict: dict,
1303         progress_timeout: float = None,
1304         total_timeout: float = None,
1305         **kwargs
1306     ) -> Action:
1307
1308         # get juju model and observer
1309         model = await self._juju_get_model(model_name=model_name)
1310         observer = self.juju_observers[model_name]
1311
1312         application = await self._juju_get_application(
1313             model_name=model_name, application_name=application_name
1314         )
1315
1316         unit = None
1317         for u in application.units:
1318             if await u.is_leader_from_status():
1319                 unit = u
1320         if unit is not None:
1321             actions = await application.get_actions()
1322             if action_name in actions:
1323                 self.log.debug(
1324                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1325                 )
1326                 action = await unit.run_action(action_name, **kwargs)
1327
1328                 # register action with observer
1329                 observer.register_action(action=action, db_dict=db_dict)
1330
1331                 await observer.wait_for_action(
1332                     action_id=action.entity_id,
1333                     progress_timeout=progress_timeout,
1334                     total_timeout=total_timeout,
1335                 )
1336                 self.log.debug("action completed with status: {}".format(action.status))
1337                 output = await model.get_action_output(action_uuid=action.entity_id)
1338                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1339                 if action.entity_id in status:
1340                     status = status[action.entity_id]
1341                 else:
1342                     status = "failed"
1343                 return output, status
1344
1345         raise N2VCExecutionException(
1346             message="Cannot execute action on charm", primitive_name=action_name
1347         )
1348
1349     async def _juju_configure_application(
1350         self,
1351         model_name: str,
1352         application_name: str,
1353         config: dict,
1354         db_dict: dict,
1355         progress_timeout: float = None,
1356         total_timeout: float = None,
1357     ):
1358
1359         # get the application
1360         application = await self._juju_get_application(
1361             model_name=model_name, application_name=application_name
1362         )
1363
1364         self.log.debug(
1365             "configuring the application {} -> {}".format(application_name, config)
1366         )
1367         res = await application.set_config(config)
1368         self.log.debug(
1369             "application {} configured. res={}".format(application_name, res)
1370         )
1371
1372         # Verify the config is set
1373         new_conf = await application.get_config()
1374         for key in config:
1375             value = new_conf[key]["value"]
1376             self.log.debug("    {} = {}".format(key, value))
1377             if config[key] != value:
1378                 raise N2VCException(
1379                     message="key {} is not configured correctly {} != {}".format(
1380                         key, config[key], new_conf[key]
1381                     )
1382                 )
1383
1384         # check if 'verify-ssh-credentials' action exists
1385         # unit = application.units[0]
1386         actions = await application.get_actions()
1387         if "verify-ssh-credentials" not in actions:
1388             msg = (
1389                 "Action verify-ssh-credentials does not exist in application {}"
1390             ).format(application_name)
1391             self.log.debug(msg=msg)
1392             return False
1393
1394         # execute verify-credentials
1395         num_retries = 20
1396         retry_timeout = 15.0
1397         for _ in range(num_retries):
1398             try:
1399                 self.log.debug("Executing action verify-ssh-credentials...")
1400                 output, ok = await self._juju_execute_action(
1401                     model_name=model_name,
1402                     application_name=application_name,
1403                     action_name="verify-ssh-credentials",
1404                     db_dict=db_dict,
1405                     progress_timeout=progress_timeout,
1406                     total_timeout=total_timeout,
1407                 )
1408                 self.log.debug("Result: {}, output: {}".format(ok, output))
1409                 return True
1410             except asyncio.CancelledError:
1411                 raise
1412             except Exception as e:
1413                 self.log.debug(
1414                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1415                 )
1416                 await asyncio.sleep(retry_timeout)
1417         else:
1418             self.log.error(
1419                 "Error executing verify-ssh-credentials after {} retries. ".format(
1420                     num_retries
1421                 )
1422             )
1423             return False
1424
1425     async def _juju_get_application(self, model_name: str, application_name: str):
1426         """Get the deployed application."""
1427
1428         model = await self._juju_get_model(model_name=model_name)
1429
1430         application_name = N2VCJujuConnector._format_app_name(application_name)
1431
1432         if model.applications and application_name in model.applications:
1433             return model.applications[application_name]
1434         else:
1435             raise N2VCException(
1436                 message="Cannot get application {} from model {}".format(
1437                     application_name, model_name
1438                 )
1439             )
1440
1441     async def _juju_get_model(self, model_name: str) -> Model:
1442         """ Get a model object from juju controller
1443         If the model does not exits, it creates it.
1444
1445         :param str model_name: name of the model
1446         :returns Model: model obtained from juju controller or Exception
1447         """
1448
1449         # format model name
1450         model_name = N2VCJujuConnector._format_model_name(model_name)
1451
1452         if model_name in self.juju_models:
1453             return self.juju_models[model_name]
1454
1455         if self._creating_model:
1456             self.log.debug("Another coroutine is creating a model. Wait...")
1457         while self._creating_model:
1458             # another coroutine is creating a model, wait
1459             await asyncio.sleep(0.1)
1460             # retry (perhaps another coroutine has created the model meanwhile)
1461             if model_name in self.juju_models:
1462                 return self.juju_models[model_name]
1463
1464         try:
1465             self._creating_model = True
1466
1467             # get juju model names from juju
1468             model_list = await self.controller.list_models()
1469             if model_name not in model_list:
1470                 self.log.info(
1471                     "Model {} does not exist. Creating new model...".format(model_name)
1472                 )
1473                 config_dict = {"authorized-keys": self.public_key}
1474                 if self.apt_mirror:
1475                     config_dict["apt-mirror"] = self.apt_mirror
1476                 if not self.enable_os_upgrade:
1477                     config_dict["enable-os-refresh-update"] = False
1478                     config_dict["enable-os-upgrade"] = False
1479                 if self.cloud in self.BUILT_IN_CLOUDS:
1480                     model = await self.controller.add_model(
1481                         model_name=model_name,
1482                         config=config_dict,
1483                         cloud_name=self.cloud,
1484                     )
1485                 else:
1486                     model = await self.controller.add_model(
1487                         model_name=model_name,
1488                         config=config_dict,
1489                         cloud_name=self.cloud,
1490                         credential_name=self.cloud,
1491                     )
1492                 self.log.info("New model created, name={}".format(model_name))
1493             else:
1494                 self.log.debug(
1495                     "Model already exists in juju. Getting model {}".format(model_name)
1496                 )
1497                 model = await self.controller.get_model(model_name)
1498                 self.log.debug("Existing model in juju, name={}".format(model_name))
1499
1500             self.juju_models[model_name] = model
1501             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1502             return model
1503
1504         except Exception as e:
1505             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1506             self.log.error(msg)
1507             raise N2VCException(msg)
1508         finally:
1509             self._creating_model = False
1510
1511     async def _juju_add_relation(
1512         self,
1513         model_name: str,
1514         application_name_1: str,
1515         application_name_2: str,
1516         relation_1: str,
1517         relation_2: str,
1518     ):
1519
1520         # get juju model and observer
1521         model = await self._juju_get_model(model_name=model_name)
1522
1523         r1 = "{}:{}".format(application_name_1, relation_1)
1524         r2 = "{}:{}".format(application_name_2, relation_2)
1525
1526         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1527         try:
1528             await model.add_relation(relation1=r1, relation2=r2)
1529         except JujuAPIError as e:
1530             # If one of the applications in the relationship doesn't exist, or the
1531             # relation has already been added,
1532             # let the operation fail silently.
1533             if "not found" in e.message:
1534                 return
1535             if "already exists" in e.message:
1536                 return
1537             # another execption, raise it
1538             raise e
1539
1540     async def _juju_destroy_application(self, model_name: str, application_name: str):
1541
1542         self.log.debug(
1543             "Destroying application {} in model {}".format(application_name, model_name)
1544         )
1545
1546         # get juju model and observer
1547         model = await self._juju_get_model(model_name=model_name)
1548         observer = self.juju_observers[model_name]
1549
1550         application = model.applications.get(application_name)
1551         if application:
1552             observer.unregister_application(application_name)
1553             await application.destroy()
1554         else:
1555             self.log.debug("Application not found: {}".format(application_name))
1556
1557     async def _juju_destroy_machine(
1558         self, model_name: str, machine_id: str, total_timeout: float = None
1559     ):
1560
1561         self.log.debug(
1562             "Destroying machine {} in model {}".format(machine_id, model_name)
1563         )
1564
1565         if total_timeout is None:
1566             total_timeout = 3600
1567
1568         # get juju model and observer
1569         model = await self._juju_get_model(model_name=model_name)
1570         observer = self.juju_observers[model_name]
1571
1572         machines = await model.get_machines()
1573         if machine_id in machines:
1574             machine = model.machines[machine_id]
1575             observer.unregister_machine(machine_id)
1576             # TODO: change this by machine.is_manual when this is upstreamed:
1577             # https://github.com/juju/python-libjuju/pull/396
1578             if "instance-id" in machine.safe_data and machine.safe_data[
1579                 "instance-id"
1580             ].startswith("manual:"):
1581                 self.log.debug("machine.destroy(force=True) started.")
1582                 await machine.destroy(force=True)
1583                 self.log.debug("machine.destroy(force=True) passed.")
1584                 # max timeout
1585                 end = time.time() + total_timeout
1586                 # wait for machine removal
1587                 machines = await model.get_machines()
1588                 while machine_id in machines and time.time() < end:
1589                     self.log.debug(
1590                         "Waiting for machine {} is destroyed".format(machine_id)
1591                     )
1592                     await asyncio.sleep(0.5)
1593                     machines = await model.get_machines()
1594                 self.log.debug("Machine destroyed: {}".format(machine_id))
1595         else:
1596             self.log.debug("Machine not found: {}".format(machine_id))
1597
1598     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1599
1600         self.log.debug("Destroying model {}".format(model_name))
1601
1602         if total_timeout is None:
1603             total_timeout = 3600
1604         end = time.time() + total_timeout
1605
1606         model = await self._juju_get_model(model_name=model_name)
1607
1608         if not model:
1609             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1610
1611         uuid = model.info.uuid
1612
1613         # destroy applications
1614         for application_name in model.applications:
1615             try:
1616                 await self._juju_destroy_application(
1617                     model_name=model_name, application_name=application_name
1618                 )
1619             except Exception as e:
1620                 self.log.error(
1621                     "Error destroying application {} in model {}: {}".format(
1622                         application_name, model_name, e
1623                     )
1624                 )
1625
1626         # destroy machines
1627         machines = await model.get_machines()
1628         for machine_id in machines:
1629             try:
1630                 await self._juju_destroy_machine(
1631                     model_name=model_name, machine_id=machine_id
1632                 )
1633             except asyncio.CancelledError:
1634                 raise
1635             except Exception:
1636                 # ignore exceptions destroying machine
1637                 pass
1638
1639         await self._juju_disconnect_model(model_name=model_name)
1640
1641         self.log.debug("destroying model {}...".format(model_name))
1642         await self.controller.destroy_model(uuid)
1643         # self.log.debug('model destroy requested {}'.format(model_name))
1644
1645         # wait for model is completely destroyed
1646         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1647         last_exception = ""
1648         while time.time() < end:
1649             try:
1650                 # await self.controller.get_model(uuid)
1651                 models = await self.controller.list_models()
1652                 if model_name not in models:
1653                     self.log.debug(
1654                         "The model {} ({}) was destroyed".format(model_name, uuid)
1655                     )
1656                     return
1657             except asyncio.CancelledError:
1658                 raise
1659             except Exception as e:
1660                 last_exception = e
1661             await asyncio.sleep(5)
1662         raise N2VCException(
1663             "Timeout waiting for model {} to be destroyed {}".format(
1664                 model_name, last_exception
1665             )
1666         )
1667
1668     async def _juju_login(self):
1669         """Connect to juju controller
1670
1671         """
1672
1673         # if already authenticated, exit function
1674         if self._authenticated:
1675             return
1676
1677         # if connecting, wait for finish
1678         # another task could be trying to connect in parallel
1679         while self._connecting:
1680             await asyncio.sleep(0.1)
1681
1682         # double check after other task has finished
1683         if self._authenticated:
1684             return
1685
1686         try:
1687             self._connecting = True
1688             self.log.info(
1689                 "connecting to juju controller: {} {}:{}{}".format(
1690                     self.url,
1691                     self.username,
1692                     self.secret[:8] + "...",
1693                     " with ca_cert" if self.ca_cert else "",
1694                 )
1695             )
1696
1697             # Create controller object
1698             self.controller = Controller(loop=self.loop)
1699             # Connect to controller
1700             await self.controller.connect(
1701                 endpoint=self.url,
1702                 username=self.username,
1703                 password=self.secret,
1704                 cacert=self.ca_cert,
1705             )
1706             self._authenticated = True
1707             self.log.info("juju controller connected")
1708         except Exception as e:
1709             message = "Exception connecting to juju: {}".format(e)
1710             self.log.error(message)
1711             raise N2VCConnectionException(message=message, url=self.url)
1712         finally:
1713             self._connecting = False
1714
1715     async def _juju_logout(self):
1716         """Logout of the Juju controller."""
1717         if not self._authenticated:
1718             return False
1719
1720         # disconnect all models
1721         for model_name in self.juju_models:
1722             try:
1723                 await self._juju_disconnect_model(model_name)
1724             except Exception as e:
1725                 self.log.error(
1726                     "Error disconnecting model {} : {}".format(model_name, e)
1727                 )
1728                 # continue with next model...
1729
1730         self.log.info("Disconnecting controller")
1731         try:
1732             await self.controller.disconnect()
1733         except Exception as e:
1734             raise N2VCConnectionException(
1735                 message="Error disconnecting controller: {}".format(e), url=self.url
1736             )
1737
1738         self.controller = None
1739         self._authenticated = False
1740         self.log.info("disconnected")
1741
1742     async def _juju_disconnect_model(self, model_name: str):
1743         self.log.debug("Disconnecting model {}".format(model_name))
1744         if model_name in self.juju_models:
1745             await self.juju_models[model_name].disconnect()
1746             self.juju_models[model_name] = None
1747             self.juju_observers[model_name] = None
1748         else:
1749             self.warning("Cannot disconnect model: {}".format(model_name))
1750
1751     def _create_juju_public_key(self):
1752         """Recreate the Juju public key on lcm container, if needed
1753         Certain libjuju commands expect to be run from the same machine as Juju
1754          is bootstrapped to. This method will write the public key to disk in
1755          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1756         """
1757
1758         # Make sure that we have a public key before writing to disk
1759         if self.public_key is None or len(self.public_key) == 0:
1760             if "OSMLCM_VCA_PUBKEY" in os.environ:
1761                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1762                 if len(self.public_key) == 0:
1763                     return
1764             else:
1765                 return
1766
1767         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1768         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1769         self.log.debug(
1770             "writing juju public key to file:\n{}\npublic key: {}".format(
1771                 file_path, self.public_key
1772             )
1773         )
1774         if not os.path.exists(pk_path):
1775             # create path and write file
1776             os.makedirs(pk_path)
1777             with open(file_path, "w") as f:
1778                 self.log.debug("Creating juju public key file: {}".format(file_path))
1779                 f.write(self.public_key)
1780         else:
1781             self.log.debug("juju public key file already exists: {}".format(file_path))
1782
1783     @staticmethod
1784     def _format_model_name(name: str) -> str:
1785         """Format the name of the model.
1786
1787         Model names may only contain lowercase letters, digits and hyphens
1788         """
1789
1790         return name.replace("_", "-").replace(" ", "-").lower()
1791
1792     @staticmethod
1793     def _format_app_name(name: str) -> str:
1794         """Format the name of the application (in order to assure valid application name).
1795
1796         Application names have restrictions (run juju deploy --help):
1797             - contains lowercase letters 'a'-'z'
1798             - contains numbers '0'-'9'
1799             - contains hyphens '-'
1800             - starts with a lowercase letter
1801             - not two or more consecutive hyphens
1802             - after a hyphen, not a group with all numbers
1803         """
1804
1805         def all_numbers(s: str) -> bool:
1806             for c in s:
1807                 if not c.isdigit():
1808                     return False
1809             return True
1810
1811         new_name = name.replace("_", "-")
1812         new_name = new_name.replace(" ", "-")
1813         new_name = new_name.lower()
1814         while new_name.find("--") >= 0:
1815             new_name = new_name.replace("--", "-")
1816         groups = new_name.split("-")
1817
1818         # find 'all numbers' groups and prefix them with a letter
1819         app_name = ""
1820         for i in range(len(groups)):
1821             group = groups[i]
1822             if all_numbers(group):
1823                 group = "z" + group
1824             if i > 0:
1825                 app_name += "-"
1826             app_name += group
1827
1828         if app_name[0].isdigit():
1829             app_name = "z" + app_name
1830
1831         return app_name