n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46     JujuK8sProxycharmNotSupported,
  47 )
  48 from n2vc.juju_observer import JujuModelObserver
  49 from n2vc.n2vc_conn import N2VCConnector
  50 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  51 from n2vc.provisioner import AsyncSSHProvisioner
  52 from n2vc.libjuju import Libjuju
  53
  54
  55 class N2VCJujuConnector(N2VCConnector):
  56
  57     """
  58 ####################################################################################
  59 ################################### P U B L I C ####################################
  60 ####################################################################################
  61     """
  62
  63     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  64
  65     def __init__(
  66         self,
  67         db: object,
  68         fs: object,
  69         log: object = None,
  70         loop: object = None,
  71         url: str = "127.0.0.1:17070",
  72         username: str = "admin",
  73         vca_config: dict = None,
  74         on_update_db=None,
  75     ):
  76         """Initialize juju N2VC connector
  77         """
  78
  79         # parent class constructor
  80         N2VCConnector.__init__(
  81             self,
  82             db=db,
  83             fs=fs,
  84             log=log,
  85             loop=loop,
  86             url=url,
  87             username=username,
  88             vca_config=vca_config,
  89             on_update_db=on_update_db,
  90         )
  91
  92         # silence websocket traffic log
  93         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  94         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  95         logging.getLogger("model").setLevel(logging.WARN)
  96
  97         self.log.info("Initializing N2VC juju connector...")
  98
  99         """
 100         ##############################################################
 101         # check arguments
 102         ##############################################################
 103         """
 104
 105         # juju URL
 106         if url is None:
 107             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 108         url_parts = url.split(":")
 109         if len(url_parts) != 2:
 110             raise N2VCBadArgumentsException(
 111                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 112             )
 113         self.hostname = url_parts[0]
 114         try:
 115             self.port = int(url_parts[1])
 116         except ValueError:
 117             raise N2VCBadArgumentsException(
 118                 "url port must be a number -> {}".format(url), ["url"]
 119             )
 120
 121         # juju USERNAME
 122         if username is None:
 123             raise N2VCBadArgumentsException(
 124                 "Argument username is mandatory", ["username"]
 125             )
 126
 127         # juju CONFIGURATION
 128         if vca_config is None:
 129             raise N2VCBadArgumentsException(
 130                 "Argument vca_config is mandatory", ["vca_config"]
 131             )
 132
 133         if "secret" in vca_config:
 134             self.secret = vca_config["secret"]
 135         else:
 136             raise N2VCBadArgumentsException(
 137                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 138             )
 139
 140         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 141         # if exists, it will be written in lcm container: _create_juju_public_key()
 142         if "public_key" in vca_config:
 143             self.public_key = vca_config["public_key"]
 144         else:
 145             self.public_key = None
 146
 147         # TODO: Verify ca_cert is valid before using. VCA will crash
 148         # if the ca_cert isn't formatted correctly.
 149         def base64_to_cacert(b64string):
 150             """Convert the base64-encoded string containing the VCA CACERT.
 151
 152             The input string....
 153
 154             """
 155             try:
 156                 cacert = base64.b64decode(b64string).decode("utf-8")
 157
 158                 cacert = re.sub(r"\\n", r"\n", cacert,)
 159             except binascii.Error as e:
 160                 self.log.debug("Caught binascii.Error: {}".format(e))
 161                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 162
 163             return cacert
 164
 165         self.ca_cert = vca_config.get("ca_cert")
 166         if self.ca_cert:
 167             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 168
 169         if "api_proxy" in vca_config and vca_config["api_proxy"] != "":
 170             self.api_proxy = vca_config["api_proxy"]
 171             self.log.debug(
 172                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 173             )
 174         else:
 175             self.warning(
 176                 "api_proxy is not configured"
 177             )
 178             self.api_proxy = None
 179
 180         if "enable_os_upgrade" in vca_config:
 181             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 182         else:
 183             self.enable_os_upgrade = True
 184
 185         if "apt_mirror" in vca_config:
 186             self.apt_mirror = vca_config["apt_mirror"]
 187         else:
 188             self.apt_mirror = None
 189
 190         self.cloud = vca_config.get('cloud')
 191         self.k8s_cloud = None
 192         if "k8s_cloud" in vca_config:
 193             self.k8s_cloud = vca_config.get("k8s_cloud")
 194         self.log.debug('Arguments have been checked')
 195
 196         # juju data
 197         self.controller = None  # it will be filled when connect to juju
 198         self.juju_models = {}  # model objects for every model_name
 199         self.juju_observers = {}  # model observers for every model_name
 200         self._connecting = (
 201             False  # while connecting to juju (to avoid duplicate connections)
 202         )
 203         self._authenticated = (
 204             False  # it will be True when juju connection be stablished
 205         )
 206         self._creating_model = False  # True during model creation
 207         self.libjuju = Libjuju(
 208             endpoint=self.url,
 209             api_proxy=self.api_proxy,
 210             enable_os_upgrade=self.enable_os_upgrade,
 211             apt_mirror=self.apt_mirror,
 212             username=self.username,
 213             password=self.secret,
 214             cacert=self.ca_cert,
 215             loop=self.loop,
 216             log=self.log,
 217             db=self.db,
 218             n2vc=self,
 219         )
 220
 221         # create juju pub key file in lcm container at
 222         # ./local/share/juju/ssh/juju_id_rsa.pub
 223         self._create_juju_public_key()
 224
 225         self.log.info("N2VC juju connector initialized")
 226
 227     async def get_status(self, namespace: str, yaml_format: bool = True):
 228
 229         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 230
 231         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 232             namespace=namespace
 233         )
 234         # model name is ns_id
 235         model_name = ns_id
 236         if model_name is None:
 237             msg = "Namespace {} not valid".format(namespace)
 238             self.log.error(msg)
 239             raise N2VCBadArgumentsException(msg, ["namespace"])
 240
 241         status = {}
 242         models = await self.libjuju.list_models(contains=ns_id)
 243
 244         for m in models:
 245             status[m] = await self.libjuju.get_model_status(m)
 246
 247         if yaml_format:
 248             return obj_to_yaml(status)
 249         else:
 250             return obj_to_dict(status)
 251
 252     async def create_execution_environment(
 253         self,
 254         namespace: str,
 255         db_dict: dict,
 256         reuse_ee_id: str = None,
 257         progress_timeout: float = None,
 258         total_timeout: float = None,
 259     ) -> (str, dict):
 260
 261         self.log.info(
 262             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 263                 namespace, reuse_ee_id
 264             )
 265         )
 266
 267         machine_id = None
 268         if reuse_ee_id:
 269             model_name, application_name, machine_id = self._get_ee_id_components(
 270                 ee_id=reuse_ee_id
 271             )
 272         else:
 273             (
 274                 _nsi_id,
 275                 ns_id,
 276                 _vnf_id,
 277                 _vdu_id,
 278                 _vdu_count,
 279             ) = self._get_namespace_components(namespace=namespace)
 280             # model name is ns_id
 281             model_name = ns_id
 282             # application name
 283             application_name = self._get_application_name(namespace=namespace)
 284
 285         self.log.debug(
 286             "model name: {}, application name:  {}, machine_id: {}".format(
 287                 model_name, application_name, machine_id
 288             )
 289         )
 290
 291         # create or reuse a new juju machine
 292         try:
 293             if not await self.libjuju.model_exists(model_name):
 294                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 295             machine, new = await self.libjuju.create_machine(
 296                 model_name=model_name,
 297                 machine_id=machine_id,
 298                 db_dict=db_dict,
 299                 progress_timeout=progress_timeout,
 300                 total_timeout=total_timeout,
 301             )
 302             # id for the execution environment
 303             ee_id = N2VCJujuConnector._build_ee_id(
 304                 model_name=model_name,
 305                 application_name=application_name,
 306                 machine_id=str(machine.entity_id),
 307             )
 308             self.log.debug("ee_id: {}".format(ee_id))
 309
 310             if new:
 311                 # write ee_id in database
 312                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 313
 314         except Exception as e:
 315             message = "Error creating machine on juju: {}".format(e)
 316             self.log.error(message)
 317             raise N2VCException(message=message)
 318
 319         # new machine credentials
 320         credentials = {
 321             "hostname": machine.dns_name,
 322         }
 323
 324         self.log.info(
 325             "Execution environment created. ee_id: {}, credentials: {}".format(
 326                 ee_id, credentials
 327             )
 328         )
 329
 330         return ee_id, credentials
 331
 332     async def register_execution_environment(
 333         self,
 334         namespace: str,
 335         credentials: dict,
 336         db_dict: dict,
 337         progress_timeout: float = None,
 338         total_timeout: float = None,
 339     ) -> str:
 340
 341         self.log.info(
 342             "Registering execution environment. namespace={}, credentials={}".format(
 343                 namespace, credentials
 344             )
 345         )
 346
 347         if credentials is None:
 348             raise N2VCBadArgumentsException(
 349                 message="credentials are mandatory", bad_args=["credentials"]
 350             )
 351         if credentials.get("hostname"):
 352             hostname = credentials["hostname"]
 353         else:
 354             raise N2VCBadArgumentsException(
 355                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 356             )
 357         if credentials.get("username"):
 358             username = credentials["username"]
 359         else:
 360             raise N2VCBadArgumentsException(
 361                 message="username is mandatory", bad_args=["credentials.username"]
 362             )
 363         if "private_key_path" in credentials:
 364             private_key_path = credentials["private_key_path"]
 365         else:
 366             # if not passed as argument, use generated private key path
 367             private_key_path = self.private_key_path
 368
 369         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 370             namespace=namespace
 371         )
 372
 373         # model name
 374         model_name = ns_id
 375         # application name
 376         application_name = self._get_application_name(namespace=namespace)
 377
 378         # register machine on juju
 379         try:
 380             if not await self.libjuju.model_exists(model_name):
 381                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 382             machine_id = await self.libjuju.provision_machine(
 383                 model_name=model_name,
 384                 hostname=hostname,
 385                 username=username,
 386                 private_key_path=private_key_path,
 387                 db_dict=db_dict,
 388                 progress_timeout=progress_timeout,
 389                 total_timeout=total_timeout,
 390             )
 391         except Exception as e:
 392             self.log.error("Error registering machine: {}".format(e))
 393             raise N2VCException(
 394                 message="Error registering machine on juju: {}".format(e)
 395             )
 396
 397         self.log.info("Machine registered: {}".format(machine_id))
 398
 399         # id for the execution environment
 400         ee_id = N2VCJujuConnector._build_ee_id(
 401             model_name=model_name,
 402             application_name=application_name,
 403             machine_id=str(machine_id),
 404         )
 405
 406         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 407
 408         return ee_id
 409
 410     async def install_configuration_sw(
 411         self,
 412         ee_id: str,
 413         artifact_path: str,
 414         db_dict: dict,
 415         progress_timeout: float = None,
 416         total_timeout: float = None,
 417         config: dict = None,
 418         num_units: int = 1,
 419     ):
 420
 421         self.log.info(
 422             (
 423                 "Installing configuration sw on ee_id: {}, "
 424                 "artifact path: {}, db_dict: {}"
 425             ).format(ee_id, artifact_path, db_dict)
 426         )
 427
 428         # check arguments
 429         if ee_id is None or len(ee_id) == 0:
 430             raise N2VCBadArgumentsException(
 431                 message="ee_id is mandatory", bad_args=["ee_id"]
 432             )
 433         if artifact_path is None or len(artifact_path) == 0:
 434             raise N2VCBadArgumentsException(
 435                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 436             )
 437         if db_dict is None:
 438             raise N2VCBadArgumentsException(
 439                 message="db_dict is mandatory", bad_args=["db_dict"]
 440             )
 441
 442         try:
 443             (
 444                 model_name,
 445                 application_name,
 446                 machine_id,
 447             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 448             self.log.debug(
 449                 "model: {}, application: {}, machine: {}".format(
 450                     model_name, application_name, machine_id
 451                 )
 452             )
 453         except Exception:
 454             raise N2VCBadArgumentsException(
 455                 message="ee_id={} is not a valid execution environment id".format(
 456                     ee_id
 457                 ),
 458                 bad_args=["ee_id"],
 459             )
 460
 461         # remove // in charm path
 462         while artifact_path.find("//") >= 0:
 463             artifact_path = artifact_path.replace("//", "/")
 464
 465         # check charm path
 466         if not self.fs.file_exists(artifact_path, mode="dir"):
 467             msg = "artifact path does not exist: {}".format(artifact_path)
 468             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 469
 470         if artifact_path.startswith("/"):
 471             full_path = self.fs.path + artifact_path
 472         else:
 473             full_path = self.fs.path + "/" + artifact_path
 474
 475         try:
 476             await self.libjuju.deploy_charm(
 477                 model_name=model_name,
 478                 application_name=application_name,
 479                 path=full_path,
 480                 machine_id=machine_id,
 481                 db_dict=db_dict,
 482                 progress_timeout=progress_timeout,
 483                 total_timeout=total_timeout,
 484                 config=config,
 485                 num_units=num_units,
 486             )
 487         except Exception as e:
 488             raise N2VCException(
 489                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 490             )
 491
 492         self.log.info("Configuration sw installed")
 493
 494     async def install_k8s_proxy_charm(
 495         self,
 496         charm_name: str,
 497         namespace: str,
 498         artifact_path: str,
 499         db_dict: dict,
 500         progress_timeout: float = None,
 501         total_timeout: float = None,
 502         config: dict = None,
 503     ) -> str:
 504         """
 505         Install a k8s proxy charm
 506
 507         :param charm_name: Name of the charm being deployed
 508         :param namespace: collection of all the uuids related to the charm.
 509         :param str artifact_path: where to locate the artifacts (parent folder) using
 510             the self.fs
 511             the final artifact path will be a combination of this artifact_path and
 512             additional string from the config_dict (e.g. charm name)
 513         :param dict db_dict: where to write into database when the status changes.
 514                         It contains a dict with
 515                             {collection: <str>, filter: {},  path: <str>},
 516                             e.g. {collection: "nsrs", filter:
 517                                 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
 518         :param float progress_timeout:
 519         :param float total_timeout:
 520         :param config: Dictionary with additional configuration
 521
 522         :returns ee_id: execution environment id.
 523         """
 524         self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
 525                       .format(charm_name, artifact_path, db_dict))
 526
 527         if not self.k8s_cloud:
 528             raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
 529
 530         if artifact_path is None or len(artifact_path) == 0:
 531             raise N2VCBadArgumentsException(
 532                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 533             )
 534         if db_dict is None:
 535             raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict'])
 536
 537         # remove // in charm path
 538         while artifact_path.find('//') >= 0:
 539             artifact_path = artifact_path.replace('//', '/')
 540
 541         # check charm path
 542         if not self.fs.file_exists(artifact_path, mode="dir"):
 543             msg = 'artifact path does not exist: {}'.format(artifact_path)
 544             raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path'])
 545
 546         if artifact_path.startswith('/'):
 547             full_path = self.fs.path + artifact_path
 548         else:
 549             full_path = self.fs.path + '/' + artifact_path
 550
 551         _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace)
 552         model_name = '{}-k8s'.format(ns_id)
 553
 554         await self.libjuju.add_model(model_name, self.k8s_cloud)
 555         application_name = self._get_application_name(namespace)
 556
 557         try:
 558             await self.libjuju.deploy_charm(
 559                 model_name=model_name,
 560                 application_name=application_name,
 561                 path=full_path,
 562                 machine_id=None,
 563                 db_dict=db_dict,
 564                 progress_timeout=progress_timeout,
 565                 total_timeout=total_timeout,
 566                 config=config
 567             )
 568         except Exception as e:
 569             raise N2VCException(message='Error deploying charm: {}'.format(e))
 570
 571         self.log.info('K8s proxy charm installed')
 572         ee_id = N2VCJujuConnector._build_ee_id(
 573             model_name=model_name,
 574             application_name=application_name,
 575             machine_id="k8s",
 576         )
 577
 578         self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 579
 580         return ee_id
 581
 582     async def get_ee_ssh_public__key(
 583         self,
 584         ee_id: str,
 585         db_dict: dict,
 586         progress_timeout: float = None,
 587         total_timeout: float = None,
 588     ) -> str:
 589
 590         self.log.info(
 591             (
 592                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 593             ).format(ee_id, db_dict)
 594         )
 595
 596         # check arguments
 597         if ee_id is None or len(ee_id) == 0:
 598             raise N2VCBadArgumentsException(
 599                 message="ee_id is mandatory", bad_args=["ee_id"]
 600             )
 601         if db_dict is None:
 602             raise N2VCBadArgumentsException(
 603                 message="db_dict is mandatory", bad_args=["db_dict"]
 604             )
 605
 606         try:
 607             (
 608                 model_name,
 609                 application_name,
 610                 machine_id,
 611             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 612             self.log.debug(
 613                 "model: {}, application: {}, machine: {}".format(
 614                     model_name, application_name, machine_id
 615                 )
 616             )
 617         except Exception:
 618             raise N2VCBadArgumentsException(
 619                 message="ee_id={} is not a valid execution environment id".format(
 620                     ee_id
 621                 ),
 622                 bad_args=["ee_id"],
 623             )
 624
 625         # try to execute ssh layer primitives (if exist):
 626         #       generate-ssh-key
 627         #       get-ssh-public-key
 628
 629         output = None
 630
 631         application_name = N2VCJujuConnector._format_app_name(application_name)
 632
 633         # execute action: generate-ssh-key
 634         try:
 635             output, _status = await self.libjuju.execute_action(
 636                 model_name=model_name,
 637                 application_name=application_name,
 638                 action_name="generate-ssh-key",
 639                 db_dict=db_dict,
 640                 progress_timeout=progress_timeout,
 641                 total_timeout=total_timeout,
 642             )
 643         except Exception as e:
 644             self.log.info(
 645                 "Skipping exception while executing action generate-ssh-key: {}".format(
 646                     e
 647                 )
 648             )
 649
 650         # execute action: get-ssh-public-key
 651         try:
 652             output, _status = await self.libjuju.execute_action(
 653                 model_name=model_name,
 654                 application_name=application_name,
 655                 action_name="get-ssh-public-key",
 656                 db_dict=db_dict,
 657                 progress_timeout=progress_timeout,
 658                 total_timeout=total_timeout,
 659             )
 660         except Exception as e:
 661             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 662             self.log.info(msg)
 663             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 664
 665         # return public key if exists
 666         return output["pubkey"] if "pubkey" in output else output
 667
 668     async def add_relation(
 669         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 670     ):
 671
 672         self.log.debug(
 673             "adding new relation between {} and {}, endpoints: {}, {}".format(
 674                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 675             )
 676         )
 677
 678         # check arguments
 679         if not ee_id_1:
 680             message = "EE 1 is mandatory"
 681             self.log.error(message)
 682             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 683         if not ee_id_2:
 684             message = "EE 2 is mandatory"
 685             self.log.error(message)
 686             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 687         if not endpoint_1:
 688             message = "endpoint 1 is mandatory"
 689             self.log.error(message)
 690             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 691         if not endpoint_2:
 692             message = "endpoint 2 is mandatory"
 693             self.log.error(message)
 694             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 695
 696         # get the model, the applications and the machines from the ee_id's
 697         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 698         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 699
 700         # model must be the same
 701         if model_1 != model_2:
 702             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 703             self.log.error(message)
 704             raise N2VCBadArgumentsException(
 705                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 706             )
 707
 708         # add juju relations between two applications
 709         try:
 710             await self.libjuju.add_relation(
 711                 model_name=model_1,
 712                 application_name_1=app_1,
 713                 application_name_2=app_2,
 714                 relation_1=endpoint_1,
 715                 relation_2=endpoint_2,
 716             )
 717         except Exception as e:
 718             message = "Error adding relation between {} and {}: {}".format(
 719                 ee_id_1, ee_id_2, e
 720             )
 721             self.log.error(message)
 722             raise N2VCException(message=message)
 723
 724     async def remove_relation(self):
 725         # TODO
 726         self.log.info("Method not implemented yet")
 727         raise MethodNotImplemented()
 728
 729     async def deregister_execution_environments(self):
 730         self.log.info("Method not implemented yet")
 731         raise MethodNotImplemented()
 732
 733     async def delete_namespace(
 734         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 735     ):
 736         self.log.info("Deleting namespace={}".format(namespace))
 737
 738         # check arguments
 739         if namespace is None:
 740             raise N2VCBadArgumentsException(
 741                 message="namespace is mandatory", bad_args=["namespace"]
 742             )
 743
 744         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 745             namespace=namespace
 746         )
 747         if ns_id is not None:
 748             try:
 749                 models = await self.libjuju.list_models(contains=ns_id)
 750                 for model in models:
 751                     await self.libjuju.destroy_model(
 752                         model_name=model, total_timeout=total_timeout
 753                     )
 754             except Exception as e:
 755                 raise N2VCException(
 756                     message="Error deleting namespace {} : {}".format(namespace, e)
 757                 )
 758         else:
 759             raise N2VCBadArgumentsException(
 760                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 761             )
 762
 763         self.log.info("Namespace {} deleted".format(namespace))
 764
 765     async def delete_execution_environment(
 766         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 767     ):
 768         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 769
 770         # check arguments
 771         if ee_id is None:
 772             raise N2VCBadArgumentsException(
 773                 message="ee_id is mandatory", bad_args=["ee_id"]
 774             )
 775
 776         model_name, application_name, _machine_id = self._get_ee_id_components(
 777             ee_id=ee_id
 778         )
 779
 780         # destroy the application
 781         try:
 782             await self.libjuju.destroy_model(
 783                 model_name=model_name, total_timeout=total_timeout
 784             )
 785         except Exception as e:
 786             raise N2VCException(
 787                 message=(
 788                     "Error deleting execution environment {} (application {}) : {}"
 789                 ).format(ee_id, application_name, e)
 790             )
 791
 792         # destroy the machine
 793         # try:
 794         #     await self._juju_destroy_machine(
 795         #         model_name=model_name,
 796         #         machine_id=machine_id,
 797         #         total_timeout=total_timeout
 798         #     )
 799         # except Exception as e:
 800         #     raise N2VCException(
 801         #        message='Error deleting execution environment {} (machine {}) : {}'
 802         #                .format(ee_id, machine_id, e))
 803
 804         self.log.info("Execution environment {} deleted".format(ee_id))
 805
 806     async def exec_primitive(
 807         self,
 808         ee_id: str,
 809         primitive_name: str,
 810         params_dict: dict,
 811         db_dict: dict = None,
 812         progress_timeout: float = None,
 813         total_timeout: float = None,
 814     ) -> str:
 815
 816         self.log.info(
 817             "Executing primitive: {} on ee: {}, params: {}".format(
 818                 primitive_name, ee_id, params_dict
 819             )
 820         )
 821
 822         # check arguments
 823         if ee_id is None or len(ee_id) == 0:
 824             raise N2VCBadArgumentsException(
 825                 message="ee_id is mandatory", bad_args=["ee_id"]
 826             )
 827         if primitive_name is None or len(primitive_name) == 0:
 828             raise N2VCBadArgumentsException(
 829                 message="action_name is mandatory", bad_args=["action_name"]
 830             )
 831         if params_dict is None:
 832             params_dict = dict()
 833
 834         try:
 835             (
 836                 model_name,
 837                 application_name,
 838                 _machine_id,
 839             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 840         except Exception:
 841             raise N2VCBadArgumentsException(
 842                 message="ee_id={} is not a valid execution environment id".format(
 843                     ee_id
 844                 ),
 845                 bad_args=["ee_id"],
 846             )
 847
 848         if primitive_name == "config":
 849             # Special case: config primitive
 850             try:
 851                 await self.libjuju.configure_application(
 852                     model_name=model_name,
 853                     application_name=application_name,
 854                     config=params_dict,
 855                 )
 856                 actions = await self.libjuju.get_actions(
 857                     application_name=application_name, model_name=model_name,
 858                 )
 859                 self.log.debug(
 860                     "Application {} has these actions: {}".format(
 861                         application_name, actions
 862                     )
 863                 )
 864                 if "verify-ssh-credentials" in actions:
 865                     # execute verify-credentials
 866                     num_retries = 20
 867                     retry_timeout = 15.0
 868                     for _ in range(num_retries):
 869                         try:
 870                             self.log.debug("Executing action verify-ssh-credentials...")
 871                             output, ok = await self.libjuju.execute_action(
 872                                 model_name=model_name,
 873                                 application_name=application_name,
 874                                 action_name="verify-ssh-credentials",
 875                                 db_dict=db_dict,
 876                                 progress_timeout=progress_timeout,
 877                                 total_timeout=total_timeout,
 878                             )
 879
 880                             if ok == "failed":
 881                                 self.log.debug(
 882                                     "Error executing verify-ssh-credentials: {}. Retrying..."
 883                                 )
 884                                 await asyncio.sleep(retry_timeout)
 885
 886                                 continue
 887                             self.log.debug("Result: {}, output: {}".format(ok, output))
 888                             break
 889                         except asyncio.CancelledError:
 890                             raise
 891                     else:
 892                         self.log.error(
 893                             "Error executing verify-ssh-credentials after {} retries. ".format(
 894                                 num_retries
 895                             )
 896                         )
 897                 else:
 898                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 899                         application_name
 900                     )
 901                     self.log.debug(msg=msg)
 902             except Exception as e:
 903                 self.log.error("Error configuring juju application: {}".format(e))
 904                 raise N2VCExecutionException(
 905                     message="Error configuring application into ee={} : {}".format(
 906                         ee_id, e
 907                     ),
 908                     primitive_name=primitive_name,
 909                 )
 910             return "CONFIG OK"
 911         else:
 912             try:
 913                 output, status = await self.libjuju.execute_action(
 914                     model_name=model_name,
 915                     application_name=application_name,
 916                     action_name=primitive_name,
 917                     db_dict=db_dict,
 918                     progress_timeout=progress_timeout,
 919                     total_timeout=total_timeout,
 920                     **params_dict
 921                 )
 922                 if status == "completed":
 923                     return output
 924                 else:
 925                     raise Exception("status is not completed: {}".format(status))
 926             except Exception as e:
 927                 self.log.error(
 928                     "Error executing primitive {}: {}".format(primitive_name, e)
 929                 )
 930                 raise N2VCExecutionException(
 931                     message="Error executing primitive {} into ee={} : {}".format(
 932                         primitive_name, ee_id, e
 933                     ),
 934                     primitive_name=primitive_name,
 935                 )
 936
 937     async def disconnect(self):
 938         self.log.info("closing juju N2VC...")
 939         try:
 940             await self.libjuju.disconnect()
 941         except Exception as e:
 942             raise N2VCConnectionException(
 943                 message="Error disconnecting controller: {}".format(e), url=self.url
 944             )
 945
 946     """
 947 ####################################################################################
 948 ################################### P R I V A T E ##################################
 949 ####################################################################################
 950     """
 951
 952     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 953
 954         # write ee_id to database: _admin.deployed.VCA.x
 955         try:
 956             the_table = db_dict["collection"]
 957             the_filter = db_dict["filter"]
 958             the_path = db_dict["path"]
 959             if not the_path[-1] == ".":
 960                 the_path = the_path + "."
 961             update_dict = {the_path + "ee_id": ee_id}
 962             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 963             self.db.set_one(
 964                 table=the_table,
 965                 q_filter=the_filter,
 966                 update_dict=update_dict,
 967                 fail_on_empty=True,
 968             )
 969         except asyncio.CancelledError:
 970             raise
 971         except Exception as e:
 972             self.log.error("Error writing ee_id to database: {}".format(e))
 973
 974     @staticmethod
 975     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 976         """
 977         Build an execution environment id form model, application and machine
 978         :param model_name:
 979         :param application_name:
 980         :param machine_id:
 981         :return:
 982         """
 983         # id for the execution environment
 984         return "{}.{}.{}".format(model_name, application_name, machine_id)
 985
 986     @staticmethod
 987     def _get_ee_id_components(ee_id: str) -> (str, str, str):
 988         """
 989         Get model, application and machine components from an execution environment id
 990         :param ee_id:
 991         :return: model_name, application_name, machine_id
 992         """
 993
 994         if ee_id is None:
 995             return None, None, None
 996
 997         # split components of id
 998         parts = ee_id.split(".")
 999         model_name = parts[0]
1000         application_name = parts[1]
1001         machine_id = parts[2]
1002         return model_name, application_name, machine_id
1003
1004     def _get_application_name(self, namespace: str) -> str:
1005         """
1006         Build application name from namespace
1007         :param namespace:
1008         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1009         """
1010
1011         # TODO: Enforce the Juju 50-character application limit
1012
1013         # split namespace components
1014         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
1015             namespace=namespace
1016         )
1017
1018         if vnf_id is None or len(vnf_id) == 0:
1019             vnf_id = ""
1020         else:
1021             # Shorten the vnf_id to its last twelve characters
1022             vnf_id = "vnf-" + vnf_id[-12:]
1023
1024         if vdu_id is None or len(vdu_id) == 0:
1025             vdu_id = ""
1026         else:
1027             # Shorten the vdu_id to its last twelve characters
1028             vdu_id = "-vdu-" + vdu_id[-12:]
1029
1030         if vdu_count is None or len(vdu_count) == 0:
1031             vdu_count = ""
1032         else:
1033             vdu_count = "-cnt-" + vdu_count
1034
1035         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
1036
1037         return N2VCJujuConnector._format_app_name(application_name)
1038
1039     async def _juju_create_machine(
1040         self,
1041         model_name: str,
1042         application_name: str,
1043         machine_id: str = None,
1044         db_dict: dict = None,
1045         progress_timeout: float = None,
1046         total_timeout: float = None,
1047     ) -> Machine:
1048
1049         self.log.debug(
1050             "creating machine in model: {}, existing machine id: {}".format(
1051                 model_name, machine_id
1052             )
1053         )
1054
1055         # get juju model and observer (create model if needed)
1056         model = await self._juju_get_model(model_name=model_name)
1057         observer = self.juju_observers[model_name]
1058
1059         # find machine id in model
1060         machine = None
1061         if machine_id is not None:
1062             self.log.debug("Finding existing machine id {} in model".format(machine_id))
1063             # get juju existing machines in the model
1064             existing_machines = await model.get_machines()
1065             if machine_id in existing_machines:
1066                 self.log.debug(
1067                     "Machine id {} found in model (reusing it)".format(machine_id)
1068                 )
1069                 machine = model.machines[machine_id]
1070
1071         if machine is None:
1072             self.log.debug("Creating a new machine in juju...")
1073             # machine does not exist, create it and wait for it
1074             machine = await model.add_machine(
1075                 spec=None, constraints=None, disks=None, series="xenial"
1076             )
1077
1078             # register machine with observer
1079             observer.register_machine(machine=machine, db_dict=db_dict)
1080
1081             # id for the execution environment
1082             ee_id = N2VCJujuConnector._build_ee_id(
1083                 model_name=model_name,
1084                 application_name=application_name,
1085                 machine_id=str(machine.entity_id),
1086             )
1087
1088             # write ee_id in database
1089             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
1090
1091             # wait for machine creation
1092             await observer.wait_for_machine(
1093                 machine_id=str(machine.entity_id),
1094                 progress_timeout=progress_timeout,
1095                 total_timeout=total_timeout,
1096             )
1097
1098         else:
1099
1100             self.log.debug("Reusing old machine pending")
1101
1102             # register machine with observer
1103             observer.register_machine(machine=machine, db_dict=db_dict)
1104
1105             # machine does exist, but it is in creation process (pending), wait for
1106             # create finalisation
1107             await observer.wait_for_machine(
1108                 machine_id=machine.entity_id,
1109                 progress_timeout=progress_timeout,
1110                 total_timeout=total_timeout,
1111             )
1112
1113         self.log.debug("Machine ready at " + str(machine.dns_name))
1114         return machine
1115
1116     async def _juju_provision_machine(
1117         self,
1118         model_name: str,
1119         hostname: str,
1120         username: str,
1121         private_key_path: str,
1122         db_dict: dict = None,
1123         progress_timeout: float = None,
1124         total_timeout: float = None,
1125     ) -> str:
1126
1127         if not self.api_proxy:
1128             msg = "Cannot provision machine: api_proxy is not defined"
1129             self.log.error(msg=msg)
1130             raise N2VCException(message=msg)
1131
1132         self.log.debug(
1133             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1134                 model_name, hostname, username
1135             )
1136         )
1137
1138         if not self._authenticated:
1139             await self._juju_login()
1140
1141         # get juju model and observer
1142         model = await self._juju_get_model(model_name=model_name)
1143         observer = self.juju_observers[model_name]
1144
1145         # TODO check if machine is already provisioned
1146         machine_list = await model.get_machines()
1147
1148         provisioner = AsyncSSHProvisioner(
1149             host=hostname,
1150             user=username,
1151             private_key_path=private_key_path,
1152             log=self.log,
1153         )
1154
1155         params = None
1156         try:
1157             params = await provisioner.provision_machine()
1158         except Exception as ex:
1159             msg = "Exception provisioning machine: {}".format(ex)
1160             self.log.error(msg)
1161             raise N2VCException(message=msg)
1162
1163         params.jobs = ["JobHostUnits"]
1164
1165         connection = model.connection()
1166
1167         # Submit the request.
1168         self.log.debug("Adding machine to model")
1169         client_facade = client.ClientFacade.from_connection(connection)
1170         results = await client_facade.AddMachines(params=[params])
1171         error = results.machines[0].error
1172         if error:
1173             msg = "Error adding machine: {}".format(error.message)
1174             self.log.error(msg=msg)
1175             raise ValueError(msg)
1176
1177         machine_id = results.machines[0].machine
1178
1179         # Need to run this after AddMachines has been called,
1180         # as we need the machine_id
1181         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1182         asyncio.ensure_future(
1183             provisioner.install_agent(
1184                 connection=connection,
1185                 nonce=params.nonce,
1186                 machine_id=machine_id,
1187                 proxy=self.api_proxy,
1188             )
1189         )
1190
1191         # wait for machine in model (now, machine is not yet in model, so we must
1192         # wait for it)
1193         machine = None
1194         for _ in range(10):
1195             machine_list = await model.get_machines()
1196             if machine_id in machine_list:
1197                 self.log.debug("Machine {} found in model!".format(machine_id))
1198                 machine = model.machines.get(machine_id)
1199                 break
1200             await asyncio.sleep(2)
1201
1202         if machine is None:
1203             msg = "Machine {} not found in model".format(machine_id)
1204             self.log.error(msg=msg)
1205             raise Exception(msg)
1206
1207         # register machine with observer
1208         observer.register_machine(machine=machine, db_dict=db_dict)
1209
1210         # wait for machine creation
1211         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1212         await observer.wait_for_machine(
1213             machine_id=machine_id,
1214             progress_timeout=progress_timeout,
1215             total_timeout=total_timeout,
1216         )
1217
1218         self.log.debug("Machine provisioned {}".format(machine_id))
1219
1220         return machine_id
1221
1222     async def _juju_deploy_charm(
1223         self,
1224         model_name: str,
1225         application_name: str,
1226         charm_path: str,
1227         machine_id: str,
1228         db_dict: dict,
1229         progress_timeout: float = None,
1230         total_timeout: float = None,
1231         config: dict = None,
1232     ) -> (Application, int):
1233
1234         # get juju model and observer
1235         model = await self._juju_get_model(model_name=model_name)
1236         observer = self.juju_observers[model_name]
1237
1238         # check if application already exists
1239         application = None
1240         if application_name in model.applications:
1241             application = model.applications[application_name]
1242
1243         if application is None:
1244
1245             # application does not exist, create it and wait for it
1246             self.log.debug(
1247                 "deploying application {} to machine {}, model {}".format(
1248                     application_name, machine_id, model_name
1249                 )
1250             )
1251             self.log.debug("charm: {}".format(charm_path))
1252             machine = model.machines[machine_id]
1253             # series = None
1254             application = await model.deploy(
1255                 entity_url=charm_path,
1256                 application_name=application_name,
1257                 channel="stable",
1258                 num_units=1,
1259                 series=machine.series,
1260                 to=machine_id,
1261                 config=config,
1262             )
1263
1264             # register application with observer
1265             observer.register_application(application=application, db_dict=db_dict)
1266
1267             self.log.debug(
1268                 "waiting for application deployed... {}".format(application.entity_id)
1269             )
1270             retries = await observer.wait_for_application(
1271                 application_id=application.entity_id,
1272                 progress_timeout=progress_timeout,
1273                 total_timeout=total_timeout,
1274             )
1275             self.log.debug("application deployed")
1276
1277         else:
1278
1279             # register application with observer
1280             observer.register_application(application=application, db_dict=db_dict)
1281
1282             # application already exists, but not finalised
1283             self.log.debug("application already exists, waiting for deployed...")
1284             retries = await observer.wait_for_application(
1285                 application_id=application.entity_id,
1286                 progress_timeout=progress_timeout,
1287                 total_timeout=total_timeout,
1288             )
1289             self.log.debug("application deployed")
1290
1291         return application, retries
1292
1293     async def _juju_execute_action(
1294         self,
1295         model_name: str,
1296         application_name: str,
1297         action_name: str,
1298         db_dict: dict,
1299         progress_timeout: float = None,
1300         total_timeout: float = None,
1301         **kwargs
1302     ) -> Action:
1303
1304         # get juju model and observer
1305         model = await self._juju_get_model(model_name=model_name)
1306         observer = self.juju_observers[model_name]
1307
1308         application = await self._juju_get_application(
1309             model_name=model_name, application_name=application_name
1310         )
1311
1312         unit = None
1313         for u in application.units:
1314             if await u.is_leader_from_status():
1315                 unit = u
1316         if unit is not None:
1317             actions = await application.get_actions()
1318             if action_name in actions:
1319                 self.log.debug(
1320                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1321                 )
1322                 action = await unit.run_action(action_name, **kwargs)
1323
1324                 # register action with observer
1325                 observer.register_action(action=action, db_dict=db_dict)
1326
1327                 await observer.wait_for_action(
1328                     action_id=action.entity_id,
1329                     progress_timeout=progress_timeout,
1330                     total_timeout=total_timeout,
1331                 )
1332                 self.log.debug("action completed with status: {}".format(action.status))
1333                 output = await model.get_action_output(action_uuid=action.entity_id)
1334                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1335                 if action.entity_id in status:
1336                     status = status[action.entity_id]
1337                 else:
1338                     status = "failed"
1339                 return output, status
1340
1341         raise N2VCExecutionException(
1342             message="Cannot execute action on charm", primitive_name=action_name
1343         )
1344
1345     async def _juju_configure_application(
1346         self,
1347         model_name: str,
1348         application_name: str,
1349         config: dict,
1350         db_dict: dict,
1351         progress_timeout: float = None,
1352         total_timeout: float = None,
1353     ):
1354
1355         # get the application
1356         application = await self._juju_get_application(
1357             model_name=model_name, application_name=application_name
1358         )
1359
1360         self.log.debug(
1361             "configuring the application {} -> {}".format(application_name, config)
1362         )
1363         res = await application.set_config(config)
1364         self.log.debug(
1365             "application {} configured. res={}".format(application_name, res)
1366         )
1367
1368         # Verify the config is set
1369         new_conf = await application.get_config()
1370         for key in config:
1371             value = new_conf[key]["value"]
1372             self.log.debug("    {} = {}".format(key, value))
1373             if config[key] != value:
1374                 raise N2VCException(
1375                     message="key {} is not configured correctly {} != {}".format(
1376                         key, config[key], new_conf[key]
1377                     )
1378                 )
1379
1380         # check if 'verify-ssh-credentials' action exists
1381         # unit = application.units[0]
1382         actions = await application.get_actions()
1383         if "verify-ssh-credentials" not in actions:
1384             msg = (
1385                 "Action verify-ssh-credentials does not exist in application {}"
1386             ).format(application_name)
1387             self.log.debug(msg=msg)
1388             return False
1389
1390         # execute verify-credentials
1391         num_retries = 20
1392         retry_timeout = 15.0
1393         for _ in range(num_retries):
1394             try:
1395                 self.log.debug("Executing action verify-ssh-credentials...")
1396                 output, ok = await self._juju_execute_action(
1397                     model_name=model_name,
1398                     application_name=application_name,
1399                     action_name="verify-ssh-credentials",
1400                     db_dict=db_dict,
1401                     progress_timeout=progress_timeout,
1402                     total_timeout=total_timeout,
1403                 )
1404                 self.log.debug("Result: {}, output: {}".format(ok, output))
1405                 return True
1406             except asyncio.CancelledError:
1407                 raise
1408             except Exception as e:
1409                 self.log.debug(
1410                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1411                 )
1412                 await asyncio.sleep(retry_timeout)
1413         else:
1414             self.log.error(
1415                 "Error executing verify-ssh-credentials after {} retries. ".format(
1416                     num_retries
1417                 )
1418             )
1419             return False
1420
1421     async def _juju_get_application(self, model_name: str, application_name: str):
1422         """Get the deployed application."""
1423
1424         model = await self._juju_get_model(model_name=model_name)
1425
1426         application_name = N2VCJujuConnector._format_app_name(application_name)
1427
1428         if model.applications and application_name in model.applications:
1429             return model.applications[application_name]
1430         else:
1431             raise N2VCException(
1432                 message="Cannot get application {} from model {}".format(
1433                     application_name, model_name
1434                 )
1435             )
1436
1437     async def _juju_get_model(self, model_name: str) -> Model:
1438         """ Get a model object from juju controller
1439         If the model does not exits, it creates it.
1440
1441         :param str model_name: name of the model
1442         :returns Model: model obtained from juju controller or Exception
1443         """
1444
1445         # format model name
1446         model_name = N2VCJujuConnector._format_model_name(model_name)
1447
1448         if model_name in self.juju_models:
1449             return self.juju_models[model_name]
1450
1451         if self._creating_model:
1452             self.log.debug("Another coroutine is creating a model. Wait...")
1453         while self._creating_model:
1454             # another coroutine is creating a model, wait
1455             await asyncio.sleep(0.1)
1456             # retry (perhaps another coroutine has created the model meanwhile)
1457             if model_name in self.juju_models:
1458                 return self.juju_models[model_name]
1459
1460         try:
1461             self._creating_model = True
1462
1463             # get juju model names from juju
1464             model_list = await self.controller.list_models()
1465             if model_name not in model_list:
1466                 self.log.info(
1467                     "Model {} does not exist. Creating new model...".format(model_name)
1468                 )
1469                 config_dict = {"authorized-keys": self.public_key}
1470                 if self.apt_mirror:
1471                     config_dict["apt-mirror"] = self.apt_mirror
1472                 if not self.enable_os_upgrade:
1473                     config_dict["enable-os-refresh-update"] = False
1474                     config_dict["enable-os-upgrade"] = False
1475                 if self.cloud in self.BUILT_IN_CLOUDS:
1476                     model = await self.controller.add_model(
1477                         model_name=model_name,
1478                         config=config_dict,
1479                         cloud_name=self.cloud,
1480                     )
1481                 else:
1482                     model = await self.controller.add_model(
1483                         model_name=model_name,
1484                         config=config_dict,
1485                         cloud_name=self.cloud,
1486                         credential_name=self.cloud,
1487                     )
1488                 self.log.info("New model created, name={}".format(model_name))
1489             else:
1490                 self.log.debug(
1491                     "Model already exists in juju. Getting model {}".format(model_name)
1492                 )
1493                 model = await self.controller.get_model(model_name)
1494                 self.log.debug("Existing model in juju, name={}".format(model_name))
1495
1496             self.juju_models[model_name] = model
1497             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1498             return model
1499
1500         except Exception as e:
1501             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1502             self.log.error(msg)
1503             raise N2VCException(msg)
1504         finally:
1505             self._creating_model = False
1506
1507     async def _juju_add_relation(
1508         self,
1509         model_name: str,
1510         application_name_1: str,
1511         application_name_2: str,
1512         relation_1: str,
1513         relation_2: str,
1514     ):
1515
1516         # get juju model and observer
1517         model = await self._juju_get_model(model_name=model_name)
1518
1519         r1 = "{}:{}".format(application_name_1, relation_1)
1520         r2 = "{}:{}".format(application_name_2, relation_2)
1521
1522         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1523         try:
1524             await model.add_relation(relation1=r1, relation2=r2)
1525         except JujuAPIError as e:
1526             # If one of the applications in the relationship doesn't exist, or the
1527             # relation has already been added,
1528             # let the operation fail silently.
1529             if "not found" in e.message:
1530                 return
1531             if "already exists" in e.message:
1532                 return
1533             # another execption, raise it
1534             raise e
1535
1536     async def _juju_destroy_application(self, model_name: str, application_name: str):
1537
1538         self.log.debug(
1539             "Destroying application {} in model {}".format(application_name, model_name)
1540         )
1541
1542         # get juju model and observer
1543         model = await self._juju_get_model(model_name=model_name)
1544         observer = self.juju_observers[model_name]
1545
1546         application = model.applications.get(application_name)
1547         if application:
1548             observer.unregister_application(application_name)
1549             await application.destroy()
1550         else:
1551             self.log.debug("Application not found: {}".format(application_name))
1552
1553     async def _juju_destroy_machine(
1554         self, model_name: str, machine_id: str, total_timeout: float = None
1555     ):
1556
1557         self.log.debug(
1558             "Destroying machine {} in model {}".format(machine_id, model_name)
1559         )
1560
1561         if total_timeout is None:
1562             total_timeout = 3600
1563
1564         # get juju model and observer
1565         model = await self._juju_get_model(model_name=model_name)
1566         observer = self.juju_observers[model_name]
1567
1568         machines = await model.get_machines()
1569         if machine_id in machines:
1570             machine = model.machines[machine_id]
1571             observer.unregister_machine(machine_id)
1572             # TODO: change this by machine.is_manual when this is upstreamed:
1573             # https://github.com/juju/python-libjuju/pull/396
1574             if "instance-id" in machine.safe_data and machine.safe_data[
1575                 "instance-id"
1576             ].startswith("manual:"):
1577                 self.log.debug("machine.destroy(force=True) started.")
1578                 await machine.destroy(force=True)
1579                 self.log.debug("machine.destroy(force=True) passed.")
1580                 # max timeout
1581                 end = time.time() + total_timeout
1582                 # wait for machine removal
1583                 machines = await model.get_machines()
1584                 while machine_id in machines and time.time() < end:
1585                     self.log.debug(
1586                         "Waiting for machine {} is destroyed".format(machine_id)
1587                     )
1588                     await asyncio.sleep(0.5)
1589                     machines = await model.get_machines()
1590                 self.log.debug("Machine destroyed: {}".format(machine_id))
1591         else:
1592             self.log.debug("Machine not found: {}".format(machine_id))
1593
1594     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1595
1596         self.log.debug("Destroying model {}".format(model_name))
1597
1598         if total_timeout is None:
1599             total_timeout = 3600
1600         end = time.time() + total_timeout
1601
1602         model = await self._juju_get_model(model_name=model_name)
1603
1604         if not model:
1605             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1606
1607         uuid = model.info.uuid
1608
1609         # destroy applications
1610         for application_name in model.applications:
1611             try:
1612                 await self._juju_destroy_application(
1613                     model_name=model_name, application_name=application_name
1614                 )
1615             except Exception as e:
1616                 self.log.error(
1617                     "Error destroying application {} in model {}: {}".format(
1618                         application_name, model_name, e
1619                     )
1620                 )
1621
1622         # destroy machines
1623         machines = await model.get_machines()
1624         for machine_id in machines:
1625             try:
1626                 await self._juju_destroy_machine(
1627                     model_name=model_name, machine_id=machine_id
1628                 )
1629             except asyncio.CancelledError:
1630                 raise
1631             except Exception:
1632                 # ignore exceptions destroying machine
1633                 pass
1634
1635         await self._juju_disconnect_model(model_name=model_name)
1636
1637         self.log.debug("destroying model {}...".format(model_name))
1638         await self.controller.destroy_model(uuid)
1639         # self.log.debug('model destroy requested {}'.format(model_name))
1640
1641         # wait for model is completely destroyed
1642         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1643         last_exception = ""
1644         while time.time() < end:
1645             try:
1646                 # await self.controller.get_model(uuid)
1647                 models = await self.controller.list_models()
1648                 if model_name not in models:
1649                     self.log.debug(
1650                         "The model {} ({}) was destroyed".format(model_name, uuid)
1651                     )
1652                     return
1653             except asyncio.CancelledError:
1654                 raise
1655             except Exception as e:
1656                 last_exception = e
1657             await asyncio.sleep(5)
1658         raise N2VCException(
1659             "Timeout waiting for model {} to be destroyed {}".format(
1660                 model_name, last_exception
1661             )
1662         )
1663
1664     async def _juju_login(self):
1665         """Connect to juju controller
1666
1667         """
1668
1669         # if already authenticated, exit function
1670         if self._authenticated:
1671             return
1672
1673         # if connecting, wait for finish
1674         # another task could be trying to connect in parallel
1675         while self._connecting:
1676             await asyncio.sleep(0.1)
1677
1678         # double check after other task has finished
1679         if self._authenticated:
1680             return
1681
1682         try:
1683             self._connecting = True
1684             self.log.info(
1685                 "connecting to juju controller: {} {}:{}{}".format(
1686                     self.url,
1687                     self.username,
1688                     self.secret[:8] + "...",
1689                     " with ca_cert" if self.ca_cert else "",
1690                 )
1691             )
1692
1693             # Create controller object
1694             self.controller = Controller(loop=self.loop)
1695             # Connect to controller
1696             await self.controller.connect(
1697                 endpoint=self.url,
1698                 username=self.username,
1699                 password=self.secret,
1700                 cacert=self.ca_cert,
1701             )
1702             self._authenticated = True
1703             self.log.info("juju controller connected")
1704         except Exception as e:
1705             message = "Exception connecting to juju: {}".format(e)
1706             self.log.error(message)
1707             raise N2VCConnectionException(message=message, url=self.url)
1708         finally:
1709             self._connecting = False
1710
1711     async def _juju_logout(self):
1712         """Logout of the Juju controller."""
1713         if not self._authenticated:
1714             return False
1715
1716         # disconnect all models
1717         for model_name in self.juju_models:
1718             try:
1719                 await self._juju_disconnect_model(model_name)
1720             except Exception as e:
1721                 self.log.error(
1722                     "Error disconnecting model {} : {}".format(model_name, e)
1723                 )
1724                 # continue with next model...
1725
1726         self.log.info("Disconnecting controller")
1727         try:
1728             await self.controller.disconnect()
1729         except Exception as e:
1730             raise N2VCConnectionException(
1731                 message="Error disconnecting controller: {}".format(e), url=self.url
1732             )
1733
1734         self.controller = None
1735         self._authenticated = False
1736         self.log.info("disconnected")
1737
1738     async def _juju_disconnect_model(self, model_name: str):
1739         self.log.debug("Disconnecting model {}".format(model_name))
1740         if model_name in self.juju_models:
1741             await self.juju_models[model_name].disconnect()
1742             self.juju_models[model_name] = None
1743             self.juju_observers[model_name] = None
1744         else:
1745             self.warning("Cannot disconnect model: {}".format(model_name))
1746
1747     def _create_juju_public_key(self):
1748         """Recreate the Juju public key on lcm container, if needed
1749         Certain libjuju commands expect to be run from the same machine as Juju
1750          is bootstrapped to. This method will write the public key to disk in
1751          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1752         """
1753
1754         # Make sure that we have a public key before writing to disk
1755         if self.public_key is None or len(self.public_key) == 0:
1756             if "OSMLCM_VCA_PUBKEY" in os.environ:
1757                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1758                 if len(self.public_key) == 0:
1759                     return
1760             else:
1761                 return
1762
1763         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1764         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1765         self.log.debug(
1766             "writing juju public key to file:\n{}\npublic key: {}".format(
1767                 file_path, self.public_key
1768             )
1769         )
1770         if not os.path.exists(pk_path):
1771             # create path and write file
1772             os.makedirs(pk_path)
1773             with open(file_path, "w") as f:
1774                 self.log.debug("Creating juju public key file: {}".format(file_path))
1775                 f.write(self.public_key)
1776         else:
1777             self.log.debug("juju public key file already exists: {}".format(file_path))
1778
1779     @staticmethod
1780     def _format_model_name(name: str) -> str:
1781         """Format the name of the model.
1782
1783         Model names may only contain lowercase letters, digits and hyphens
1784         """
1785
1786         return name.replace("_", "-").replace(" ", "-").lower()
1787
1788     @staticmethod
1789     def _format_app_name(name: str) -> str:
1790         """Format the name of the application (in order to assure valid application name).
1791
1792         Application names have restrictions (run juju deploy --help):
1793             - contains lowercase letters 'a'-'z'
1794             - contains numbers '0'-'9'
1795             - contains hyphens '-'
1796             - starts with a lowercase letter
1797             - not two or more consecutive hyphens
1798             - after a hyphen, not a group with all numbers
1799         """
1800
1801         def all_numbers(s: str) -> bool:
1802             for c in s:
1803                 if not c.isdigit():
1804                     return False
1805             return True
1806
1807         new_name = name.replace("_", "-")
1808         new_name = new_name.replace(" ", "-")
1809         new_name = new_name.lower()
1810         while new_name.find("--") >= 0:
1811             new_name = new_name.replace("--", "-")
1812         groups = new_name.split("-")
1813
1814         # find 'all numbers' groups and prefix them with a letter
1815         app_name = ""
1816         for i in range(len(groups)):
1817             group = groups[i]
1818             if all_numbers(group):
1819                 group = "z" + group
1820             if i > 0:
1821                 app_name += "-"
1822             app_name += group
1823
1824         if app_name[0].isdigit():
1825             app_name = "z" + app_name
1826
1827         return app_name