n2vc/n2vc_juju_conn.py

   1 ##
   2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
   3 # This file is part of OSM
   4 # All Rights Reserved.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License");
   7 # you may not use this file except in compliance with the License.
   8 # You may obtain a copy of the License at
   9 #
  10 #    http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS,
  14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  15 # implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact with: nfvlabs@tid.es
  21 ##
  22
  23 import asyncio
  24 import base64
  25 import binascii
  26 import logging
  27 import os
  28 import re
  29 import time
  30
  31 from juju.action import Action
  32 from juju.application import Application
  33 from juju.client import client
  34 from juju.controller import Controller
  35 from juju.errors import JujuAPIError
  36 from juju.machine import Machine
  37 from juju.model import Model
  38 from n2vc.exceptions import (
  39     N2VCBadArgumentsException,
  40     N2VCException,
  41     N2VCConnectionException,
  42     N2VCExecutionException,
  43     N2VCInvalidCertificate,
  44     N2VCNotFound,
  45     MethodNotImplemented,
  46     JujuK8sProxycharmNotSupported,
  47 )
  48 from n2vc.juju_observer import JujuModelObserver
  49 from n2vc.n2vc_conn import N2VCConnector
  50 from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml
  51 from n2vc.provisioner import AsyncSSHProvisioner
  52 from n2vc.libjuju import Libjuju
  53
  54
  55 class N2VCJujuConnector(N2VCConnector):
  56
  57     """
  58 ####################################################################################
  59 ################################### P U B L I C ####################################
  60 ####################################################################################
  61     """
  62
  63     BUILT_IN_CLOUDS = ["localhost", "microk8s"]
  64
  65     def __init__(
  66         self,
  67         db: object,
  68         fs: object,
  69         log: object = None,
  70         loop: object = None,
  71         url: str = "127.0.0.1:17070",
  72         username: str = "admin",
  73         vca_config: dict = None,
  74         on_update_db=None,
  75     ):
  76         """Initialize juju N2VC connector
  77         """
  78
  79         # parent class constructor
  80         N2VCConnector.__init__(
  81             self,
  82             db=db,
  83             fs=fs,
  84             log=log,
  85             loop=loop,
  86             url=url,
  87             username=username,
  88             vca_config=vca_config,
  89             on_update_db=on_update_db,
  90         )
  91
  92         # silence websocket traffic log
  93         logging.getLogger("websockets.protocol").setLevel(logging.INFO)
  94         logging.getLogger("juju.client.connection").setLevel(logging.WARN)
  95         logging.getLogger("model").setLevel(logging.WARN)
  96
  97         self.log.info("Initializing N2VC juju connector...")
  98
  99         """
 100         ##############################################################
 101         # check arguments
 102         ##############################################################
 103         """
 104
 105         # juju URL
 106         if url is None:
 107             raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
 108         url_parts = url.split(":")
 109         if len(url_parts) != 2:
 110             raise N2VCBadArgumentsException(
 111                 "Argument url: bad format (localhost:port) -> {}".format(url), ["url"]
 112             )
 113         self.hostname = url_parts[0]
 114         try:
 115             self.port = int(url_parts[1])
 116         except ValueError:
 117             raise N2VCBadArgumentsException(
 118                 "url port must be a number -> {}".format(url), ["url"]
 119             )
 120
 121         # juju USERNAME
 122         if username is None:
 123             raise N2VCBadArgumentsException(
 124                 "Argument username is mandatory", ["username"]
 125             )
 126
 127         # juju CONFIGURATION
 128         if vca_config is None:
 129             raise N2VCBadArgumentsException(
 130                 "Argument vca_config is mandatory", ["vca_config"]
 131             )
 132
 133         if "secret" in vca_config:
 134             self.secret = vca_config["secret"]
 135         else:
 136             raise N2VCBadArgumentsException(
 137                 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
 138             )
 139
 140         # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
 141         # if exists, it will be written in lcm container: _create_juju_public_key()
 142         if "public_key" in vca_config:
 143             self.public_key = vca_config["public_key"]
 144         else:
 145             self.public_key = None
 146
 147         # TODO: Verify ca_cert is valid before using. VCA will crash
 148         # if the ca_cert isn't formatted correctly.
 149         def base64_to_cacert(b64string):
 150             """Convert the base64-encoded string containing the VCA CACERT.
 151
 152             The input string....
 153
 154             """
 155             try:
 156                 cacert = base64.b64decode(b64string).decode("utf-8")
 157
 158                 cacert = re.sub(r"\\n", r"\n", cacert,)
 159             except binascii.Error as e:
 160                 self.log.debug("Caught binascii.Error: {}".format(e))
 161                 raise N2VCInvalidCertificate(message="Invalid CA Certificate")
 162
 163             return cacert
 164
 165         self.ca_cert = vca_config.get("ca_cert")
 166         if self.ca_cert:
 167             self.ca_cert = base64_to_cacert(vca_config["ca_cert"])
 168
 169         if "api_proxy" in vca_config and vca_config["api_proxy"] != "":
 170             self.api_proxy = vca_config["api_proxy"]
 171             self.log.debug(
 172                 "api_proxy for native charms configured: {}".format(self.api_proxy)
 173             )
 174         else:
 175             self.warning(
 176                 "api_proxy is not configured"
 177             )
 178             self.api_proxy = None
 179
 180         if "enable_os_upgrade" in vca_config:
 181             self.enable_os_upgrade = vca_config["enable_os_upgrade"]
 182         else:
 183             self.enable_os_upgrade = True
 184
 185         if "apt_mirror" in vca_config:
 186             self.apt_mirror = vca_config["apt_mirror"]
 187         else:
 188             self.apt_mirror = None
 189
 190         self.cloud = vca_config.get('cloud')
 191         self.k8s_cloud = None
 192         if "k8s_cloud" in vca_config:
 193             self.k8s_cloud = vca_config.get("k8s_cloud")
 194         self.log.debug('Arguments have been checked')
 195
 196         # juju data
 197         self.controller = None  # it will be filled when connect to juju
 198         self.juju_models = {}  # model objects for every model_name
 199         self.juju_observers = {}  # model observers for every model_name
 200         self._connecting = (
 201             False  # while connecting to juju (to avoid duplicate connections)
 202         )
 203         self._authenticated = (
 204             False  # it will be True when juju connection be stablished
 205         )
 206         self._creating_model = False  # True during model creation
 207         self.libjuju = Libjuju(
 208             endpoint=self.url,
 209             api_proxy=self.api_proxy,
 210             enable_os_upgrade=self.enable_os_upgrade,
 211             apt_mirror=self.apt_mirror,
 212             username=self.username,
 213             password=self.secret,
 214             cacert=self.ca_cert,
 215             loop=self.loop,
 216             log=self.log,
 217             db=self.db,
 218             n2vc=self,
 219         )
 220
 221         # create juju pub key file in lcm container at
 222         # ./local/share/juju/ssh/juju_id_rsa.pub
 223         self._create_juju_public_key()
 224
 225         self.log.info("N2VC juju connector initialized")
 226
 227     async def get_status(self, namespace: str, yaml_format: bool = True):
 228
 229         # self.log.info('Getting NS status. namespace: {}'.format(namespace))
 230
 231         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 232             namespace=namespace
 233         )
 234         # model name is ns_id
 235         model_name = ns_id
 236         if model_name is None:
 237             msg = "Namespace {} not valid".format(namespace)
 238             self.log.error(msg)
 239             raise N2VCBadArgumentsException(msg, ["namespace"])
 240
 241         status = {}
 242         models = await self.libjuju.list_models(contains=ns_id)
 243
 244         for m in models:
 245             status[m] = await self.libjuju.get_model_status(m)
 246
 247         if yaml_format:
 248             return obj_to_yaml(status)
 249         else:
 250             return obj_to_dict(status)
 251
 252     async def create_execution_environment(
 253         self,
 254         namespace: str,
 255         db_dict: dict,
 256         reuse_ee_id: str = None,
 257         progress_timeout: float = None,
 258         total_timeout: float = None,
 259     ) -> (str, dict):
 260
 261         self.log.info(
 262             "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
 263                 namespace, reuse_ee_id
 264             )
 265         )
 266
 267         machine_id = None
 268         if reuse_ee_id:
 269             model_name, application_name, machine_id = self._get_ee_id_components(
 270                 ee_id=reuse_ee_id
 271             )
 272         else:
 273             (
 274                 _nsi_id,
 275                 ns_id,
 276                 _vnf_id,
 277                 _vdu_id,
 278                 _vdu_count,
 279             ) = self._get_namespace_components(namespace=namespace)
 280             # model name is ns_id
 281             model_name = ns_id
 282             # application name
 283             application_name = self._get_application_name(namespace=namespace)
 284
 285         self.log.debug(
 286             "model name: {}, application name:  {}, machine_id: {}".format(
 287                 model_name, application_name, machine_id
 288             )
 289         )
 290
 291         # create or reuse a new juju machine
 292         try:
 293             if not await self.libjuju.model_exists(model_name):
 294                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 295             machine, new = await self.libjuju.create_machine(
 296                 model_name=model_name,
 297                 machine_id=machine_id,
 298                 db_dict=db_dict,
 299                 progress_timeout=progress_timeout,
 300                 total_timeout=total_timeout,
 301             )
 302             # id for the execution environment
 303             ee_id = N2VCJujuConnector._build_ee_id(
 304                 model_name=model_name,
 305                 application_name=application_name,
 306                 machine_id=str(machine.entity_id),
 307             )
 308             self.log.debug("ee_id: {}".format(ee_id))
 309
 310             if new:
 311                 # write ee_id in database
 312                 self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 313
 314         except Exception as e:
 315             message = "Error creating machine on juju: {}".format(e)
 316             self.log.error(message)
 317             raise N2VCException(message=message)
 318
 319         # new machine credentials
 320         credentials = {
 321             "hostname": machine.dns_name,
 322         }
 323
 324         self.log.info(
 325             "Execution environment created. ee_id: {}, credentials: {}".format(
 326                 ee_id, credentials
 327             )
 328         )
 329
 330         return ee_id, credentials
 331
 332     async def register_execution_environment(
 333         self,
 334         namespace: str,
 335         credentials: dict,
 336         db_dict: dict,
 337         progress_timeout: float = None,
 338         total_timeout: float = None,
 339     ) -> str:
 340
 341         self.log.info(
 342             "Registering execution environment. namespace={}, credentials={}".format(
 343                 namespace, credentials
 344             )
 345         )
 346
 347         if credentials is None:
 348             raise N2VCBadArgumentsException(
 349                 message="credentials are mandatory", bad_args=["credentials"]
 350             )
 351         if credentials.get("hostname"):
 352             hostname = credentials["hostname"]
 353         else:
 354             raise N2VCBadArgumentsException(
 355                 message="hostname is mandatory", bad_args=["credentials.hostname"]
 356             )
 357         if credentials.get("username"):
 358             username = credentials["username"]
 359         else:
 360             raise N2VCBadArgumentsException(
 361                 message="username is mandatory", bad_args=["credentials.username"]
 362             )
 363         if "private_key_path" in credentials:
 364             private_key_path = credentials["private_key_path"]
 365         else:
 366             # if not passed as argument, use generated private key path
 367             private_key_path = self.private_key_path
 368
 369         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 370             namespace=namespace
 371         )
 372
 373         # model name
 374         model_name = ns_id
 375         # application name
 376         application_name = self._get_application_name(namespace=namespace)
 377
 378         # register machine on juju
 379         try:
 380             if not await self.libjuju.model_exists(model_name):
 381                 await self.libjuju.add_model(model_name, cloud_name=self.cloud)
 382             machine_id = await self.libjuju.provision_machine(
 383                 model_name=model_name,
 384                 hostname=hostname,
 385                 username=username,
 386                 private_key_path=private_key_path,
 387                 db_dict=db_dict,
 388                 progress_timeout=progress_timeout,
 389                 total_timeout=total_timeout,
 390             )
 391         except Exception as e:
 392             self.log.error("Error registering machine: {}".format(e))
 393             raise N2VCException(
 394                 message="Error registering machine on juju: {}".format(e)
 395             )
 396
 397         self.log.info("Machine registered: {}".format(machine_id))
 398
 399         # id for the execution environment
 400         ee_id = N2VCJujuConnector._build_ee_id(
 401             model_name=model_name,
 402             application_name=application_name,
 403             machine_id=str(machine_id),
 404         )
 405
 406         self.log.info("Execution environment registered. ee_id: {}".format(ee_id))
 407
 408         return ee_id
 409
 410     async def install_configuration_sw(
 411         self,
 412         ee_id: str,
 413         artifact_path: str,
 414         db_dict: dict,
 415         progress_timeout: float = None,
 416         total_timeout: float = None,
 417         config: dict = None,
 418         num_units: int = 1,
 419     ):
 420
 421         self.log.info(
 422             (
 423                 "Installing configuration sw on ee_id: {}, "
 424                 "artifact path: {}, db_dict: {}"
 425             ).format(ee_id, artifact_path, db_dict)
 426         )
 427
 428         # check arguments
 429         if ee_id is None or len(ee_id) == 0:
 430             raise N2VCBadArgumentsException(
 431                 message="ee_id is mandatory", bad_args=["ee_id"]
 432             )
 433         if artifact_path is None or len(artifact_path) == 0:
 434             raise N2VCBadArgumentsException(
 435                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 436             )
 437         if db_dict is None:
 438             raise N2VCBadArgumentsException(
 439                 message="db_dict is mandatory", bad_args=["db_dict"]
 440             )
 441
 442         try:
 443             (
 444                 model_name,
 445                 application_name,
 446                 machine_id,
 447             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 448             self.log.debug(
 449                 "model: {}, application: {}, machine: {}".format(
 450                     model_name, application_name, machine_id
 451                 )
 452             )
 453         except Exception:
 454             raise N2VCBadArgumentsException(
 455                 message="ee_id={} is not a valid execution environment id".format(
 456                     ee_id
 457                 ),
 458                 bad_args=["ee_id"],
 459             )
 460
 461         # remove // in charm path
 462         while artifact_path.find("//") >= 0:
 463             artifact_path = artifact_path.replace("//", "/")
 464
 465         # check charm path
 466         if not self.fs.file_exists(artifact_path, mode="dir"):
 467             msg = "artifact path does not exist: {}".format(artifact_path)
 468             raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"])
 469
 470         if artifact_path.startswith("/"):
 471             full_path = self.fs.path + artifact_path
 472         else:
 473             full_path = self.fs.path + "/" + artifact_path
 474
 475         try:
 476             await self.libjuju.deploy_charm(
 477                 model_name=model_name,
 478                 application_name=application_name,
 479                 path=full_path,
 480                 machine_id=machine_id,
 481                 db_dict=db_dict,
 482                 progress_timeout=progress_timeout,
 483                 total_timeout=total_timeout,
 484                 config=config,
 485                 num_units=num_units,
 486             )
 487         except Exception as e:
 488             raise N2VCException(
 489                 message="Error desploying charm into ee={} : {}".format(ee_id, e)
 490             )
 491
 492         self.log.info("Configuration sw installed")
 493
 494     async def install_k8s_proxy_charm(
 495         self,
 496         charm_name: str,
 497         namespace: str,
 498         artifact_path: str,
 499         db_dict: dict,
 500         progress_timeout: float = None,
 501         total_timeout: float = None,
 502         config: dict = None,
 503     ) -> str:
 504         """
 505         Install a k8s proxy charm
 506
 507         :param charm_name: Name of the charm being deployed
 508         :param namespace: collection of all the uuids related to the charm.
 509         :param str artifact_path: where to locate the artifacts (parent folder) using
 510             the self.fs
 511             the final artifact path will be a combination of this artifact_path and
 512             additional string from the config_dict (e.g. charm name)
 513         :param dict db_dict: where to write into database when the status changes.
 514                         It contains a dict with
 515                             {collection: <str>, filter: {},  path: <str>},
 516                             e.g. {collection: "nsrs", filter:
 517                                 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
 518         :param float progress_timeout:
 519         :param float total_timeout:
 520         :param config: Dictionary with additional configuration
 521
 522         :returns ee_id: execution environment id.
 523         """
 524         self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
 525                       .format(charm_name, artifact_path, db_dict))
 526
 527         if not self.k8s_cloud:
 528             raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
 529
 530         if artifact_path is None or len(artifact_path) == 0:
 531             raise N2VCBadArgumentsException(
 532                 message="artifact_path is mandatory", bad_args=["artifact_path"]
 533             )
 534         if db_dict is None:
 535             raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict'])
 536
 537         # remove // in charm path
 538         while artifact_path.find('//') >= 0:
 539             artifact_path = artifact_path.replace('//', '/')
 540
 541         # check charm path
 542         if not self.fs.file_exists(artifact_path, mode="dir"):
 543             msg = 'artifact path does not exist: {}'.format(artifact_path)
 544             raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path'])
 545
 546         if artifact_path.startswith('/'):
 547             full_path = self.fs.path + artifact_path
 548         else:
 549             full_path = self.fs.path + '/' + artifact_path
 550
 551         _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace)
 552         model_name = '{}-k8s'.format(ns_id)
 553
 554         await self.libjuju.add_model(model_name, self.k8s_cloud)
 555         application_name = self._get_application_name(namespace)
 556
 557         try:
 558             await self.libjuju.deploy_charm(
 559                 model_name=model_name,
 560                 application_name=application_name,
 561                 path=full_path,
 562                 machine_id=None,
 563                 db_dict=db_dict,
 564                 progress_timeout=progress_timeout,
 565                 total_timeout=total_timeout,
 566                 config=config
 567             )
 568         except Exception as e:
 569             raise N2VCException(message='Error deploying charm: {}'.format(e))
 570
 571         self.log.info('K8s proxy charm installed')
 572         ee_id = N2VCJujuConnector._build_ee_id(
 573             model_name=model_name,
 574             application_name=application_name,
 575             machine_id="k8s",
 576         )
 577
 578         self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
 579
 580         return ee_id
 581
 582     async def get_ee_ssh_public__key(
 583         self,
 584         ee_id: str,
 585         db_dict: dict,
 586         progress_timeout: float = None,
 587         total_timeout: float = None,
 588     ) -> str:
 589
 590         self.log.info(
 591             (
 592                 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
 593             ).format(ee_id, db_dict)
 594         )
 595
 596         # check arguments
 597         if ee_id is None or len(ee_id) == 0:
 598             raise N2VCBadArgumentsException(
 599                 message="ee_id is mandatory", bad_args=["ee_id"]
 600             )
 601         if db_dict is None:
 602             raise N2VCBadArgumentsException(
 603                 message="db_dict is mandatory", bad_args=["db_dict"]
 604             )
 605
 606         try:
 607             (
 608                 model_name,
 609                 application_name,
 610                 machine_id,
 611             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 612             self.log.debug(
 613                 "model: {}, application: {}, machine: {}".format(
 614                     model_name, application_name, machine_id
 615                 )
 616             )
 617         except Exception:
 618             raise N2VCBadArgumentsException(
 619                 message="ee_id={} is not a valid execution environment id".format(
 620                     ee_id
 621                 ),
 622                 bad_args=["ee_id"],
 623             )
 624
 625         # try to execute ssh layer primitives (if exist):
 626         #       generate-ssh-key
 627         #       get-ssh-public-key
 628
 629         output = None
 630
 631         application_name = N2VCJujuConnector._format_app_name(application_name)
 632
 633         # execute action: generate-ssh-key
 634         try:
 635             output, _status = await self.libjuju.execute_action(
 636                 model_name=model_name,
 637                 application_name=application_name,
 638                 action_name="generate-ssh-key",
 639                 db_dict=db_dict,
 640                 progress_timeout=progress_timeout,
 641                 total_timeout=total_timeout,
 642             )
 643         except Exception as e:
 644             self.log.info(
 645                 "Skipping exception while executing action generate-ssh-key: {}".format(
 646                     e
 647                 )
 648             )
 649
 650         # execute action: get-ssh-public-key
 651         try:
 652             output, _status = await self.libjuju.execute_action(
 653                 model_name=model_name,
 654                 application_name=application_name,
 655                 action_name="get-ssh-public-key",
 656                 db_dict=db_dict,
 657                 progress_timeout=progress_timeout,
 658                 total_timeout=total_timeout,
 659             )
 660         except Exception as e:
 661             msg = "Cannot execute action get-ssh-public-key: {}\n".format(e)
 662             self.log.info(msg)
 663             raise N2VCExecutionException(e, primitive_name="get-ssh-public-key")
 664
 665         # return public key if exists
 666         return output["pubkey"] if "pubkey" in output else output
 667
 668     async def add_relation(
 669         self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str
 670     ):
 671
 672         self.log.debug(
 673             "adding new relation between {} and {}, endpoints: {}, {}".format(
 674                 ee_id_1, ee_id_2, endpoint_1, endpoint_2
 675             )
 676         )
 677
 678         # check arguments
 679         if not ee_id_1:
 680             message = "EE 1 is mandatory"
 681             self.log.error(message)
 682             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"])
 683         if not ee_id_2:
 684             message = "EE 2 is mandatory"
 685             self.log.error(message)
 686             raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"])
 687         if not endpoint_1:
 688             message = "endpoint 1 is mandatory"
 689             self.log.error(message)
 690             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"])
 691         if not endpoint_2:
 692             message = "endpoint 2 is mandatory"
 693             self.log.error(message)
 694             raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"])
 695
 696         # get the model, the applications and the machines from the ee_id's
 697         model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1)
 698         model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2)
 699
 700         # model must be the same
 701         if model_1 != model_2:
 702             message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2)
 703             self.log.error(message)
 704             raise N2VCBadArgumentsException(
 705                 message=message, bad_args=["ee_id_1", "ee_id_2"]
 706             )
 707
 708         # add juju relations between two applications
 709         try:
 710             await self.libjuju.add_relation(
 711                 model_name=model_1,
 712                 endpoint_1="{}:{}".format(app_1, endpoint_1),
 713                 endpoint_2="{}:{}".format(app_2, endpoint_2),
 714             )
 715         except Exception as e:
 716             message = "Error adding relation between {} and {}: {}".format(
 717                 ee_id_1, ee_id_2, e
 718             )
 719             self.log.error(message)
 720             raise N2VCException(message=message)
 721
 722     async def remove_relation(self):
 723         # TODO
 724         self.log.info("Method not implemented yet")
 725         raise MethodNotImplemented()
 726
 727     async def deregister_execution_environments(self):
 728         self.log.info("Method not implemented yet")
 729         raise MethodNotImplemented()
 730
 731     async def delete_namespace(
 732         self, namespace: str, db_dict: dict = None, total_timeout: float = None
 733     ):
 734         self.log.info("Deleting namespace={}".format(namespace))
 735
 736         # check arguments
 737         if namespace is None:
 738             raise N2VCBadArgumentsException(
 739                 message="namespace is mandatory", bad_args=["namespace"]
 740             )
 741
 742         _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components(
 743             namespace=namespace
 744         )
 745         if ns_id is not None:
 746             try:
 747                 models = await self.libjuju.list_models(contains=ns_id)
 748                 for model in models:
 749                     await self.libjuju.destroy_model(
 750                         model_name=model, total_timeout=total_timeout
 751                     )
 752             except Exception as e:
 753                 raise N2VCException(
 754                     message="Error deleting namespace {} : {}".format(namespace, e)
 755                 )
 756         else:
 757             raise N2VCBadArgumentsException(
 758                 message="only ns_id is permitted to delete yet", bad_args=["namespace"]
 759             )
 760
 761         self.log.info("Namespace {} deleted".format(namespace))
 762
 763     async def delete_execution_environment(
 764         self, ee_id: str, db_dict: dict = None, total_timeout: float = None
 765     ):
 766         self.log.info("Deleting execution environment ee_id={}".format(ee_id))
 767
 768         # check arguments
 769         if ee_id is None:
 770             raise N2VCBadArgumentsException(
 771                 message="ee_id is mandatory", bad_args=["ee_id"]
 772             )
 773
 774         model_name, application_name, _machine_id = self._get_ee_id_components(
 775             ee_id=ee_id
 776         )
 777
 778         # destroy the application
 779         try:
 780             await self.libjuju.destroy_model(
 781                 model_name=model_name, total_timeout=total_timeout
 782             )
 783         except Exception as e:
 784             raise N2VCException(
 785                 message=(
 786                     "Error deleting execution environment {} (application {}) : {}"
 787                 ).format(ee_id, application_name, e)
 788             )
 789
 790         # destroy the machine
 791         # try:
 792         #     await self._juju_destroy_machine(
 793         #         model_name=model_name,
 794         #         machine_id=machine_id,
 795         #         total_timeout=total_timeout
 796         #     )
 797         # except Exception as e:
 798         #     raise N2VCException(
 799         #        message='Error deleting execution environment {} (machine {}) : {}'
 800         #                .format(ee_id, machine_id, e))
 801
 802         self.log.info("Execution environment {} deleted".format(ee_id))
 803
 804     async def exec_primitive(
 805         self,
 806         ee_id: str,
 807         primitive_name: str,
 808         params_dict: dict,
 809         db_dict: dict = None,
 810         progress_timeout: float = None,
 811         total_timeout: float = None,
 812     ) -> str:
 813
 814         self.log.info(
 815             "Executing primitive: {} on ee: {}, params: {}".format(
 816                 primitive_name, ee_id, params_dict
 817             )
 818         )
 819
 820         # check arguments
 821         if ee_id is None or len(ee_id) == 0:
 822             raise N2VCBadArgumentsException(
 823                 message="ee_id is mandatory", bad_args=["ee_id"]
 824             )
 825         if primitive_name is None or len(primitive_name) == 0:
 826             raise N2VCBadArgumentsException(
 827                 message="action_name is mandatory", bad_args=["action_name"]
 828             )
 829         if params_dict is None:
 830             params_dict = dict()
 831
 832         try:
 833             (
 834                 model_name,
 835                 application_name,
 836                 _machine_id,
 837             ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id)
 838         except Exception:
 839             raise N2VCBadArgumentsException(
 840                 message="ee_id={} is not a valid execution environment id".format(
 841                     ee_id
 842                 ),
 843                 bad_args=["ee_id"],
 844             )
 845
 846         if primitive_name == "config":
 847             # Special case: config primitive
 848             try:
 849                 await self.libjuju.configure_application(
 850                     model_name=model_name,
 851                     application_name=application_name,
 852                     config=params_dict,
 853                 )
 854                 actions = await self.libjuju.get_actions(
 855                     application_name=application_name, model_name=model_name,
 856                 )
 857                 self.log.debug(
 858                     "Application {} has these actions: {}".format(
 859                         application_name, actions
 860                     )
 861                 )
 862                 if "verify-ssh-credentials" in actions:
 863                     # execute verify-credentials
 864                     num_retries = 20
 865                     retry_timeout = 15.0
 866                     for _ in range(num_retries):
 867                         try:
 868                             self.log.debug("Executing action verify-ssh-credentials...")
 869                             output, ok = await self.libjuju.execute_action(
 870                                 model_name=model_name,
 871                                 application_name=application_name,
 872                                 action_name="verify-ssh-credentials",
 873                                 db_dict=db_dict,
 874                                 progress_timeout=progress_timeout,
 875                                 total_timeout=total_timeout,
 876                             )
 877
 878                             if ok == "failed":
 879                                 self.log.debug(
 880                                     "Error executing verify-ssh-credentials: {}. Retrying..."
 881                                 )
 882                                 await asyncio.sleep(retry_timeout)
 883
 884                                 continue
 885                             self.log.debug("Result: {}, output: {}".format(ok, output))
 886                             break
 887                         except asyncio.CancelledError:
 888                             raise
 889                     else:
 890                         self.log.error(
 891                             "Error executing verify-ssh-credentials after {} retries. ".format(
 892                                 num_retries
 893                             )
 894                         )
 895                 else:
 896                     msg = "Action verify-ssh-credentials does not exist in application {}".format(
 897                         application_name
 898                     )
 899                     self.log.debug(msg=msg)
 900             except Exception as e:
 901                 self.log.error("Error configuring juju application: {}".format(e))
 902                 raise N2VCExecutionException(
 903                     message="Error configuring application into ee={} : {}".format(
 904                         ee_id, e
 905                     ),
 906                     primitive_name=primitive_name,
 907                 )
 908             return "CONFIG OK"
 909         else:
 910             try:
 911                 output, status = await self.libjuju.execute_action(
 912                     model_name=model_name,
 913                     application_name=application_name,
 914                     action_name=primitive_name,
 915                     db_dict=db_dict,
 916                     progress_timeout=progress_timeout,
 917                     total_timeout=total_timeout,
 918                     **params_dict
 919                 )
 920                 if status == "completed":
 921                     return output
 922                 else:
 923                     raise Exception("status is not completed: {}".format(status))
 924             except Exception as e:
 925                 self.log.error(
 926                     "Error executing primitive {}: {}".format(primitive_name, e)
 927                 )
 928                 raise N2VCExecutionException(
 929                     message="Error executing primitive {} into ee={} : {}".format(
 930                         primitive_name, ee_id, e
 931                     ),
 932                     primitive_name=primitive_name,
 933                 )
 934
 935     async def disconnect(self):
 936         self.log.info("closing juju N2VC...")
 937         try:
 938             await self.libjuju.disconnect()
 939         except Exception as e:
 940             raise N2VCConnectionException(
 941                 message="Error disconnecting controller: {}".format(e), url=self.url
 942             )
 943
 944     """
 945 ####################################################################################
 946 ################################### P R I V A T E ##################################
 947 ####################################################################################
 948     """
 949
 950     def _write_ee_id_db(self, db_dict: dict, ee_id: str):
 951
 952         # write ee_id to database: _admin.deployed.VCA.x
 953         try:
 954             the_table = db_dict["collection"]
 955             the_filter = db_dict["filter"]
 956             the_path = db_dict["path"]
 957             if not the_path[-1] == ".":
 958                 the_path = the_path + "."
 959             update_dict = {the_path + "ee_id": ee_id}
 960             # self.log.debug('Writing ee_id to database: {}'.format(the_path))
 961             self.db.set_one(
 962                 table=the_table,
 963                 q_filter=the_filter,
 964                 update_dict=update_dict,
 965                 fail_on_empty=True,
 966             )
 967         except asyncio.CancelledError:
 968             raise
 969         except Exception as e:
 970             self.log.error("Error writing ee_id to database: {}".format(e))
 971
 972     @staticmethod
 973     def _build_ee_id(model_name: str, application_name: str, machine_id: str):
 974         """
 975         Build an execution environment id form model, application and machine
 976         :param model_name:
 977         :param application_name:
 978         :param machine_id:
 979         :return:
 980         """
 981         # id for the execution environment
 982         return "{}.{}.{}".format(model_name, application_name, machine_id)
 983
 984     @staticmethod
 985     def _get_ee_id_components(ee_id: str) -> (str, str, str):
 986         """
 987         Get model, application and machine components from an execution environment id
 988         :param ee_id:
 989         :return: model_name, application_name, machine_id
 990         """
 991
 992         if ee_id is None:
 993             return None, None, None
 994
 995         # split components of id
 996         parts = ee_id.split(".")
 997         model_name = parts[0]
 998         application_name = parts[1]
 999         machine_id = parts[2]
1000         return model_name, application_name, machine_id
1001
1002     def _get_application_name(self, namespace: str) -> str:
1003         """
1004         Build application name from namespace
1005         :param namespace:
1006         :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1007         """
1008
1009         # TODO: Enforce the Juju 50-character application limit
1010
1011         # split namespace components
1012         _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components(
1013             namespace=namespace
1014         )
1015
1016         if vnf_id is None or len(vnf_id) == 0:
1017             vnf_id = ""
1018         else:
1019             # Shorten the vnf_id to its last twelve characters
1020             vnf_id = "vnf-" + vnf_id[-12:]
1021
1022         if vdu_id is None or len(vdu_id) == 0:
1023             vdu_id = ""
1024         else:
1025             # Shorten the vdu_id to its last twelve characters
1026             vdu_id = "-vdu-" + vdu_id[-12:]
1027
1028         if vdu_count is None or len(vdu_count) == 0:
1029             vdu_count = ""
1030         else:
1031             vdu_count = "-cnt-" + vdu_count
1032
1033         application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count)
1034
1035         return N2VCJujuConnector._format_app_name(application_name)
1036
1037     async def _juju_create_machine(
1038         self,
1039         model_name: str,
1040         application_name: str,
1041         machine_id: str = None,
1042         db_dict: dict = None,
1043         progress_timeout: float = None,
1044         total_timeout: float = None,
1045     ) -> Machine:
1046
1047         self.log.debug(
1048             "creating machine in model: {}, existing machine id: {}".format(
1049                 model_name, machine_id
1050             )
1051         )
1052
1053         # get juju model and observer (create model if needed)
1054         model = await self._juju_get_model(model_name=model_name)
1055         observer = self.juju_observers[model_name]
1056
1057         # find machine id in model
1058         machine = None
1059         if machine_id is not None:
1060             self.log.debug("Finding existing machine id {} in model".format(machine_id))
1061             # get juju existing machines in the model
1062             existing_machines = await model.get_machines()
1063             if machine_id in existing_machines:
1064                 self.log.debug(
1065                     "Machine id {} found in model (reusing it)".format(machine_id)
1066                 )
1067                 machine = model.machines[machine_id]
1068
1069         if machine is None:
1070             self.log.debug("Creating a new machine in juju...")
1071             # machine does not exist, create it and wait for it
1072             machine = await model.add_machine(
1073                 spec=None, constraints=None, disks=None, series="xenial"
1074             )
1075
1076             # register machine with observer
1077             observer.register_machine(machine=machine, db_dict=db_dict)
1078
1079             # id for the execution environment
1080             ee_id = N2VCJujuConnector._build_ee_id(
1081                 model_name=model_name,
1082                 application_name=application_name,
1083                 machine_id=str(machine.entity_id),
1084             )
1085
1086             # write ee_id in database
1087             self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id)
1088
1089             # wait for machine creation
1090             await observer.wait_for_machine(
1091                 machine_id=str(machine.entity_id),
1092                 progress_timeout=progress_timeout,
1093                 total_timeout=total_timeout,
1094             )
1095
1096         else:
1097
1098             self.log.debug("Reusing old machine pending")
1099
1100             # register machine with observer
1101             observer.register_machine(machine=machine, db_dict=db_dict)
1102
1103             # machine does exist, but it is in creation process (pending), wait for
1104             # create finalisation
1105             await observer.wait_for_machine(
1106                 machine_id=machine.entity_id,
1107                 progress_timeout=progress_timeout,
1108                 total_timeout=total_timeout,
1109             )
1110
1111         self.log.debug("Machine ready at " + str(machine.dns_name))
1112         return machine
1113
1114     async def _juju_provision_machine(
1115         self,
1116         model_name: str,
1117         hostname: str,
1118         username: str,
1119         private_key_path: str,
1120         db_dict: dict = None,
1121         progress_timeout: float = None,
1122         total_timeout: float = None,
1123     ) -> str:
1124
1125         if not self.api_proxy:
1126             msg = "Cannot provision machine: api_proxy is not defined"
1127             self.log.error(msg=msg)
1128             raise N2VCException(message=msg)
1129
1130         self.log.debug(
1131             "provisioning machine. model: {}, hostname: {}, username: {}".format(
1132                 model_name, hostname, username
1133             )
1134         )
1135
1136         if not self._authenticated:
1137             await self._juju_login()
1138
1139         # get juju model and observer
1140         model = await self._juju_get_model(model_name=model_name)
1141         observer = self.juju_observers[model_name]
1142
1143         # TODO check if machine is already provisioned
1144         machine_list = await model.get_machines()
1145
1146         provisioner = AsyncSSHProvisioner(
1147             host=hostname,
1148             user=username,
1149             private_key_path=private_key_path,
1150             log=self.log,
1151         )
1152
1153         params = None
1154         try:
1155             params = await provisioner.provision_machine()
1156         except Exception as ex:
1157             msg = "Exception provisioning machine: {}".format(ex)
1158             self.log.error(msg)
1159             raise N2VCException(message=msg)
1160
1161         params.jobs = ["JobHostUnits"]
1162
1163         connection = model.connection()
1164
1165         # Submit the request.
1166         self.log.debug("Adding machine to model")
1167         client_facade = client.ClientFacade.from_connection(connection)
1168         results = await client_facade.AddMachines(params=[params])
1169         error = results.machines[0].error
1170         if error:
1171             msg = "Error adding machine: {}".format(error.message)
1172             self.log.error(msg=msg)
1173             raise ValueError(msg)
1174
1175         machine_id = results.machines[0].machine
1176
1177         # Need to run this after AddMachines has been called,
1178         # as we need the machine_id
1179         self.log.debug("Installing Juju agent into machine {}".format(machine_id))
1180         asyncio.ensure_future(
1181             provisioner.install_agent(
1182                 connection=connection,
1183                 nonce=params.nonce,
1184                 machine_id=machine_id,
1185                 proxy=self.api_proxy,
1186             )
1187         )
1188
1189         # wait for machine in model (now, machine is not yet in model, so we must
1190         # wait for it)
1191         machine = None
1192         for _ in range(10):
1193             machine_list = await model.get_machines()
1194             if machine_id in machine_list:
1195                 self.log.debug("Machine {} found in model!".format(machine_id))
1196                 machine = model.machines.get(machine_id)
1197                 break
1198             await asyncio.sleep(2)
1199
1200         if machine is None:
1201             msg = "Machine {} not found in model".format(machine_id)
1202             self.log.error(msg=msg)
1203             raise Exception(msg)
1204
1205         # register machine with observer
1206         observer.register_machine(machine=machine, db_dict=db_dict)
1207
1208         # wait for machine creation
1209         self.log.debug("waiting for provision finishes... {}".format(machine_id))
1210         await observer.wait_for_machine(
1211             machine_id=machine_id,
1212             progress_timeout=progress_timeout,
1213             total_timeout=total_timeout,
1214         )
1215
1216         self.log.debug("Machine provisioned {}".format(machine_id))
1217
1218         return machine_id
1219
1220     async def _juju_deploy_charm(
1221         self,
1222         model_name: str,
1223         application_name: str,
1224         charm_path: str,
1225         machine_id: str,
1226         db_dict: dict,
1227         progress_timeout: float = None,
1228         total_timeout: float = None,
1229         config: dict = None,
1230     ) -> (Application, int):
1231
1232         # get juju model and observer
1233         model = await self._juju_get_model(model_name=model_name)
1234         observer = self.juju_observers[model_name]
1235
1236         # check if application already exists
1237         application = None
1238         if application_name in model.applications:
1239             application = model.applications[application_name]
1240
1241         if application is None:
1242
1243             # application does not exist, create it and wait for it
1244             self.log.debug(
1245                 "deploying application {} to machine {}, model {}".format(
1246                     application_name, machine_id, model_name
1247                 )
1248             )
1249             self.log.debug("charm: {}".format(charm_path))
1250             machine = model.machines[machine_id]
1251             # series = None
1252             application = await model.deploy(
1253                 entity_url=charm_path,
1254                 application_name=application_name,
1255                 channel="stable",
1256                 num_units=1,
1257                 series=machine.series,
1258                 to=machine_id,
1259                 config=config,
1260             )
1261
1262             # register application with observer
1263             observer.register_application(application=application, db_dict=db_dict)
1264
1265             self.log.debug(
1266                 "waiting for application deployed... {}".format(application.entity_id)
1267             )
1268             retries = await observer.wait_for_application(
1269                 application_id=application.entity_id,
1270                 progress_timeout=progress_timeout,
1271                 total_timeout=total_timeout,
1272             )
1273             self.log.debug("application deployed")
1274
1275         else:
1276
1277             # register application with observer
1278             observer.register_application(application=application, db_dict=db_dict)
1279
1280             # application already exists, but not finalised
1281             self.log.debug("application already exists, waiting for deployed...")
1282             retries = await observer.wait_for_application(
1283                 application_id=application.entity_id,
1284                 progress_timeout=progress_timeout,
1285                 total_timeout=total_timeout,
1286             )
1287             self.log.debug("application deployed")
1288
1289         return application, retries
1290
1291     async def _juju_execute_action(
1292         self,
1293         model_name: str,
1294         application_name: str,
1295         action_name: str,
1296         db_dict: dict,
1297         progress_timeout: float = None,
1298         total_timeout: float = None,
1299         **kwargs
1300     ) -> Action:
1301
1302         # get juju model and observer
1303         model = await self._juju_get_model(model_name=model_name)
1304         observer = self.juju_observers[model_name]
1305
1306         application = await self._juju_get_application(
1307             model_name=model_name, application_name=application_name
1308         )
1309
1310         unit = None
1311         for u in application.units:
1312             if await u.is_leader_from_status():
1313                 unit = u
1314         if unit is not None:
1315             actions = await application.get_actions()
1316             if action_name in actions:
1317                 self.log.debug(
1318                     'executing action "{}" using params: {}'.format(action_name, kwargs)
1319                 )
1320                 action = await unit.run_action(action_name, **kwargs)
1321
1322                 # register action with observer
1323                 observer.register_action(action=action, db_dict=db_dict)
1324
1325                 await observer.wait_for_action(
1326                     action_id=action.entity_id,
1327                     progress_timeout=progress_timeout,
1328                     total_timeout=total_timeout,
1329                 )
1330                 self.log.debug("action completed with status: {}".format(action.status))
1331                 output = await model.get_action_output(action_uuid=action.entity_id)
1332                 status = await model.get_action_status(uuid_or_prefix=action.entity_id)
1333                 if action.entity_id in status:
1334                     status = status[action.entity_id]
1335                 else:
1336                     status = "failed"
1337                 return output, status
1338
1339         raise N2VCExecutionException(
1340             message="Cannot execute action on charm", primitive_name=action_name
1341         )
1342
1343     async def _juju_configure_application(
1344         self,
1345         model_name: str,
1346         application_name: str,
1347         config: dict,
1348         db_dict: dict,
1349         progress_timeout: float = None,
1350         total_timeout: float = None,
1351     ):
1352
1353         # get the application
1354         application = await self._juju_get_application(
1355             model_name=model_name, application_name=application_name
1356         )
1357
1358         self.log.debug(
1359             "configuring the application {} -> {}".format(application_name, config)
1360         )
1361         res = await application.set_config(config)
1362         self.log.debug(
1363             "application {} configured. res={}".format(application_name, res)
1364         )
1365
1366         # Verify the config is set
1367         new_conf = await application.get_config()
1368         for key in config:
1369             value = new_conf[key]["value"]
1370             self.log.debug("    {} = {}".format(key, value))
1371             if config[key] != value:
1372                 raise N2VCException(
1373                     message="key {} is not configured correctly {} != {}".format(
1374                         key, config[key], new_conf[key]
1375                     )
1376                 )
1377
1378         # check if 'verify-ssh-credentials' action exists
1379         # unit = application.units[0]
1380         actions = await application.get_actions()
1381         if "verify-ssh-credentials" not in actions:
1382             msg = (
1383                 "Action verify-ssh-credentials does not exist in application {}"
1384             ).format(application_name)
1385             self.log.debug(msg=msg)
1386             return False
1387
1388         # execute verify-credentials
1389         num_retries = 20
1390         retry_timeout = 15.0
1391         for _ in range(num_retries):
1392             try:
1393                 self.log.debug("Executing action verify-ssh-credentials...")
1394                 output, ok = await self._juju_execute_action(
1395                     model_name=model_name,
1396                     application_name=application_name,
1397                     action_name="verify-ssh-credentials",
1398                     db_dict=db_dict,
1399                     progress_timeout=progress_timeout,
1400                     total_timeout=total_timeout,
1401                 )
1402                 self.log.debug("Result: {}, output: {}".format(ok, output))
1403                 return True
1404             except asyncio.CancelledError:
1405                 raise
1406             except Exception as e:
1407                 self.log.debug(
1408                     "Error executing verify-ssh-credentials: {}. Retrying...".format(e)
1409                 )
1410                 await asyncio.sleep(retry_timeout)
1411         else:
1412             self.log.error(
1413                 "Error executing verify-ssh-credentials after {} retries. ".format(
1414                     num_retries
1415                 )
1416             )
1417             return False
1418
1419     async def _juju_get_application(self, model_name: str, application_name: str):
1420         """Get the deployed application."""
1421
1422         model = await self._juju_get_model(model_name=model_name)
1423
1424         application_name = N2VCJujuConnector._format_app_name(application_name)
1425
1426         if model.applications and application_name in model.applications:
1427             return model.applications[application_name]
1428         else:
1429             raise N2VCException(
1430                 message="Cannot get application {} from model {}".format(
1431                     application_name, model_name
1432                 )
1433             )
1434
1435     async def _juju_get_model(self, model_name: str) -> Model:
1436         """ Get a model object from juju controller
1437         If the model does not exits, it creates it.
1438
1439         :param str model_name: name of the model
1440         :returns Model: model obtained from juju controller or Exception
1441         """
1442
1443         # format model name
1444         model_name = N2VCJujuConnector._format_model_name(model_name)
1445
1446         if model_name in self.juju_models:
1447             return self.juju_models[model_name]
1448
1449         if self._creating_model:
1450             self.log.debug("Another coroutine is creating a model. Wait...")
1451         while self._creating_model:
1452             # another coroutine is creating a model, wait
1453             await asyncio.sleep(0.1)
1454             # retry (perhaps another coroutine has created the model meanwhile)
1455             if model_name in self.juju_models:
1456                 return self.juju_models[model_name]
1457
1458         try:
1459             self._creating_model = True
1460
1461             # get juju model names from juju
1462             model_list = await self.controller.list_models()
1463             if model_name not in model_list:
1464                 self.log.info(
1465                     "Model {} does not exist. Creating new model...".format(model_name)
1466                 )
1467                 config_dict = {"authorized-keys": self.public_key}
1468                 if self.apt_mirror:
1469                     config_dict["apt-mirror"] = self.apt_mirror
1470                 if not self.enable_os_upgrade:
1471                     config_dict["enable-os-refresh-update"] = False
1472                     config_dict["enable-os-upgrade"] = False
1473                 if self.cloud in self.BUILT_IN_CLOUDS:
1474                     model = await self.controller.add_model(
1475                         model_name=model_name,
1476                         config=config_dict,
1477                         cloud_name=self.cloud,
1478                     )
1479                 else:
1480                     model = await self.controller.add_model(
1481                         model_name=model_name,
1482                         config=config_dict,
1483                         cloud_name=self.cloud,
1484                         credential_name=self.cloud,
1485                     )
1486                 self.log.info("New model created, name={}".format(model_name))
1487             else:
1488                 self.log.debug(
1489                     "Model already exists in juju. Getting model {}".format(model_name)
1490                 )
1491                 model = await self.controller.get_model(model_name)
1492                 self.log.debug("Existing model in juju, name={}".format(model_name))
1493
1494             self.juju_models[model_name] = model
1495             self.juju_observers[model_name] = JujuModelObserver(n2vc=self, model=model)
1496             return model
1497
1498         except Exception as e:
1499             msg = "Cannot get model {}. Exception: {}".format(model_name, e)
1500             self.log.error(msg)
1501             raise N2VCException(msg)
1502         finally:
1503             self._creating_model = False
1504
1505     async def _juju_add_relation(
1506         self,
1507         model_name: str,
1508         application_name_1: str,
1509         application_name_2: str,
1510         relation_1: str,
1511         relation_2: str,
1512     ):
1513
1514         # get juju model and observer
1515         model = await self._juju_get_model(model_name=model_name)
1516
1517         r1 = "{}:{}".format(application_name_1, relation_1)
1518         r2 = "{}:{}".format(application_name_2, relation_2)
1519
1520         self.log.debug("adding relation: {} -> {}".format(r1, r2))
1521         try:
1522             await model.add_relation(relation1=r1, relation2=r2)
1523         except JujuAPIError as e:
1524             # If one of the applications in the relationship doesn't exist, or the
1525             # relation has already been added,
1526             # let the operation fail silently.
1527             if "not found" in e.message:
1528                 return
1529             if "already exists" in e.message:
1530                 return
1531             # another execption, raise it
1532             raise e
1533
1534     async def _juju_destroy_application(self, model_name: str, application_name: str):
1535
1536         self.log.debug(
1537             "Destroying application {} in model {}".format(application_name, model_name)
1538         )
1539
1540         # get juju model and observer
1541         model = await self._juju_get_model(model_name=model_name)
1542         observer = self.juju_observers[model_name]
1543
1544         application = model.applications.get(application_name)
1545         if application:
1546             observer.unregister_application(application_name)
1547             await application.destroy()
1548         else:
1549             self.log.debug("Application not found: {}".format(application_name))
1550
1551     async def _juju_destroy_machine(
1552         self, model_name: str, machine_id: str, total_timeout: float = None
1553     ):
1554
1555         self.log.debug(
1556             "Destroying machine {} in model {}".format(machine_id, model_name)
1557         )
1558
1559         if total_timeout is None:
1560             total_timeout = 3600
1561
1562         # get juju model and observer
1563         model = await self._juju_get_model(model_name=model_name)
1564         observer = self.juju_observers[model_name]
1565
1566         machines = await model.get_machines()
1567         if machine_id in machines:
1568             machine = model.machines[machine_id]
1569             observer.unregister_machine(machine_id)
1570             # TODO: change this by machine.is_manual when this is upstreamed:
1571             # https://github.com/juju/python-libjuju/pull/396
1572             if "instance-id" in machine.safe_data and machine.safe_data[
1573                 "instance-id"
1574             ].startswith("manual:"):
1575                 self.log.debug("machine.destroy(force=True) started.")
1576                 await machine.destroy(force=True)
1577                 self.log.debug("machine.destroy(force=True) passed.")
1578                 # max timeout
1579                 end = time.time() + total_timeout
1580                 # wait for machine removal
1581                 machines = await model.get_machines()
1582                 while machine_id in machines and time.time() < end:
1583                     self.log.debug(
1584                         "Waiting for machine {} is destroyed".format(machine_id)
1585                     )
1586                     await asyncio.sleep(0.5)
1587                     machines = await model.get_machines()
1588                 self.log.debug("Machine destroyed: {}".format(machine_id))
1589         else:
1590             self.log.debug("Machine not found: {}".format(machine_id))
1591
1592     async def _juju_destroy_model(self, model_name: str, total_timeout: float = None):
1593
1594         self.log.debug("Destroying model {}".format(model_name))
1595
1596         if total_timeout is None:
1597             total_timeout = 3600
1598         end = time.time() + total_timeout
1599
1600         model = await self._juju_get_model(model_name=model_name)
1601
1602         if not model:
1603             raise N2VCNotFound(message="Model {} does not exist".format(model_name))
1604
1605         uuid = model.info.uuid
1606
1607         # destroy applications
1608         for application_name in model.applications:
1609             try:
1610                 await self._juju_destroy_application(
1611                     model_name=model_name, application_name=application_name
1612                 )
1613             except Exception as e:
1614                 self.log.error(
1615                     "Error destroying application {} in model {}: {}".format(
1616                         application_name, model_name, e
1617                     )
1618                 )
1619
1620         # destroy machines
1621         machines = await model.get_machines()
1622         for machine_id in machines:
1623             try:
1624                 await self._juju_destroy_machine(
1625                     model_name=model_name, machine_id=machine_id
1626                 )
1627             except asyncio.CancelledError:
1628                 raise
1629             except Exception:
1630                 # ignore exceptions destroying machine
1631                 pass
1632
1633         await self._juju_disconnect_model(model_name=model_name)
1634
1635         self.log.debug("destroying model {}...".format(model_name))
1636         await self.controller.destroy_model(uuid)
1637         # self.log.debug('model destroy requested {}'.format(model_name))
1638
1639         # wait for model is completely destroyed
1640         self.log.debug("Waiting for model {} to be destroyed...".format(model_name))
1641         last_exception = ""
1642         while time.time() < end:
1643             try:
1644                 # await self.controller.get_model(uuid)
1645                 models = await self.controller.list_models()
1646                 if model_name not in models:
1647                     self.log.debug(
1648                         "The model {} ({}) was destroyed".format(model_name, uuid)
1649                     )
1650                     return
1651             except asyncio.CancelledError:
1652                 raise
1653             except Exception as e:
1654                 last_exception = e
1655             await asyncio.sleep(5)
1656         raise N2VCException(
1657             "Timeout waiting for model {} to be destroyed {}".format(
1658                 model_name, last_exception
1659             )
1660         )
1661
1662     async def _juju_login(self):
1663         """Connect to juju controller
1664
1665         """
1666
1667         # if already authenticated, exit function
1668         if self._authenticated:
1669             return
1670
1671         # if connecting, wait for finish
1672         # another task could be trying to connect in parallel
1673         while self._connecting:
1674             await asyncio.sleep(0.1)
1675
1676         # double check after other task has finished
1677         if self._authenticated:
1678             return
1679
1680         try:
1681             self._connecting = True
1682             self.log.info(
1683                 "connecting to juju controller: {} {}:{}{}".format(
1684                     self.url,
1685                     self.username,
1686                     self.secret[:8] + "...",
1687                     " with ca_cert" if self.ca_cert else "",
1688                 )
1689             )
1690
1691             # Create controller object
1692             self.controller = Controller(loop=self.loop)
1693             # Connect to controller
1694             await self.controller.connect(
1695                 endpoint=self.url,
1696                 username=self.username,
1697                 password=self.secret,
1698                 cacert=self.ca_cert,
1699             )
1700             self._authenticated = True
1701             self.log.info("juju controller connected")
1702         except Exception as e:
1703             message = "Exception connecting to juju: {}".format(e)
1704             self.log.error(message)
1705             raise N2VCConnectionException(message=message, url=self.url)
1706         finally:
1707             self._connecting = False
1708
1709     async def _juju_logout(self):
1710         """Logout of the Juju controller."""
1711         if not self._authenticated:
1712             return False
1713
1714         # disconnect all models
1715         for model_name in self.juju_models:
1716             try:
1717                 await self._juju_disconnect_model(model_name)
1718             except Exception as e:
1719                 self.log.error(
1720                     "Error disconnecting model {} : {}".format(model_name, e)
1721                 )
1722                 # continue with next model...
1723
1724         self.log.info("Disconnecting controller")
1725         try:
1726             await self.controller.disconnect()
1727         except Exception as e:
1728             raise N2VCConnectionException(
1729                 message="Error disconnecting controller: {}".format(e), url=self.url
1730             )
1731
1732         self.controller = None
1733         self._authenticated = False
1734         self.log.info("disconnected")
1735
1736     async def _juju_disconnect_model(self, model_name: str):
1737         self.log.debug("Disconnecting model {}".format(model_name))
1738         if model_name in self.juju_models:
1739             await self.juju_models[model_name].disconnect()
1740             self.juju_models[model_name] = None
1741             self.juju_observers[model_name] = None
1742         else:
1743             self.warning("Cannot disconnect model: {}".format(model_name))
1744
1745     def _create_juju_public_key(self):
1746         """Recreate the Juju public key on lcm container, if needed
1747         Certain libjuju commands expect to be run from the same machine as Juju
1748          is bootstrapped to. This method will write the public key to disk in
1749          that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1750         """
1751
1752         # Make sure that we have a public key before writing to disk
1753         if self.public_key is None or len(self.public_key) == 0:
1754             if "OSMLCM_VCA_PUBKEY" in os.environ:
1755                 self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "")
1756                 if len(self.public_key) == 0:
1757                     return
1758             else:
1759                 return
1760
1761         pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~"))
1762         file_path = "{}/juju_id_rsa.pub".format(pk_path)
1763         self.log.debug(
1764             "writing juju public key to file:\n{}\npublic key: {}".format(
1765                 file_path, self.public_key
1766             )
1767         )
1768         if not os.path.exists(pk_path):
1769             # create path and write file
1770             os.makedirs(pk_path)
1771             with open(file_path, "w") as f:
1772                 self.log.debug("Creating juju public key file: {}".format(file_path))
1773                 f.write(self.public_key)
1774         else:
1775             self.log.debug("juju public key file already exists: {}".format(file_path))
1776
1777     @staticmethod
1778     def _format_model_name(name: str) -> str:
1779         """Format the name of the model.
1780
1781         Model names may only contain lowercase letters, digits and hyphens
1782         """
1783
1784         return name.replace("_", "-").replace(" ", "-").lower()
1785
1786     @staticmethod
1787     def _format_app_name(name: str) -> str:
1788         """Format the name of the application (in order to assure valid application name).
1789
1790         Application names have restrictions (run juju deploy --help):
1791             - contains lowercase letters 'a'-'z'
1792             - contains numbers '0'-'9'
1793             - contains hyphens '-'
1794             - starts with a lowercase letter
1795             - not two or more consecutive hyphens
1796             - after a hyphen, not a group with all numbers
1797         """
1798
1799         def all_numbers(s: str) -> bool:
1800             for c in s:
1801                 if not c.isdigit():
1802                     return False
1803             return True
1804
1805         new_name = name.replace("_", "-")
1806         new_name = new_name.replace(" ", "-")
1807         new_name = new_name.lower()
1808         while new_name.find("--") >= 0:
1809             new_name = new_name.replace("--", "-")
1810         groups = new_name.split("-")
1811
1812         # find 'all numbers' groups and prefix them with a letter
1813         app_name = ""
1814         for i in range(len(groups)):
1815             group = groups[i]
1816             if all_numbers(group):
1817                 group = "z" + group
1818             if i > 0:
1819                 app_name += "-"
1820             app_name += group
1821
1822         if app_name[0].isdigit():
1823             app_name = "z" + app_name
1824
1825         return app_name