X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=n2vc%2Fn2vc_juju_conn.py;h=55220d64ab389f178fd4ac13dea63a5809f88b49;hb=HEAD;hp=7714d4e9a2c4ff0b68d32197e6a0d9d0bc4de82d;hpb=d1d5541e1fd65d04675edae6e82f3d8a9035592c;p=osm%2FN2VC.git diff --git a/n2vc/n2vc_juju_conn.py b/n2vc/n2vc_juju_conn.py index 7714d4e..f28a9bd 100644 --- a/n2vc/n2vc_juju_conn.py +++ b/n2vc/n2vc_juju_conn.py @@ -22,59 +22,58 @@ import asyncio import logging -import os +from n2vc.config import EnvironConfig +from n2vc.definitions import RelationEndpoint from n2vc.exceptions import ( N2VCBadArgumentsException, N2VCException, N2VCConnectionException, N2VCExecutionException, + N2VCApplicationExists, + JujuApplicationExists, # N2VCNotFound, MethodNotImplemented, - JujuK8sProxycharmNotSupported, ) from n2vc.n2vc_conn import N2VCConnector from n2vc.n2vc_conn import obj_to_dict, obj_to_yaml -from n2vc.libjuju import Libjuju -from n2vc.utils import base64_to_cacert +from n2vc.libjuju import Libjuju, retry_callback +from n2vc.store import MotorStore +from n2vc.utils import get_ee_id_components, generate_random_alfanum_string +from n2vc.vca.connection import get_connection +from retrying_async import retry +from typing import Tuple class N2VCJujuConnector(N2VCConnector): """ -#################################################################################### -################################### P U B L I C #################################### -#################################################################################### + #################################################################################### + ################################### P U B L I C #################################### + #################################################################################### """ BUILT_IN_CLOUDS = ["localhost", "microk8s"] + libjuju = None def __init__( self, db: object, fs: object, log: object = None, - loop: object = None, - url: str = "127.0.0.1:17070", - username: str = "admin", - vca_config: dict = None, on_update_db=None, ): - """Initialize juju N2VC connector + """ + Constructor + + :param: db: Database object from osm_common + :param: fs: Filesystem object from osm_common + :param: log: Logger + :param: on_update_db: Callback function to be called for updating the database. """ # parent class constructor - N2VCConnector.__init__( - self, - db=db, - fs=fs, - log=log, - loop=loop, - url=url, - username=username, - vca_config=vca_config, - on_update_db=on_update_db, - ) + N2VCConnector.__init__(self, db=db, fs=fs, log=log, on_update_db=on_update_db) # silence websocket traffic log logging.getLogger("websockets.protocol").setLevel(logging.INFO) @@ -83,122 +82,29 @@ class N2VCJujuConnector(N2VCConnector): self.log.info("Initializing N2VC juju connector...") - """ - ############################################################## - # check arguments - ############################################################## - """ - - # juju URL - if url is None: - raise N2VCBadArgumentsException("Argument url is mandatory", ["url"]) - url_parts = url.split(":") - if len(url_parts) != 2: - raise N2VCBadArgumentsException( - "Argument url: bad format (localhost:port) -> {}".format(url), ["url"] - ) - self.hostname = url_parts[0] - try: - self.port = int(url_parts[1]) - except ValueError: - raise N2VCBadArgumentsException( - "url port must be a number -> {}".format(url), ["url"] - ) - - # juju USERNAME - if username is None: - raise N2VCBadArgumentsException( - "Argument username is mandatory", ["username"] - ) - - # juju CONFIGURATION - if vca_config is None: - raise N2VCBadArgumentsException( - "Argument vca_config is mandatory", ["vca_config"] - ) - - if "secret" in vca_config: - self.secret = vca_config["secret"] - else: - raise N2VCBadArgumentsException( - "Argument vca_config.secret is mandatory", ["vca_config.secret"] - ) - - # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub - # if exists, it will be written in lcm container: _create_juju_public_key() - if "public_key" in vca_config: - self.public_key = vca_config["public_key"] - else: - self.public_key = None - - # TODO: Verify ca_cert is valid before using. VCA will crash - # if the ca_cert isn't formatted correctly. - - self.ca_cert = vca_config.get("ca_cert") - if self.ca_cert: - self.ca_cert = base64_to_cacert(vca_config["ca_cert"]) - - if "api_proxy" in vca_config and vca_config["api_proxy"] != "": - self.api_proxy = vca_config["api_proxy"] - self.log.debug( - "api_proxy for native charms configured: {}".format(self.api_proxy) - ) - else: - self.warning( - "api_proxy is not configured" - ) - self.api_proxy = None - - if "enable_os_upgrade" in vca_config: - self.enable_os_upgrade = vca_config["enable_os_upgrade"] - else: - self.enable_os_upgrade = True - - if "apt_mirror" in vca_config: - self.apt_mirror = vca_config["apt_mirror"] - else: - self.apt_mirror = None - - self.cloud = vca_config.get('cloud') - self.k8s_cloud = None - if "k8s_cloud" in vca_config: - self.k8s_cloud = vca_config.get("k8s_cloud") - self.log.debug('Arguments have been checked') - - # juju data - self.controller = None # it will be filled when connect to juju - self.juju_models = {} # model objects for every model_name - self.juju_observers = {} # model observers for every model_name - self._connecting = ( - False # while connecting to juju (to avoid duplicate connections) - ) - self._authenticated = ( - False # it will be True when juju connection be stablished - ) - self._creating_model = False # True during model creation - self.libjuju = Libjuju( - endpoint=self.url, - api_proxy=self.api_proxy, - enable_os_upgrade=self.enable_os_upgrade, - apt_mirror=self.apt_mirror, - username=self.username, - password=self.secret, - cacert=self.ca_cert, - loop=self.loop, - log=self.log, - db=self.db, - n2vc=self, - ) - - # create juju pub key file in lcm container at - # ./local/share/juju/ssh/juju_id_rsa.pub - self._create_juju_public_key() - + db_uri = EnvironConfig(prefixes=["OSMLCM_", "OSMMON_"]).get("database_uri") + self._store = MotorStore(db_uri) + self.loading_libjuju = asyncio.Lock() + self.delete_namespace_locks = {} self.log.info("N2VC juju connector initialized") - async def get_status(self, namespace: str, yaml_format: bool = True): + async def get_status( + self, namespace: str, yaml_format: bool = True, vca_id: str = None + ): + """ + Get status from all juju models from a VCA + + :param namespace: we obtain ns from namespace + :param yaml_format: returns a yaml string + :param: vca_id: VCA ID from which the status will be retrieved. + """ + # TODO: Review where is this function used. It is not optimal at all to get the status + # from all the juju models of a particular VCA. Additionally, these models might + # not have been deployed by OSM, in that case we are getting information from + # deployments outside of OSM's scope. # self.log.info('Getting NS status. namespace: {}'.format(namespace)) + libjuju = await self._get_libjuju(vca_id) _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components( namespace=namespace @@ -211,16 +117,44 @@ class N2VCJujuConnector(N2VCConnector): raise N2VCBadArgumentsException(msg, ["namespace"]) status = {} - models = await self.libjuju.list_models(contains=ns_id) + models = await libjuju.list_models(contains=ns_id) for m in models: - status[m] = await self.libjuju.get_model_status(m) + status[m] = await libjuju.get_model_status(m) if yaml_format: return obj_to_yaml(status) else: return obj_to_dict(status) + async def update_vca_status(self, vcastatus: dict, vca_id: str = None): + """ + Add all configs, actions, executed actions of all applications in a model to vcastatus dict. + + :param vcastatus: dict containing vcaStatus + :param: vca_id: VCA ID + + :return: None + """ + try: + libjuju = await self._get_libjuju(vca_id) + for model_name in vcastatus: + # Adding executed actions + vcastatus[model_name][ + "executedActions" + ] = await libjuju.get_executed_actions(model_name) + for application in vcastatus[model_name]["applications"]: + # Adding application actions + vcastatus[model_name]["applications"][application][ + "actions" + ] = await libjuju.get_actions(application, model_name) + # Adding application configs + vcastatus[model_name]["applications"][application][ + "configs" + ] = await libjuju.get_application_configs(model_name, application) + except Exception as e: + self.log.debug("Error in updating vca status: {}".format(str(e))) + async def create_execution_environment( self, namespace: str, @@ -228,15 +162,34 @@ class N2VCJujuConnector(N2VCConnector): reuse_ee_id: str = None, progress_timeout: float = None, total_timeout: float = None, - cloud_name: str = None, - credential_name: str = None, + vca_id: str = None, ) -> (str, dict): + """ + Create an Execution Environment. Returns when it is created or raises an + exception on failing + + :param: namespace: Contains a dot separate string. + LCM will use: []...[-] + :param: db_dict: where to write to database when the status changes. + It contains a dictionary with {collection: str, filter: {}, path: str}, + e.g. {collection: "nsrs", filter: {_id: , path: + "_admin.deployed.VCA.3"} + :param: reuse_ee_id: ee id from an older execution. It allows us to reuse an + older environment + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout + :param: vca_id: VCA ID + + :returns: id of the new execution environment and credentials for it + (credentials can contains hostname, username, etc depending on underlying cloud) + """ self.log.info( "Creating execution environment. namespace: {}, reuse_ee_id: {}".format( namespace, reuse_ee_id ) ) + libjuju = await self._get_libjuju(vca_id) machine_id = None if reuse_ee_id: @@ -264,15 +217,9 @@ class N2VCJujuConnector(N2VCConnector): # create or reuse a new juju machine try: - if not await self.libjuju.model_exists(model_name): - cloud = cloud_name or self.cloud - credential = credential_name or cloud_name if cloud_name else self.cloud - await self.libjuju.add_model( - model_name, - cloud_name=cloud, - credential_name=credential - ) - machine, new = await self.libjuju.create_machine( + if not await libjuju.model_exists(model_name): + await libjuju.add_model(model_name, libjuju.vca_connection.lxd_cloud) + machine, new = await libjuju.create_machine( model_name=model_name, machine_id=machine_id, db_dict=db_dict, @@ -297,9 +244,7 @@ class N2VCJujuConnector(N2VCConnector): raise N2VCException(message=message) # new machine credentials - credentials = { - "hostname": machine.dns_name, - } + credentials = {"hostname": machine.dns_name} self.log.info( "Execution environment created. ee_id: {}, credentials: {}".format( @@ -316,15 +261,34 @@ class N2VCJujuConnector(N2VCConnector): db_dict: dict, progress_timeout: float = None, total_timeout: float = None, - cloud_name: str = None, - credential_name: str = None, + vca_id: str = None, ) -> str: - + """ + Register an existing execution environment at the VCA + + :param: namespace: Contains a dot separate string. + LCM will use: []...[-] + :param: credentials: credentials to access the existing execution environment + (it can contains hostname, username, path to private key, + etc depending on underlying cloud) + :param: db_dict: where to write to database when the status changes. + It contains a dictionary with {collection: str, filter: {}, path: str}, + e.g. {collection: "nsrs", filter: {_id: , path: + "_admin.deployed.VCA.3"} + :param: reuse_ee_id: ee id from an older execution. It allows us to reuse an + older environment + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout + :param: vca_id: VCA ID + + :returns: id of the execution environment + """ self.log.info( "Registering execution environment. namespace={}, credentials={}".format( namespace, credentials ) ) + libjuju = await self._get_libjuju(vca_id) if credentials is None: raise N2VCBadArgumentsException( @@ -359,15 +323,9 @@ class N2VCJujuConnector(N2VCConnector): # register machine on juju try: - if not await self.libjuju.model_exists(model_name): - cloud = cloud_name or self.cloud - credential = credential_name or cloud_name if cloud_name else self.cloud - await self.libjuju.add_model( - model_name, - cloud_name=cloud, - credential_name=credential - ) - machine_id = await self.libjuju.provision_machine( + if not await libjuju.model_exists(model_name): + await libjuju.add_model(model_name, libjuju.vca_connection.lxd_cloud) + machine_id = await libjuju.provision_machine( model_name=model_name, hostname=hostname, username=username, @@ -395,6 +353,15 @@ class N2VCJujuConnector(N2VCConnector): return ee_id + # In case of native_charm is being deployed, if JujuApplicationExists error happens + # it will try to add_unit + @retry( + attempts=3, + delay=5, + retry_exceptions=(N2VCApplicationExists,), + timeout=None, + callback=retry_callback, + ) async def install_configuration_sw( self, ee_id: str, @@ -404,7 +371,33 @@ class N2VCJujuConnector(N2VCConnector): total_timeout: float = None, config: dict = None, num_units: int = 1, + vca_id: str = None, + scaling_out: bool = False, + vca_type: str = None, ): + """ + Install the software inside the execution environment identified by ee_id + + :param: ee_id: the id of the execution environment returned by + create_execution_environment or register_execution_environment + :param: artifact_path: where to locate the artifacts (parent folder) using + the self.fs + the final artifact path will be a combination of this + artifact_path and additional string from the config_dict + (e.g. charm name) + :param: db_dict: where to write into database when the status changes. + It contains a dict with + {collection: , filter: {}, path: }, + e.g. {collection: "nsrs", filter: + {_id: , path: "_admin.deployed.VCA.3"} + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout + :param: config: Dictionary with deployment config information. + :param: num_units: Number of units to deploy of a particular charm. + :param: vca_id: VCA ID + :param: scaling_out: Boolean to indicate if it is a scaling out operation + :param: vca_type: VCA type + """ self.log.info( ( @@ -412,6 +405,7 @@ class N2VCJujuConnector(N2VCConnector): "artifact path: {}, db_dict: {}" ).format(ee_id, artifact_path, db_dict) ) + libjuju = await self._get_libjuju(vca_id) # check arguments if ee_id is None or len(ee_id) == 0: @@ -451,7 +445,7 @@ class N2VCJujuConnector(N2VCConnector): artifact_path = artifact_path.replace("//", "/") # check charm path - if not self.fs.file_exists(artifact_path, mode="dir"): + if not self.fs.file_exists(artifact_path): msg = "artifact path does not exist: {}".format(artifact_path) raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"]) @@ -461,20 +455,36 @@ class N2VCJujuConnector(N2VCConnector): full_path = self.fs.path + "/" + artifact_path try: - await self.libjuju.deploy_charm( - model_name=model_name, - application_name=application_name, - path=full_path, - machine_id=machine_id, - db_dict=db_dict, - progress_timeout=progress_timeout, - total_timeout=total_timeout, - config=config, - num_units=num_units, + if vca_type == "native_charm" and await libjuju.check_application_exists( + model_name, application_name + ): + await libjuju.add_unit( + application_name=application_name, + model_name=model_name, + machine_id=machine_id, + db_dict=db_dict, + progress_timeout=progress_timeout, + total_timeout=total_timeout, + ) + else: + await libjuju.deploy_charm( + model_name=model_name, + application_name=application_name, + path=full_path, + machine_id=machine_id, + db_dict=db_dict, + progress_timeout=progress_timeout, + total_timeout=total_timeout, + config=config, + num_units=num_units, + ) + except JujuApplicationExists as e: + raise N2VCApplicationExists( + message="Error deploying charm into ee={} : {}".format(ee_id, e.message) ) except Exception as e: raise N2VCException( - message="Error desploying charm into ee={} : {}".format(ee_id, e) + message="Error deploying charm into ee={} : {}".format(ee_id, e) ) self.log.info("Configuration sw installed") @@ -488,8 +498,7 @@ class N2VCJujuConnector(N2VCConnector): progress_timeout: float = None, total_timeout: float = None, config: dict = None, - cloud_name: str = None, - credential_name: str = None, + vca_id: str = None, ) -> str: """ Install a k8s proxy charm @@ -505,56 +514,51 @@ class N2VCJujuConnector(N2VCConnector): {collection: , filter: {}, path: }, e.g. {collection: "nsrs", filter: {_id: , path: "_admin.deployed.VCA.3"} - :param float progress_timeout: - :param float total_timeout: + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout :param config: Dictionary with additional configuration - :param cloud_name: Cloud Name in which the charms will be deployed - :param credential_name: Credential Name to use in the cloud_name. - If not set, cloud_name will be used as credential_name + :param vca_id: VCA ID :returns ee_id: execution environment id. """ - self.log.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}' - .format(charm_name, artifact_path, db_dict)) - - if not self.k8s_cloud: - raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available") + self.log.info( + "Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}".format( + charm_name, artifact_path, db_dict + ) + ) + libjuju = await self._get_libjuju(vca_id) if artifact_path is None or len(artifact_path) == 0: raise N2VCBadArgumentsException( message="artifact_path is mandatory", bad_args=["artifact_path"] ) if db_dict is None: - raise N2VCBadArgumentsException(message='db_dict is mandatory', bad_args=['db_dict']) + raise N2VCBadArgumentsException( + message="db_dict is mandatory", bad_args=["db_dict"] + ) # remove // in charm path - while artifact_path.find('//') >= 0: - artifact_path = artifact_path.replace('//', '/') + while artifact_path.find("//") >= 0: + artifact_path = artifact_path.replace("//", "/") # check charm path - if not self.fs.file_exists(artifact_path, mode="dir"): - msg = 'artifact path does not exist: {}'.format(artifact_path) - raise N2VCBadArgumentsException(message=msg, bad_args=['artifact_path']) + if not self.fs.file_exists(artifact_path): + msg = "artifact path does not exist: {}".format(artifact_path) + raise N2VCBadArgumentsException(message=msg, bad_args=["artifact_path"]) - if artifact_path.startswith('/'): + if artifact_path.startswith("/"): full_path = self.fs.path + artifact_path else: - full_path = self.fs.path + '/' + artifact_path + full_path = self.fs.path + "/" + artifact_path _, ns_id, _, _, _ = self._get_namespace_components(namespace=namespace) - model_name = '{}-k8s'.format(ns_id) - if not await self.libjuju.model_exists(model_name): - cloud = cloud_name or self.k8s_cloud - credential = credential_name or cloud_name if cloud_name else self.k8s_cloud - await self.libjuju.add_model( - model_name, - cloud_name=cloud, - credential_name=credential - ) + model_name = "{}-k8s".format(ns_id) + if not await libjuju.model_exists(model_name): + await libjuju.add_model(model_name, libjuju.vca_connection.k8s_cloud) application_name = self._get_application_name(namespace) try: - await self.libjuju.deploy_charm( + await libjuju.deploy_charm( model_name=model_name, application_name=application_name, path=full_path, @@ -562,16 +566,14 @@ class N2VCJujuConnector(N2VCConnector): db_dict=db_dict, progress_timeout=progress_timeout, total_timeout=total_timeout, - config=config + config=config, ) except Exception as e: - raise N2VCException(message='Error deploying charm: {}'.format(e)) + raise N2VCException(message="Error deploying charm: {}".format(e)) - self.log.info('K8s proxy charm installed') + self.log.info("K8s proxy charm installed") ee_id = N2VCJujuConnector._build_ee_id( - model_name=model_name, - application_name=application_name, - machine_id="k8s", + model_name=model_name, application_name=application_name, machine_id="k8s" ) self._write_ee_id_db(db_dict=db_dict, ee_id=ee_id) @@ -584,13 +586,33 @@ class N2VCJujuConnector(N2VCConnector): db_dict: dict, progress_timeout: float = None, total_timeout: float = None, + vca_id: str = None, ) -> str: + """ + Get Execution environment ssh public key + + :param: ee_id: the id of the execution environment returned by + create_execution_environment or register_execution_environment + :param: db_dict: where to write into database when the status changes. + It contains a dict with + {collection: , filter: {}, path: }, + e.g. {collection: "nsrs", filter: + {_id: , path: "_admin.deployed.VCA.3"} + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout + :param vca_id: VCA ID + :returns: public key of the execution environment + For the case of juju proxy charm ssh-layered, it is the one + returned by 'get-ssh-public-key' primitive. + It raises a N2VC exception if fails + """ self.log.info( ( "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}" ).format(ee_id, db_dict) ) + libjuju = await self._get_libjuju(vca_id) # check arguments if ee_id is None or len(ee_id) == 0: @@ -631,7 +653,7 @@ class N2VCJujuConnector(N2VCConnector): # execute action: generate-ssh-key try: - output, _status = await self.libjuju.execute_action( + output, _status = await libjuju.execute_action( model_name=model_name, application_name=application_name, action_name="generate-ssh-key", @@ -648,7 +670,7 @@ class N2VCJujuConnector(N2VCConnector): # execute action: get-ssh-public-key try: - output, _status = await self.libjuju.execute_action( + output, _status = await libjuju.execute_action( model_name=model_name, application_name=application_name, action_name="get-ssh-public-key", @@ -664,60 +686,61 @@ class N2VCJujuConnector(N2VCConnector): # return public key if exists return output["pubkey"] if "pubkey" in output else output - async def get_metrics(self, model_name: str, application_name: str) -> dict: - return await self.libjuju.get_metrics(model_name, application_name) + async def get_metrics( + self, model_name: str, application_name: str, vca_id: str = None + ) -> dict: + """ + Get metrics from application + + :param: model_name: Model name + :param: application_name: Application name + :param: vca_id: VCA ID + + :return: Dictionary with obtained metrics + """ + libjuju = await self._get_libjuju(vca_id) + return await libjuju.get_metrics(model_name, application_name) async def add_relation( - self, ee_id_1: str, ee_id_2: str, endpoint_1: str, endpoint_2: str + self, provider: RelationEndpoint, requirer: RelationEndpoint ): + """ + Add relation between two charmed endpoints - self.log.debug( - "adding new relation between {} and {}, endpoints: {}, {}".format( - ee_id_1, ee_id_2, endpoint_1, endpoint_2 - ) + :param: provider: Provider relation endpoint + :param: requirer: Requirer relation endpoint + """ + self.log.debug(f"adding new relation between {provider} and {requirer}") + cross_model_relation = ( + provider.model_name != requirer.model_name + or provider.vca_id != requirer.vca_id ) - - # check arguments - if not ee_id_1: - message = "EE 1 is mandatory" - self.log.error(message) - raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_1"]) - if not ee_id_2: - message = "EE 2 is mandatory" - self.log.error(message) - raise N2VCBadArgumentsException(message=message, bad_args=["ee_id_2"]) - if not endpoint_1: - message = "endpoint 1 is mandatory" - self.log.error(message) - raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_1"]) - if not endpoint_2: - message = "endpoint 2 is mandatory" - self.log.error(message) - raise N2VCBadArgumentsException(message=message, bad_args=["endpoint_2"]) - - # get the model, the applications and the machines from the ee_id's - model_1, app_1, _machine_1 = self._get_ee_id_components(ee_id_1) - model_2, app_2, _machine_2 = self._get_ee_id_components(ee_id_2) - - # model must be the same - if model_1 != model_2: - message = "EE models are not the same: {} vs {}".format(ee_id_1, ee_id_2) - self.log.error(message) - raise N2VCBadArgumentsException( - message=message, bad_args=["ee_id_1", "ee_id_2"] - ) - - # add juju relations between two applications try: - await self.libjuju.add_relation( - model_name=model_1, - endpoint_1="{}:{}".format(app_1, endpoint_1), - endpoint_2="{}:{}".format(app_2, endpoint_2), - ) + if cross_model_relation: + # Cross-model relation + provider_libjuju = await self._get_libjuju(provider.vca_id) + requirer_libjuju = await self._get_libjuju(requirer.vca_id) + offer = await provider_libjuju.offer(provider) + if offer: + saas_name = await requirer_libjuju.consume( + requirer.model_name, offer, provider_libjuju + ) + await requirer_libjuju.add_relation( + requirer.model_name, requirer.endpoint, saas_name + ) + else: + # Standard relation + vca_id = provider.vca_id + model = provider.model_name + libjuju = await self._get_libjuju(vca_id) + # add juju relations between two applications + await libjuju.add_relation( + model_name=model, + endpoint_1=provider.endpoint, + endpoint_2=requirer.endpoint, + ) except Exception as e: - message = "Error adding relation between {} and {}: {}".format( - ee_id_1, ee_id_2, e - ) + message = f"Error adding relation between {provider} and {requirer}: {e}" self.log.error(message) raise N2VCException(message=message) @@ -731,42 +754,106 @@ class N2VCJujuConnector(N2VCConnector): raise MethodNotImplemented() async def delete_namespace( - self, namespace: str, db_dict: dict = None, total_timeout: float = None + self, + namespace: str, + db_dict: dict = None, + total_timeout: float = None, + vca_id: str = None, ): + """ + Remove a network scenario and its execution environments + :param: namespace: []. + :param: db_dict: where to write into database when the status changes. + It contains a dict with + {collection: , filter: {}, path: }, + e.g. {collection: "nsrs", filter: + {_id: , path: "_admin.deployed.VCA.3"} + :param: total_timeout: Total timeout + :param: vca_id: VCA ID + """ self.log.info("Deleting namespace={}".format(namespace)) + will_not_delete = False + if namespace not in self.delete_namespace_locks: + self.delete_namespace_locks[namespace] = asyncio.Lock() + delete_lock = self.delete_namespace_locks[namespace] - # check arguments - if namespace is None: - raise N2VCBadArgumentsException( - message="namespace is mandatory", bad_args=["namespace"] - ) + while delete_lock.locked(): + will_not_delete = True + await asyncio.sleep(0.1) - _nsi_id, ns_id, _vnf_id, _vdu_id, _vdu_count = self._get_namespace_components( - namespace=namespace - ) - if ns_id is not None: - try: - models = await self.libjuju.list_models(contains=ns_id) - for model in models: - await self.libjuju.destroy_model( - model_name=model, total_timeout=total_timeout + if will_not_delete: + self.log.info("Namespace {} deleted by another worker.".format(namespace)) + return + + try: + async with delete_lock: + libjuju = await self._get_libjuju(vca_id) + + # check arguments + if namespace is None: + raise N2VCBadArgumentsException( + message="namespace is mandatory", bad_args=["namespace"] ) - except Exception as e: - raise N2VCException( - message="Error deleting namespace {} : {}".format(namespace, e) - ) - else: - raise N2VCBadArgumentsException( - message="only ns_id is permitted to delete yet", bad_args=["namespace"] - ) + ( + _nsi_id, + ns_id, + _vnf_id, + _vdu_id, + _vdu_count, + ) = self._get_namespace_components(namespace=namespace) + if ns_id is not None: + try: + models = await libjuju.list_models(contains=ns_id) + for model in models: + await libjuju.destroy_model( + model_name=model, total_timeout=total_timeout + ) + except Exception as e: + self.log.error(f"Error deleting namespace {namespace} : {e}") + raise N2VCException( + message="Error deleting namespace {} : {}".format( + namespace, e + ) + ) + else: + raise N2VCBadArgumentsException( + message="only ns_id is permitted to delete yet", + bad_args=["namespace"], + ) + except Exception as e: + self.log.error(f"Error deleting namespace {namespace} : {e}") + raise e + finally: + self.delete_namespace_locks.pop(namespace) self.log.info("Namespace {} deleted".format(namespace)) async def delete_execution_environment( - self, ee_id: str, db_dict: dict = None, total_timeout: float = None, - scaling_in: bool = False + self, + ee_id: str, + db_dict: dict = None, + total_timeout: float = None, + scaling_in: bool = False, + vca_type: str = None, + vca_id: str = None, + application_to_delete: str = None, ): + """ + Delete an execution environment + :param str ee_id: id of the execution environment to delete + :param dict db_dict: where to write into database when the status changes. + It contains a dict with + {collection: , filter: {}, path: }, + e.g. {collection: "nsrs", filter: + {_id: , path: "_admin.deployed.VCA.3"} + :param total_timeout: Total timeout + :param scaling_in: Boolean to indicate if it is a scaling in operation + :param vca_type: VCA type + :param vca_id: VCA ID + :param application_to_delete: name of the single application to be deleted + """ self.log.info("Deleting execution environment ee_id={}".format(ee_id)) + libjuju = await self._get_libjuju(vca_id) # check arguments if ee_id is None: @@ -774,23 +861,49 @@ class N2VCJujuConnector(N2VCConnector): message="ee_id is mandatory", bad_args=["ee_id"] ) - model_name, application_name, _machine_id = self._get_ee_id_components( + model_name, application_name, machine_id = self._get_ee_id_components( ee_id=ee_id ) try: - if not scaling_in: + if application_to_delete == application_name: + # destroy the application + await libjuju.destroy_application( + model_name=model_name, + application_name=application_name, + total_timeout=total_timeout, + ) + # if model is empty delete it + controller = await libjuju.get_controller() + model = await libjuju.get_model( + controller=controller, + model_name=model_name, + ) + if not model.applications: + self.log.info("Model {} is empty, deleting it".format(model_name)) + await libjuju.destroy_model( + model_name=model_name, + total_timeout=total_timeout, + ) + elif not scaling_in: # destroy the model - # TODO: should this be removed? - await self.libjuju.destroy_model( + await libjuju.destroy_model( model_name=model_name, total_timeout=total_timeout ) + elif vca_type == "native_charm" and scaling_in: + # destroy the unit in the application + await libjuju.destroy_unit( + application_name=application_name, + model_name=model_name, + machine_id=machine_id, + total_timeout=total_timeout, + ) else: - # get juju model and observer - controller = await self.libjuju.get_controller() - model = await self.libjuju.get_model(controller, model_name) # destroy the application - await self.libjuju.destroy_application( - model=model, application_name=application_name) + await libjuju.destroy_application( + model_name=model_name, + application_name=application_name, + total_timeout=total_timeout, + ) except Exception as e: raise N2VCException( message=( @@ -808,13 +921,36 @@ class N2VCJujuConnector(N2VCConnector): db_dict: dict = None, progress_timeout: float = None, total_timeout: float = None, + vca_id: str = None, + vca_type: str = None, ) -> str: + """ + Execute a primitive in the execution environment + + :param: ee_id: the one returned by create_execution_environment or + register_execution_environment + :param: primitive_name: must be one defined in the software. There is one + called 'config', where, for the proxy case, the 'credentials' of VM are + provided + :param: params_dict: parameters of the action + :param: db_dict: where to write into database when the status changes. + It contains a dict with + {collection: , filter: {}, path: }, + e.g. {collection: "nsrs", filter: + {_id: , path: "_admin.deployed.VCA.3"} + :param: progress_timeout: Progress timeout + :param: total_timeout: Total timeout + :param: vca_id: VCA ID + :param: vca_type: VCA type + :returns str: primitive result, if ok. It raises exceptions in case of fail + """ self.log.info( "Executing primitive: {} on ee: {}, params: {}".format( primitive_name, ee_id, params_dict ) ) + libjuju = await self._get_libjuju(vca_id) # check arguments if ee_id is None or len(ee_id) == 0: @@ -832,8 +968,12 @@ class N2VCJujuConnector(N2VCConnector): ( model_name, application_name, - _machine_id, + machine_id, ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id) + # To run action on the leader unit in libjuju.execute_action function, + # machine_id must be set to None if vca_type is not native_charm + if vca_type != "native_charm": + machine_id = None except Exception: raise N2VCBadArgumentsException( message="ee_id={} is not a valid execution environment id".format( @@ -845,13 +985,13 @@ class N2VCJujuConnector(N2VCConnector): if primitive_name == "config": # Special case: config primitive try: - await self.libjuju.configure_application( + await libjuju.configure_application( model_name=model_name, application_name=application_name, config=params_dict, ) - actions = await self.libjuju.get_actions( - application_name=application_name, model_name=model_name, + actions = await libjuju.get_actions( + application_name=application_name, model_name=model_name ) self.log.debug( "Application {} has these actions: {}".format( @@ -865,7 +1005,7 @@ class N2VCJujuConnector(N2VCConnector): for _ in range(num_retries): try: self.log.debug("Executing action verify-ssh-credentials...") - output, ok = await self.libjuju.execute_action( + output, ok = await libjuju.execute_action( model_name=model_name, application_name=application_name, action_name="verify-ssh-credentials", @@ -907,37 +1047,108 @@ class N2VCJujuConnector(N2VCConnector): return "CONFIG OK" else: try: - output, status = await self.libjuju.execute_action( + output, status = await libjuju.execute_action( model_name=model_name, application_name=application_name, action_name=primitive_name, db_dict=db_dict, + machine_id=machine_id, progress_timeout=progress_timeout, total_timeout=total_timeout, - **params_dict + **params_dict, ) if status == "completed": return output else: - raise Exception("status is not completed: {}".format(status)) + if "output" in output: + raise Exception(f'{status}: {output["output"]}') + else: + raise Exception( + f"{status}: No further information received from action" + ) + except Exception as e: - self.log.error( - "Error executing primitive {}: {}".format(primitive_name, e) - ) + self.log.error(f"Error executing primitive {primitive_name}: {e}") raise N2VCExecutionException( - message="Error executing primitive {} into ee={} : {}".format( - primitive_name, ee_id, e - ), + message=f"Error executing primitive {primitive_name} in ee={ee_id}: {e}", primitive_name=primitive_name, ) - async def disconnect(self): + async def upgrade_charm( + self, + ee_id: str = None, + path: str = None, + charm_id: str = None, + charm_type: str = None, + timeout: float = None, + ) -> str: + """This method upgrade charms in VNFs + + Args: + ee_id: Execution environment id + path: Local path to the charm + charm_id: charm-id + charm_type: Charm type can be lxc-proxy-charm, native-charm or k8s-proxy-charm + timeout: (Float) Timeout for the ns update operation + + Returns: + The output of the update operation if status equals to "completed" + + """ + self.log.info("Upgrading charm: {} on ee: {}".format(path, ee_id)) + libjuju = await self._get_libjuju(charm_id) + + # check arguments + if ee_id is None or len(ee_id) == 0: + raise N2VCBadArgumentsException( + message="ee_id is mandatory", bad_args=["ee_id"] + ) + try: + ( + model_name, + application_name, + machine_id, + ) = N2VCJujuConnector._get_ee_id_components(ee_id=ee_id) + + except Exception: + raise N2VCBadArgumentsException( + message="ee_id={} is not a valid execution environment id".format( + ee_id + ), + bad_args=["ee_id"], + ) + + try: + await libjuju.upgrade_charm( + application_name=application_name, + path=path, + model_name=model_name, + total_timeout=timeout, + ) + + return f"Charm upgraded with application name {application_name}" + + except Exception as e: + self.log.error("Error upgrading charm {}: {}".format(path, e)) + + raise N2VCException( + message="Error upgrading charm {} in ee={} : {}".format(path, ee_id, e) + ) + + async def disconnect(self, vca_id: str = None): + """ + Disconnect from VCA + + :param: vca_id: VCA ID + """ self.log.info("closing juju N2VC...") + libjuju = await self._get_libjuju(vca_id) try: - await self.libjuju.disconnect() + await libjuju.disconnect() except Exception as e: raise N2VCConnectionException( - message="Error disconnecting controller: {}".format(e), url=self.url + message="Error disconnecting controller: {}".format(e), + url=libjuju.vca_connection.data.endpoints, ) """ @@ -946,8 +1157,27 @@ class N2VCJujuConnector(N2VCConnector): #################################################################################### """ - def _write_ee_id_db(self, db_dict: dict, ee_id: str): + async def _get_libjuju(self, vca_id: str = None) -> Libjuju: + """ + Get libjuju object + + :param: vca_id: VCA ID + If None, get a libjuju object with a Connection to the default VCA + Else, geta libjuju object with a Connection to the specified VCA + """ + if not vca_id: + while self.loading_libjuju.locked(): + await asyncio.sleep(0.1) + if not self.libjuju: + async with self.loading_libjuju: + vca_connection = await get_connection(self._store) + self.libjuju = Libjuju(vca_connection, log=self.log) + return self.libjuju + else: + vca_connection = await get_connection(self._store, vca_id) + return Libjuju(vca_connection, log=self.log, n2vc=self) + def _write_ee_id_db(self, db_dict: dict, ee_id: str): # write ee_id to database: _admin.deployed.VCA.x try: the_table = db_dict["collection"] @@ -988,30 +1218,43 @@ class N2VCJujuConnector(N2VCConnector): :return: model_name, application_name, machine_id """ - if ee_id is None: - return None, None, None - - # split components of id - parts = ee_id.split(".") - model_name = parts[0] - application_name = parts[1] - machine_id = parts[2] - return model_name, application_name, machine_id + return get_ee_id_components(ee_id) - def _get_application_name(self, namespace: str) -> str: - """ - Build application name from namespace - :param namespace: - :return: app-vnf--vdu--cnt- + @staticmethod + def _find_charm_level(vnf_id: str, vdu_id: str) -> str: + """Decides the charm level. + Args: + vnf_id (str): VNF id + vdu_id (str): VDU id + + Returns: + charm_level (str): ns-level or vnf-level or vdu-level """ + if vdu_id and not vnf_id: + raise N2VCException(message="If vdu-id exists, vnf-id should be provided.") + if vnf_id and vdu_id: + return "vdu-level" + if vnf_id and not vdu_id: + return "vnf-level" + if not vnf_id and not vdu_id: + return "ns-level" - # TODO: Enforce the Juju 50-character application limit + @staticmethod + def _generate_backward_compatible_application_name( + vnf_id: str, vdu_id: str, vdu_count: str + ) -> str: + """Generate backward compatible application name + by limiting the app name to 50 characters. - # split namespace components - _, _, vnf_id, vdu_id, vdu_count = self._get_namespace_components( - namespace=namespace - ) + Args: + vnf_id (str): VNF ID + vdu_id (str): VDU ID + vdu_count (str): vdu-count-index + + Returns: + application_name (str): generated application name + """ if vnf_id is None or len(vnf_id) == 0: vnf_id = "" else: @@ -1029,41 +1272,235 @@ class N2VCJujuConnector(N2VCConnector): else: vdu_count = "-cnt-" + vdu_count - application_name = "app-{}{}{}".format(vnf_id, vdu_id, vdu_count) + # Generate a random suffix with 5 characters (the default size used by K8s) + random_suffix = generate_random_alfanum_string(size=5) - return N2VCJujuConnector._format_app_name(application_name) + application_name = "app-{}{}{}-{}".format( + vnf_id, vdu_id, vdu_count, random_suffix + ) + return application_name + + @staticmethod + def _get_vca_record(search_key: str, vca_records: list, vdu_id: str) -> dict: + """Get the correct VCA record dict depending on the search key + + Args: + search_key (str): keyword to find the correct VCA record + vca_records (list): All VCA records as list + vdu_id (str): VDU ID + + Returns: + vca_record (dict): Dictionary which includes the correct VCA record - def _create_juju_public_key(self): - """Recreate the Juju public key on lcm container, if needed - Certain libjuju commands expect to be run from the same machine as Juju - is bootstrapped to. This method will write the public key to disk in - that location: ~/.local/share/juju/ssh/juju_id_rsa.pub """ + return next( + filter(lambda record: record[search_key] == vdu_id, vca_records), {} + ) - # Make sure that we have a public key before writing to disk - if self.public_key is None or len(self.public_key) == 0: - if "OSMLCM_VCA_PUBKEY" in os.environ: - self.public_key = os.getenv("OSMLCM_VCA_PUBKEY", "") - if len(self.public_key) == 0: - return - else: - return + @staticmethod + def _generate_application_name( + charm_level: str, + vnfrs: dict, + vca_records: list, + vnf_count: str = None, + vdu_id: str = None, + vdu_count: str = None, + ) -> str: + """Generate application name to make the relevant charm of VDU/KDU + in the VNFD descriptor become clearly visible. + Limiting the app name to 50 characters. - pk_path = "{}/.local/share/juju/ssh".format(os.path.expanduser("~")) - file_path = "{}/juju_id_rsa.pub".format(pk_path) - self.log.debug( - "writing juju public key to file:\n{}\npublic key: {}".format( - file_path, self.public_key + Args: + charm_level (str): level of charm + vnfrs (dict): vnf record dict + vca_records (list): db_nsr["_admin"]["deployed"]["VCA"] as list + vnf_count (str): vnf count index + vdu_id (str): VDU ID + vdu_count (str): vdu count index + + Returns: + application_name (str): generated application name + + """ + application_name = "" + if charm_level == "ns-level": + if len(vca_records) != 1: + raise N2VCException(message="One VCA record is expected.") + # Only one VCA record is expected if it's ns-level charm. + # Shorten the charm name to its first 40 characters. + charm_name = vca_records[0]["charm_name"][:40] + if not charm_name: + raise N2VCException(message="Charm name should be provided.") + application_name = charm_name + "-ns" + + elif charm_level == "vnf-level": + if len(vca_records) < 1: + raise N2VCException(message="One or more VCA record is expected.") + # If VNF is scaled, more than one VCA record may be included in vca_records + # but ee_descriptor_id is same. + # Shorten the ee_descriptor_id and member-vnf-index-ref + # to first 12 characters. + application_name = ( + vca_records[0]["ee_descriptor_id"][:12] + + "-" + + vnf_count + + "-" + + vnfrs["member-vnf-index-ref"][:12] + + "-vnf" + ) + elif charm_level == "vdu-level": + if len(vca_records) < 1: + raise N2VCException(message="One or more VCA record is expected.") + + # Charms are also used for deployments with Helm charts. + # If deployment unit is a Helm chart/KDU, + # vdu_profile_id and vdu_count will be empty string. + if vdu_count is None: + vdu_count = "" + + # If vnf/vdu is scaled, more than one VCA record may be included in vca_records + # but ee_descriptor_id is same. + # Shorten the ee_descriptor_id, member-vnf-index-ref and vdu_profile_id + # to first 12 characters. + if not vdu_id: + raise N2VCException(message="vdu-id should be provided.") + + vca_record = N2VCJujuConnector._get_vca_record( + "vdu_id", vca_records, vdu_id + ) + + if not vca_record: + vca_record = N2VCJujuConnector._get_vca_record( + "kdu_name", vca_records, vdu_id + ) + + application_name = ( + vca_record["ee_descriptor_id"][:12] + + "-" + + vnf_count + + "-" + + vnfrs["member-vnf-index-ref"][:12] + + "-" + + vdu_id[:12] + + "-" + + vdu_count + + "-vdu" + ) + + return application_name + + def _get_vnf_count_and_record( + self, charm_level: str, vnf_id_and_count: str + ) -> Tuple[str, dict]: + """Get the vnf count and VNF record depend on charm level + + Args: + charm_level (str) + vnf_id_and_count (str) + + Returns: + (vnf_count (str), db_vnfr(dict)) as Tuple + + """ + vnf_count = "" + db_vnfr = {} + + if charm_level in ("vnf-level", "vdu-level"): + vnf_id = "-".join(vnf_id_and_count.split("-")[:-1]) + vnf_count = vnf_id_and_count.split("-")[-1] + db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_id}) + + # If the charm is ns level, it returns empty vnf_count and db_vnfr + return vnf_count, db_vnfr + + @staticmethod + def _get_vca_records(charm_level: str, db_nsr: dict, db_vnfr: dict) -> list: + """Get the VCA records from db_nsr dict + + Args: + charm_level (str): level of charm + db_nsr (dict): NS record from database + db_vnfr (dict): VNF record from database + + Returns: + vca_records (list): List of VCA record dictionaries + + """ + vca_records = {} + if charm_level == "ns-level": + vca_records = list( + filter( + lambda vca_record: vca_record["target_element"] == "ns", + db_nsr["_admin"]["deployed"]["VCA"], + ) ) + elif charm_level in ["vnf-level", "vdu-level"]: + vca_records = list( + filter( + lambda vca_record: vca_record["member-vnf-index"] + == db_vnfr["member-vnf-index-ref"], + db_nsr["_admin"]["deployed"]["VCA"], + ) + ) + + return vca_records + + def _get_application_name(self, namespace: str) -> str: + """Build application name from namespace + + Application name structure: + NS level: -ns + VNF level: -z--vnf + VDU level: -z-- + -z-vdu + + Application naming for backward compatibility (old structure): + NS level: app- + VNF level: app-vnf--z- + VDU level: app-vnf--z-vdu- + -cnt--z- + + Args: + namespace (str) + + Returns: + application_name (str) + + """ + # split namespace components + ( + nsi_id, + ns_id, + vnf_id_and_count, + vdu_id, + vdu_count, + ) = self._get_namespace_components(namespace=namespace) + + if not ns_id: + raise N2VCException(message="ns-id should be provided.") + + charm_level = self._find_charm_level(vnf_id_and_count, vdu_id) + db_nsr = self.db.get_one("nsrs", {"_id": ns_id}) + vnf_count, db_vnfr = self._get_vnf_count_and_record( + charm_level, vnf_id_and_count ) - if not os.path.exists(pk_path): - # create path and write file - os.makedirs(pk_path) - with open(file_path, "w") as f: - self.log.debug("Creating juju public key file: {}".format(file_path)) - f.write(self.public_key) + vca_records = self._get_vca_records(charm_level, db_nsr, db_vnfr) + + if all("charm_name" in vca_record.keys() for vca_record in vca_records): + application_name = self._generate_application_name( + charm_level, + db_vnfr, + vca_records, + vnf_count=vnf_count, + vdu_id=vdu_id, + vdu_count=vdu_count, + ) else: - self.log.debug("juju public key file already exists: {}".format(file_path)) + application_name = self._generate_backward_compatible_application_name( + vnf_id_and_count, vdu_id, vdu_count + ) + + return N2VCJujuConnector._format_app_name(application_name) @staticmethod def _format_model_name(name: str) -> str: @@ -1114,3 +1551,14 @@ class N2VCJujuConnector(N2VCConnector): app_name = "z" + app_name return app_name + + async def validate_vca(self, vca_id: str): + """ + Validate a VCA by connecting/disconnecting to/from it + + :param: vca_id: VCA ID + """ + vca_connection = await get_connection(self._store, vca_id=vca_id) + libjuju = Libjuju(vca_connection, log=self.log, n2vc=self) + controller = await libjuju.get_controller() + await libjuju.disconnect_controller(controller)