osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import traceback
  24 import json
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.ng_ro import NgRoClient, NgRoException
  29 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase, deep_get, get_iterable, populate_dict
  30 from n2vc.k8s_helm_conn import K8sHelmConnector
  31 from n2vc.k8s_juju_conn import K8sJujuConnector
  32
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35
  36 from n2vc.n2vc_juju_conn import N2VCJujuConnector
  37 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
  38
  39 from osm_lcm.lcm_helm_conn import LCMHelmConn
  40
  41 from copy import copy, deepcopy
  42 from http import HTTPStatus
  43 from time import time
  44 from uuid import uuid4
  45
  46 from random import randint
  47
  48 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  49
  50
  51 class N2VCJujuConnectorLCM(N2VCJujuConnector):
  52
  53     async def create_execution_environment(self, namespace: str, db_dict: dict, reuse_ee_id: str = None,
  54                                            progress_timeout: float = None, total_timeout: float = None,
  55                                            config: dict = None, artifact_path: str = None,
  56                                            vca_type: str = None) -> (str, dict):
  57         # admit two new parameters, artifact_path and vca_type
  58         if vca_type == "k8s_proxy_charm":
  59             ee_id = await self.install_k8s_proxy_charm(
  60                 charm_name=artifact_path[artifact_path.rfind("/") + 1:],
  61                 namespace=namespace,
  62                 artifact_path=artifact_path,
  63                 db_dict=db_dict)
  64             return ee_id, None
  65         else:
  66             return await super().create_execution_environment(
  67                 namespace=namespace, db_dict=db_dict, reuse_ee_id=reuse_ee_id,
  68                 progress_timeout=progress_timeout, total_timeout=total_timeout)
  69
  70     async def install_configuration_sw(self, ee_id: str, artifact_path: str, db_dict: dict,
  71                                        progress_timeout: float = None, total_timeout: float = None,
  72                                        config: dict = None, num_units: int = 1, vca_type: str = "lxc_proxy_charm"):
  73         if vca_type == "k8s_proxy_charm":
  74             return
  75         return await super().install_configuration_sw(
  76             ee_id=ee_id, artifact_path=artifact_path, db_dict=db_dict, progress_timeout=progress_timeout,
  77             total_timeout=total_timeout, config=config, num_units=num_units)
  78
  79
  80 class NsLcm(LcmBase):
  81     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  82     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
  83     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
  84     timeout_charm_delete = 10 * 60
  85     timeout_primitive = 30 * 60  # timeout for primitive execution
  86     timeout_progress_primitive = 10 * 60  # timeout for some progress in a primitive execution
  87
  88     SUBOPERATION_STATUS_NOT_FOUND = -1
  89     SUBOPERATION_STATUS_NEW = -2
  90     SUBOPERATION_STATUS_SKIP = -3
  91     task_name_deploy_vca = "Deploying VCA"
  92
  93     def __init__(self, db, msg, fs, lcm_tasks, config, loop, prometheus=None):
  94         """
  95         Init, Connect to database, filesystem storage, and messaging
  96         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  97         :return: None
  98         """
  99         super().__init__(
 100             db=db,
 101             msg=msg,
 102             fs=fs,
 103             logger=logging.getLogger('lcm.ns')
 104         )
 105
 106         self.loop = loop
 107         self.lcm_tasks = lcm_tasks
 108         self.timeout = config["timeout"]
 109         self.ro_config = config["ro_config"]
 110         self.ng_ro = config["ro_config"].get("ng")
 111         self.vca_config = config["VCA"].copy()
 112
 113         # create N2VC connector
 114         self.n2vc = N2VCJujuConnectorLCM(
 115             db=self.db,
 116             fs=self.fs,
 117             log=self.logger,
 118             loop=self.loop,
 119             url='{}:{}'.format(self.vca_config['host'], self.vca_config['port']),
 120             username=self.vca_config.get('user', None),
 121             vca_config=self.vca_config,
 122             on_update_db=self._on_update_n2vc_db
 123         )
 124
 125         self.conn_helm_ee = LCMHelmConn(
 126             db=self.db,
 127             fs=self.fs,
 128             log=self.logger,
 129             loop=self.loop,
 130             url=None,
 131             username=None,
 132             vca_config=self.vca_config,
 133             on_update_db=self._on_update_n2vc_db
 134         )
 135
 136         self.k8sclusterhelm = K8sHelmConnector(
 137             kubectl_command=self.vca_config.get("kubectlpath"),
 138             helm_command=self.vca_config.get("helmpath"),
 139             fs=self.fs,
 140             log=self.logger,
 141             db=self.db,
 142             on_update_db=None,
 143         )
 144
 145         self.k8sclusterjuju = K8sJujuConnector(
 146             kubectl_command=self.vca_config.get("kubectlpath"),
 147             juju_command=self.vca_config.get("jujupath"),
 148             fs=self.fs,
 149             log=self.logger,
 150             db=self.db,
 151             on_update_db=None,
 152         )
 153
 154         self.k8scluster_map = {
 155             "helm-chart": self.k8sclusterhelm,
 156             "chart": self.k8sclusterhelm,
 157             "juju-bundle": self.k8sclusterjuju,
 158             "juju": self.k8sclusterjuju,
 159         }
 160
 161         self.vca_map = {
 162             "lxc_proxy_charm": self.n2vc,
 163             "native_charm": self.n2vc,
 164             "k8s_proxy_charm": self.n2vc,
 165             "helm": self.conn_helm_ee
 166         }
 167
 168         self.prometheus = prometheus
 169
 170         # create RO client
 171         if self.ng_ro:
 172             self.RO = NgRoClient(self.loop, **self.ro_config)
 173         else:
 174             self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 175
 176     def _on_update_ro_db(self, nsrs_id, ro_descriptor):
 177
 178         # self.logger.debug('_on_update_ro_db(nsrs_id={}'.format(nsrs_id))
 179
 180         try:
 181             # TODO filter RO descriptor fields...
 182
 183             # write to database
 184             db_dict = dict()
 185             # db_dict['deploymentStatus'] = yaml.dump(ro_descriptor, default_flow_style=False, indent=2)
 186             db_dict['deploymentStatus'] = ro_descriptor
 187             self.update_db_2("nsrs", nsrs_id, db_dict)
 188
 189         except Exception as e:
 190             self.logger.warn('Cannot write database RO deployment for ns={} -> {}'.format(nsrs_id, e))
 191
 192     async def _on_update_n2vc_db(self, table, filter, path, updated_data):
 193
 194         # remove last dot from path (if exists)
 195         if path.endswith('.'):
 196             path = path[:-1]
 197
 198         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 199         #                   .format(table, filter, path, updated_data))
 200
 201         try:
 202
 203             nsr_id = filter.get('_id')
 204
 205             # read ns record from database
 206             nsr = self.db.get_one(table='nsrs', q_filter=filter)
 207             current_ns_status = nsr.get('nsState')
 208
 209             # get vca status for NS
 210             status_dict = await self.n2vc.get_status(namespace='.' + nsr_id, yaml_format=False)
 211
 212             # vcaStatus
 213             db_dict = dict()
 214             db_dict['vcaStatus'] = status_dict
 215
 216             # update configurationStatus for this VCA
 217             try:
 218                 vca_index = int(path[path.rfind(".")+1:])
 219
 220                 vca_list = deep_get(target_dict=nsr, key_list=('_admin', 'deployed', 'VCA'))
 221                 vca_status = vca_list[vca_index].get('status')
 222
 223                 configuration_status_list = nsr.get('configurationStatus')
 224                 config_status = configuration_status_list[vca_index].get('status')
 225
 226                 if config_status == 'BROKEN' and vca_status != 'failed':
 227                     db_dict['configurationStatus'][vca_index] = 'READY'
 228                 elif config_status != 'BROKEN' and vca_status == 'failed':
 229                     db_dict['configurationStatus'][vca_index] = 'BROKEN'
 230             except Exception as e:
 231                 # not update configurationStatus
 232                 self.logger.debug('Error updating vca_index (ignore): {}'.format(e))
 233
 234             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 235             # if nsState = 'DEGRADED' check if all is OK
 236             is_degraded = False
 237             if current_ns_status in ('READY', 'DEGRADED'):
 238                 error_description = ''
 239                 # check machines
 240                 if status_dict.get('machines'):
 241                     for machine_id in status_dict.get('machines'):
 242                         machine = status_dict.get('machines').get(machine_id)
 243                         # check machine agent-status
 244                         if machine.get('agent-status'):
 245                             s = machine.get('agent-status').get('status')
 246                             if s != 'started':
 247                                 is_degraded = True
 248                                 error_description += 'machine {} agent-status={} ; '.format(machine_id, s)
 249                         # check machine instance status
 250                         if machine.get('instance-status'):
 251                             s = machine.get('instance-status').get('status')
 252                             if s != 'running':
 253                                 is_degraded = True
 254                                 error_description += 'machine {} instance-status={} ; '.format(machine_id, s)
 255                 # check applications
 256                 if status_dict.get('applications'):
 257                     for app_id in status_dict.get('applications'):
 258                         app = status_dict.get('applications').get(app_id)
 259                         # check application status
 260                         if app.get('status'):
 261                             s = app.get('status').get('status')
 262                             if s != 'active':
 263                                 is_degraded = True
 264                                 error_description += 'application {} status={} ; '.format(app_id, s)
 265
 266                 if error_description:
 267                     db_dict['errorDescription'] = error_description
 268                 if current_ns_status == 'READY' and is_degraded:
 269                     db_dict['nsState'] = 'DEGRADED'
 270                 if current_ns_status == 'DEGRADED' and not is_degraded:
 271                     db_dict['nsState'] = 'READY'
 272
 273             # write to database
 274             self.update_db_2("nsrs", nsr_id, db_dict)
 275
 276         except (asyncio.CancelledError, asyncio.TimeoutError):
 277             raise
 278         except Exception as e:
 279             self.logger.warn('Error updating NS state for ns={}: {}'.format(nsr_id, e))
 280
 281     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 282         """
 283         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 284         :param vnfd: input vnfd
 285         :param new_id: overrides vnf id if provided
 286         :param additionalParams: Instantiation params for VNFs provided
 287         :param nsrId: Id of the NSR
 288         :return: copy of vnfd
 289         """
 290         try:
 291             vnfd_RO = deepcopy(vnfd)
 292             # remove unused by RO configuration, monitoring, scaling and internal keys
 293             vnfd_RO.pop("_id", None)
 294             vnfd_RO.pop("_admin", None)
 295             vnfd_RO.pop("vnf-configuration", None)
 296             vnfd_RO.pop("monitoring-param", None)
 297             vnfd_RO.pop("scaling-group-descriptor", None)
 298             vnfd_RO.pop("kdu", None)
 299             vnfd_RO.pop("k8s-cluster", None)
 300             if new_id:
 301                 vnfd_RO["id"] = new_id
 302
 303             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 304             for vdu in get_iterable(vnfd_RO, "vdu"):
 305                 cloud_init_file = None
 306                 if vdu.get("cloud-init-file"):
 307                     base_folder = vnfd["_admin"]["storage"]
 308                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 309                                                                    vdu["cloud-init-file"])
 310                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 311                         cloud_init_content = ci_file.read()
 312                     vdu.pop("cloud-init-file", None)
 313                 elif vdu.get("cloud-init"):
 314                     cloud_init_content = vdu["cloud-init"]
 315                 else:
 316                     continue
 317
 318                 env = Environment()
 319                 ast = env.parse(cloud_init_content)
 320                 mandatory_vars = meta.find_undeclared_variables(ast)
 321                 if mandatory_vars:
 322                     for var in mandatory_vars:
 323                         if not additionalParams or var not in additionalParams.keys():
 324                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 325                                                "file, must be provided in the instantiation parameters inside the "
 326                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 327                 template = Template(cloud_init_content)
 328                 cloud_init_content = template.render(additionalParams or {})
 329                 vdu["cloud-init"] = cloud_init_content
 330
 331             return vnfd_RO
 332         except FsException as e:
 333             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 334                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 335         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 336             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 337                                format(vnfd["id"], vdu["id"], e))
 338
 339     def _ns_params_2_RO(self, ns_params, nsd, vnfd_dict, db_vnfrs, n2vc_key_list):
 340         """
 341         Creates a RO ns descriptor from OSM ns_instantiate params
 342         :param ns_params: OSM instantiate params
 343         :param vnfd_dict: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
 344         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index. {member-vnf-index: {vnfr_object}, ...}
 345         :return: The RO ns descriptor
 346         """
 347         vim_2_RO = {}
 348         wim_2_RO = {}
 349         # TODO feature 1417: Check that no instantiation is set over PDU
 350         # check if PDU forces a concrete vim-network-id and add it
 351         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 352
 353         def vim_account_2_RO(vim_account):
 354             if vim_account in vim_2_RO:
 355                 return vim_2_RO[vim_account]
 356
 357             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 358             if db_vim["_admin"]["operationalState"] != "ENABLED":
 359                 raise LcmException("VIM={} is not available. operationalState={}".format(
 360                     vim_account, db_vim["_admin"]["operationalState"]))
 361             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 362             vim_2_RO[vim_account] = RO_vim_id
 363             return RO_vim_id
 364
 365         def wim_account_2_RO(wim_account):
 366             if isinstance(wim_account, str):
 367                 if wim_account in wim_2_RO:
 368                     return wim_2_RO[wim_account]
 369
 370                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 371                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 372                     raise LcmException("WIM={} is not available. operationalState={}".format(
 373                         wim_account, db_wim["_admin"]["operationalState"]))
 374                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 375                 wim_2_RO[wim_account] = RO_wim_id
 376                 return RO_wim_id
 377             else:
 378                 return wim_account
 379
 380         def ip_profile_2_RO(ip_profile):
 381             RO_ip_profile = deepcopy((ip_profile))
 382             if "dns-server" in RO_ip_profile:
 383                 if isinstance(RO_ip_profile["dns-server"], list):
 384                     RO_ip_profile["dns-address"] = []
 385                     for ds in RO_ip_profile.pop("dns-server"):
 386                         RO_ip_profile["dns-address"].append(ds['address'])
 387                 else:
 388                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 389             if RO_ip_profile.get("ip-version") == "ipv4":
 390                 RO_ip_profile["ip-version"] = "IPv4"
 391             if RO_ip_profile.get("ip-version") == "ipv6":
 392                 RO_ip_profile["ip-version"] = "IPv6"
 393             if "dhcp-params" in RO_ip_profile:
 394                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 395             return RO_ip_profile
 396
 397         if not ns_params:
 398             return None
 399         RO_ns_params = {
 400             # "name": ns_params["nsName"],
 401             # "description": ns_params.get("nsDescription"),
 402             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 403             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 404             # "scenario": ns_params["nsdId"],
 405         }
 406         # set vim_account of each vnf if different from general vim_account.
 407         # Get this information from <vnfr> database content, key vim-account-id
 408         # Vim account can be set by placement_engine and it may be different from
 409         # the instantiate parameters (vnfs.member-vnf-index.datacenter).
 410         for vnf_index, vnfr in db_vnfrs.items():
 411             if vnfr.get("vim-account-id") and vnfr["vim-account-id"] != ns_params["vimAccountId"]:
 412                 populate_dict(RO_ns_params, ("vnfs", vnf_index, "datacenter"), vim_account_2_RO(vnfr["vim-account-id"]))
 413
 414         n2vc_key_list = n2vc_key_list or []
 415         for vnfd_ref, vnfd in vnfd_dict.items():
 416             vdu_needed_access = []
 417             mgmt_cp = None
 418             if vnfd.get("vnf-configuration"):
 419                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 420                 if ssh_required and vnfd.get("mgmt-interface"):
 421                     if vnfd["mgmt-interface"].get("vdu-id"):
 422                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 423                     elif vnfd["mgmt-interface"].get("cp"):
 424                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 425
 426             for vdu in vnfd.get("vdu", ()):
 427                 if vdu.get("vdu-configuration"):
 428                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 429                     if ssh_required:
 430                         vdu_needed_access.append(vdu["id"])
 431                 elif mgmt_cp:
 432                     for vdu_interface in vdu.get("interface"):
 433                         if vdu_interface.get("external-connection-point-ref") and \
 434                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 435                             vdu_needed_access.append(vdu["id"])
 436                             mgmt_cp = None
 437                             break
 438
 439             if vdu_needed_access:
 440                 for vnf_member in nsd.get("constituent-vnfd"):
 441                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 442                         continue
 443                     for vdu in vdu_needed_access:
 444                         populate_dict(RO_ns_params,
 445                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 446                                       n2vc_key_list)
 447
 448         if ns_params.get("vduImage"):
 449             RO_ns_params["vduImage"] = ns_params["vduImage"]
 450
 451         if ns_params.get("ssh_keys"):
 452             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 453         for vnf_params in get_iterable(ns_params, "vnf"):
 454             for constituent_vnfd in nsd["constituent-vnfd"]:
 455                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 456                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 457                     break
 458             else:
 459                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 460                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 461
 462             for vdu_params in get_iterable(vnf_params, "vdu"):
 463                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 464                 if vdu_params.get("volume"):
 465                     for volume_params in vdu_params["volume"]:
 466                         if volume_params.get("vim-volume-id"):
 467                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 468                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 469                                           volume_params["vim-volume-id"])
 470                 if vdu_params.get("interface"):
 471                     for interface_params in vdu_params["interface"]:
 472                         if interface_params.get("ip-address"):
 473                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 474                                                          vdu_params["id"], "interfaces", interface_params["name"],
 475                                                          "ip_address"),
 476                                           interface_params["ip-address"])
 477                         if interface_params.get("mac-address"):
 478                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 479                                                          vdu_params["id"], "interfaces", interface_params["name"],
 480                                                          "mac_address"),
 481                                           interface_params["mac-address"])
 482                         if interface_params.get("floating-ip-required"):
 483                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 484                                                          vdu_params["id"], "interfaces", interface_params["name"],
 485                                                          "floating-ip"),
 486                                           interface_params["floating-ip-required"])
 487
 488             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 489                 if internal_vld_params.get("vim-network-name"):
 490                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 491                                                  internal_vld_params["name"], "vim-network-name"),
 492                                   internal_vld_params["vim-network-name"])
 493                 if internal_vld_params.get("vim-network-id"):
 494                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 495                                                  internal_vld_params["name"], "vim-network-id"),
 496                                   internal_vld_params["vim-network-id"])
 497                 if internal_vld_params.get("ip-profile"):
 498                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 499                                                  internal_vld_params["name"], "ip-profile"),
 500                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 501                 if internal_vld_params.get("provider-network"):
 502
 503                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 504                                                  internal_vld_params["name"], "provider-network"),
 505                                   internal_vld_params["provider-network"].copy())
 506
 507                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 508                     # look for interface
 509                     iface_found = False
 510                     for vdu_descriptor in vnf_descriptor["vdu"]:
 511                         for vdu_interface in vdu_descriptor["interface"]:
 512                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 513                                 if icp_params.get("ip-address"):
 514                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 515                                                                  vdu_descriptor["id"], "interfaces",
 516                                                                  vdu_interface["name"], "ip_address"),
 517                                                   icp_params["ip-address"])
 518
 519                                 if icp_params.get("mac-address"):
 520                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 521                                                                  vdu_descriptor["id"], "interfaces",
 522                                                                  vdu_interface["name"], "mac_address"),
 523                                                   icp_params["mac-address"])
 524                                 iface_found = True
 525                                 break
 526                         if iface_found:
 527                             break
 528                     else:
 529                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 530                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 531                                            "connection-point".format(vnf_params["member-vnf-index"],
 532                                                                      icp_params["id-ref"]))
 533
 534         for vld_params in get_iterable(ns_params, "vld"):
 535             if "ip-profile" in vld_params:
 536                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 537                               ip_profile_2_RO(vld_params["ip-profile"]))
 538
 539             if vld_params.get("provider-network"):
 540
 541                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "provider-network"),
 542                               vld_params["provider-network"].copy())
 543
 544             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 545                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 546                               wim_account_2_RO(vld_params["wimAccountId"])),
 547             if vld_params.get("vim-network-name"):
 548                 RO_vld_sites = []
 549                 if isinstance(vld_params["vim-network-name"], dict):
 550                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 551                         RO_vld_sites.append({
 552                             "netmap-use": vim_net,
 553                             "datacenter": vim_account_2_RO(vim_account)
 554                         })
 555                 else:  # isinstance str
 556                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 557                 if RO_vld_sites:
 558                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 559
 560             if vld_params.get("vim-network-id"):
 561                 RO_vld_sites = []
 562                 if isinstance(vld_params["vim-network-id"], dict):
 563                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 564                         RO_vld_sites.append({
 565                             "netmap-use": vim_net,
 566                             "datacenter": vim_account_2_RO(vim_account)
 567                         })
 568                 else:  # isinstance str
 569                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 570                 if RO_vld_sites:
 571                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 572             if vld_params.get("ns-net"):
 573                 if isinstance(vld_params["ns-net"], dict):
 574                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 575                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 576                         populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 577             if "vnfd-connection-point-ref" in vld_params:
 578                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 579                     # look for interface
 580                     for constituent_vnfd in nsd["constituent-vnfd"]:
 581                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 582                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 583                             break
 584                     else:
 585                         raise LcmException(
 586                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 587                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 588                     match_cp = False
 589                     for vdu_descriptor in vnf_descriptor["vdu"]:
 590                         for interface_descriptor in vdu_descriptor["interface"]:
 591                             if interface_descriptor.get("external-connection-point-ref") == \
 592                                     cp_params["vnfd-connection-point-ref"]:
 593                                 match_cp = True
 594                                 break
 595                         if match_cp:
 596                             break
 597                     else:
 598                         raise LcmException(
 599                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 600                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 601                                 cp_params["member-vnf-index-ref"],
 602                                 cp_params["vnfd-connection-point-ref"],
 603                                 vnf_descriptor["id"]))
 604                     if cp_params.get("ip-address"):
 605                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 606                                                      vdu_descriptor["id"], "interfaces",
 607                                                      interface_descriptor["name"], "ip_address"),
 608                                       cp_params["ip-address"])
 609                     if cp_params.get("mac-address"):
 610                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 611                                                      vdu_descriptor["id"], "interfaces",
 612                                                      interface_descriptor["name"], "mac_address"),
 613                                       cp_params["mac-address"])
 614         return RO_ns_params
 615
 616     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 617         # make a copy to do not change
 618         vdu_create = copy(vdu_create)
 619         vdu_delete = copy(vdu_delete)
 620
 621         vdurs = db_vnfr.get("vdur")
 622         if vdurs is None:
 623             vdurs = []
 624         vdu_index = len(vdurs)
 625         while vdu_index:
 626             vdu_index -= 1
 627             vdur = vdurs[vdu_index]
 628             if vdur.get("pdu-type"):
 629                 continue
 630             vdu_id_ref = vdur["vdu-id-ref"]
 631             if vdu_create and vdu_create.get(vdu_id_ref):
 632                 for index in range(0, vdu_create[vdu_id_ref]):
 633                     vdur = deepcopy(vdur)
 634                     vdur["_id"] = str(uuid4())
 635                     vdur["count-index"] += 1
 636                     vdurs.insert(vdu_index+1+index, vdur)
 637                 del vdu_create[vdu_id_ref]
 638             if vdu_delete and vdu_delete.get(vdu_id_ref):
 639                 del vdurs[vdu_index]
 640                 vdu_delete[vdu_id_ref] -= 1
 641                 if not vdu_delete[vdu_id_ref]:
 642                     del vdu_delete[vdu_id_ref]
 643         # check all operations are done
 644         if vdu_create or vdu_delete:
 645             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 646                 vdu_create))
 647         if vdu_delete:
 648             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 649                 vdu_delete))
 650
 651         vnfr_update = {"vdur": vdurs}
 652         db_vnfr["vdur"] = vdurs
 653         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 654
 655     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 656         """
 657         Updates database nsr with the RO info for the created vld
 658         :param ns_update_nsr: dictionary to be filled with the updated info
 659         :param db_nsr: content of db_nsr. This is also modified
 660         :param nsr_desc_RO: nsr descriptor from RO
 661         :return: Nothing, LcmException is raised on errors
 662         """
 663
 664         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 665             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 666                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 667                     continue
 668                 vld["vim-id"] = net_RO.get("vim_net_id")
 669                 vld["name"] = net_RO.get("vim_name")
 670                 vld["status"] = net_RO.get("status")
 671                 vld["status-detailed"] = net_RO.get("error_msg")
 672                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 673                 break
 674             else:
 675                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 676
 677     def set_vnfr_at_error(self, db_vnfrs, error_text):
 678         try:
 679             for db_vnfr in db_vnfrs.values():
 680                 vnfr_update = {"status": "ERROR"}
 681                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 682                     if "status" not in vdur:
 683                         vdur["status"] = "ERROR"
 684                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 685                         if error_text:
 686                             vdur["status-detailed"] = str(error_text)
 687                             vnfr_update["vdur.{}.status-detailed".format(vdu_index)] = "ERROR"
 688                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 689         except DbException as e:
 690             self.logger.error("Cannot update vnf. {}".format(e))
 691
 692     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 693         """
 694         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 695         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 696         :param nsr_desc_RO: nsr descriptor from RO
 697         :return: Nothing, LcmException is raised on errors
 698         """
 699         for vnf_index, db_vnfr in db_vnfrs.items():
 700             for vnf_RO in nsr_desc_RO["vnfs"]:
 701                 if vnf_RO["member_vnf_index"] != vnf_index:
 702                     continue
 703                 vnfr_update = {}
 704                 if vnf_RO.get("ip_address"):
 705                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 706                 elif not db_vnfr.get("ip-address"):
 707                     if db_vnfr.get("vdur"):   # if not VDUs, there is not ip_address
 708                         raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 709
 710                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 711                     vdur_RO_count_index = 0
 712                     if vdur.get("pdu-type"):
 713                         continue
 714                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 715                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 716                             continue
 717                         if vdur["count-index"] != vdur_RO_count_index:
 718                             vdur_RO_count_index += 1
 719                             continue
 720                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 721                         if vdur_RO.get("ip_address"):
 722                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 723                         else:
 724                             vdur["ip-address"] = None
 725                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 726                         vdur["name"] = vdur_RO.get("vim_name")
 727                         vdur["status"] = vdur_RO.get("status")
 728                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 729                         for ifacer in get_iterable(vdur, "interfaces"):
 730                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 731                                 if ifacer["name"] == interface_RO.get("internal_name"):
 732                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 733                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 734                                     break
 735                             else:
 736                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 737                                                    "from VIM info"
 738                                                    .format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 739                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 740                         break
 741                     else:
 742                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 743                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 744
 745                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 746                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 747                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 748                             continue
 749                         vld["vim-id"] = net_RO.get("vim_net_id")
 750                         vld["name"] = net_RO.get("vim_name")
 751                         vld["status"] = net_RO.get("status")
 752                         vld["status-detailed"] = net_RO.get("error_msg")
 753                         vnfr_update["vld.{}".format(vld_index)] = vld
 754                         break
 755                     else:
 756                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 757                             vnf_index, vld["id"]))
 758
 759                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 760                 break
 761
 762             else:
 763                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 764
 765     def _get_ns_config_info(self, nsr_id):
 766         """
 767         Generates a mapping between vnf,vdu elements and the N2VC id
 768         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 769         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 770             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 771             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 772         """
 773         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 774         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 775         mapping = {}
 776         ns_config_info = {"osm-config-mapping": mapping}
 777         for vca in vca_deployed_list:
 778             if not vca["member-vnf-index"]:
 779                 continue
 780             if not vca["vdu_id"]:
 781                 mapping[vca["member-vnf-index"]] = vca["application"]
 782             else:
 783                 mapping["{}.{}.{}".format(vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"])] =\
 784                     vca["application"]
 785         return ns_config_info
 786
 787     @staticmethod
 788     def _get_initial_config_primitive_list(desc_primitive_list, vca_deployed, ee_descriptor_id):
 789         """
 790         Generates a list of initial-config-primitive based on the list provided by the descriptor. It includes internal
 791         primitives as verify-ssh-credentials, or config when needed
 792         :param desc_primitive_list: information of the descriptor
 793         :param vca_deployed: information of the deployed, needed for known if it is related to an NS, VNF, VDU and if
 794             this element contains a ssh public key
 795         :param ee_descriptor_id: execution environment descriptor id. It is the value of
 796             XXX_configuration.execution-environment-list.INDEX.id; it can be None
 797         :return: The modified list. Can ba an empty list, but always a list
 798         """
 799
 800         primitive_list = desc_primitive_list or []
 801
 802         # filter primitives by ee_id
 803         primitive_list = [p for p in primitive_list if p.get("execution-environment-ref") == ee_descriptor_id]
 804
 805         # sort by 'seq'
 806         if primitive_list:
 807             primitive_list.sort(key=lambda val: int(val['seq']))
 808
 809         # look for primitive config, and get the position. None if not present
 810         config_position = None
 811         for index, primitive in enumerate(primitive_list):
 812             if primitive["name"] == "config":
 813                 config_position = index
 814                 break
 815
 816         # for NS, add always a config primitive if not present (bug 874)
 817         if not vca_deployed["member-vnf-index"] and config_position is None:
 818             primitive_list.insert(0, {"name": "config", "parameter": []})
 819             config_position = 0
 820         # TODO revise if needed: for VNF/VDU add verify-ssh-credentials after config
 821         if vca_deployed["member-vnf-index"] and config_position is not None and vca_deployed.get("ssh-public-key"):
 822             primitive_list.insert(config_position + 1, {"name": "verify-ssh-credentials", "parameter": []})
 823         return primitive_list
 824
 825     async def _instantiate_ng_ro(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
 826                                  n2vc_key_list, stage, start_deploy, timeout_ns_deploy):
 827         nslcmop_id = db_nslcmop["_id"]
 828         target = {
 829             "name": db_nsr["name"],
 830             "ns": {"vld": []},
 831             "vnf": [],
 832             "image": deepcopy(db_nsr["image"]),
 833             "flavor": deepcopy(db_nsr["flavor"]),
 834             "action_id": nslcmop_id,
 835         }
 836         for image in target["image"]:
 837             image["vim_info"] = []
 838         for flavor in target["flavor"]:
 839             flavor["vim_info"] = []
 840
 841         ns_params = db_nslcmop.get("operationParams")
 842         ssh_keys = []
 843         if ns_params.get("ssh_keys"):
 844             ssh_keys += ns_params.get("ssh_keys")
 845         if n2vc_key_list:
 846             ssh_keys += n2vc_key_list
 847
 848         cp2target = {}
 849         for vld_index, vld in enumerate(nsd.get("vld")):
 850             target_vld = {"id": vld["id"],
 851                           "name": vld["name"],
 852                           "mgmt-network": vld.get("mgmt-network", False),
 853                           "type": vld.get("type"),
 854                           "vim_info": [{"vim-network-name": vld.get("vim-network-name"),
 855                                         "vim_account_id": ns_params["vimAccountId"]}],
 856                           }
 857             for cp in vld["vnfd-connection-point-ref"]:
 858                 cp2target["member_vnf:{}.{}".format(cp["member-vnf-index-ref"], cp["vnfd-connection-point-ref"])] = \
 859                     "nsrs:{}:vld.{}".format(nsr_id, vld_index)
 860             target["ns"]["vld"].append(target_vld)
 861         for vnfr in db_vnfrs.values():
 862             vnfd = db_vnfds_ref[vnfr["vnfd-ref"]]
 863             target_vnf = deepcopy(vnfr)
 864             for vld in target_vnf.get("vld", ()):
 865                 # check if connected to a ns.vld
 866                 vnf_cp = next((cp for cp in vnfd.get("connection-point", ()) if
 867                                cp.get("internal-vld-ref") == vld["id"]), None)
 868                 if vnf_cp:
 869                     ns_cp = "member_vnf:{}.{}".format(vnfr["member-vnf-index-ref"], vnf_cp["id"])
 870                     if cp2target.get(ns_cp):
 871                         vld["target"] = cp2target[ns_cp]
 872                 vld["vim_info"] = [{"vim-network-name": vld.get("vim-network-name"),
 873                                     "vim_account_id": vnfr["vim-account-id"]}]
 874
 875             for vdur in target_vnf.get("vdur", ()):
 876                 vdur["vim_info"] = [{"vim_account_id": vnfr["vim-account-id"]}]
 877                 vdud_index, vdud = next(k for k in enumerate(vnfd["vdu"]) if k[1]["id"] == vdur["vdu-id-ref"])
 878                 # vdur["additionalParams"] = vnfr.get("additionalParamsForVnf")  # TODO additional params for VDU
 879
 880                 if ssh_keys:
 881                     if deep_get(vdud, ("vdu-configuration", "config-access", "ssh-access", "required")):
 882                         vdur["ssh-keys"] = ssh_keys
 883                         vdur["ssh-access-required"] = True
 884                     elif deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required")) and \
 885                             any(iface.get("mgmt-vnf") for iface in vdur["interfaces"]):
 886                         vdur["ssh-keys"] = ssh_keys
 887                         vdur["ssh-access-required"] = True
 888
 889                 # cloud-init
 890                 if vdud.get("cloud-init-file"):
 891                     vdur["cloud-init"] = "{}:file:{}".format(vnfd["_id"], vdud.get("cloud-init-file"))
 892                 elif vdud.get("cloud-init"):
 893                     vdur["cloud-init"] = "{}:vdu:{}".format(vnfd["_id"], vdud_index)
 894
 895                 # flavor
 896                 ns_flavor = target["flavor"][int(vdur["ns-flavor-id"])]
 897                 if not next((vi for vi in ns_flavor["vim_info"] if
 898                              vi and vi.get("vim_account_id") == vnfr["vim-account-id"]), None):
 899                     ns_flavor["vim_info"].append({"vim_account_id": vnfr["vim-account-id"]})
 900                 # image
 901                 ns_image = target["image"][int(vdur["ns-image-id"])]
 902                 if not next((vi for vi in ns_image["vim_info"] if
 903                              vi and vi.get("vim_account_id") == vnfr["vim-account-id"]), None):
 904                     ns_image["vim_info"].append({"vim_account_id": vnfr["vim-account-id"]})
 905
 906                 vdur["vim_info"] = [{"vim_account_id": vnfr["vim-account-id"]}]
 907             target["vnf"].append(target_vnf)
 908
 909         desc = await self.RO.deploy(nsr_id, target)
 910         action_id = desc["action_id"]
 911         await self._wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_deploy, timeout_ns_deploy, stage)
 912
 913         # Updating NSR
 914         db_nsr_update = {
 915             "_admin.deployed.RO.operational-status": "running",
 916             "detailed-status": " ".join(stage)
 917         }
 918         # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
 919         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 920         self._write_op_status(nslcmop_id, stage)
 921         self.logger.debug(logging_text + "ns deployed at RO. RO_id={}".format(action_id))
 922         return
 923
 924     async def _wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_time, timeout, stage):
 925         detailed_status_old = None
 926         db_nsr_update = {}
 927         while time() <= start_time + timeout:
 928             desc_status = await self.RO.status(nsr_id, action_id)
 929             if desc_status["status"] == "FAILED":
 930                 raise NgRoException(desc_status["details"])
 931             elif desc_status["status"] == "BUILD":
 932                 stage[2] = "VIM: ({})".format(desc_status["details"])
 933             elif desc_status["status"] == "DONE":
 934                 stage[2] = "Deployed at VIM"
 935                 break
 936             else:
 937                 assert False, "ROclient.check_ns_status returns unknown {}".format(desc_status["status"])
 938             if stage[2] != detailed_status_old:
 939                 detailed_status_old = stage[2]
 940                 db_nsr_update["detailed-status"] = " ".join(stage)
 941                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 942                 self._write_op_status(nslcmop_id, stage)
 943             await asyncio.sleep(5, loop=self.loop)
 944         else:  # timeout_ns_deploy
 945             raise NgRoException("Timeout waiting ns to deploy")
 946
 947     async def _terminate_ng_ro(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
 948         db_nsr_update = {}
 949         failed_detail = []
 950         action_id = None
 951         start_deploy = time()
 952         try:
 953             target = {
 954                 "ns": {"vld": []},
 955                 "vnf": [],
 956                 "image": [],
 957                 "flavor": [],
 958             }
 959             desc = await self.RO.deploy(nsr_id, target)
 960             action_id = desc["action_id"]
 961             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = action_id
 962             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETING"
 963             self.logger.debug(logging_text + "ns terminate action at RO. action_id={}".format(action_id))
 964
 965             # wait until done
 966             delete_timeout = 20 * 60  # 20 minutes
 967             await self._wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_deploy, delete_timeout, stage)
 968
 969             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
 970             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 971             # delete all nsr
 972             await self.RO.delete(nsr_id)
 973         except Exception as e:
 974             if isinstance(e, NgRoException) and e.http_code == 404:  # not found
 975                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 976                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 977                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
 978                 self.logger.debug(logging_text + "RO_action_id={} already deleted".format(action_id))
 979             elif isinstance(e, NgRoException) and e.http_code == 409:  # conflict
 980                 failed_detail.append("delete conflict: {}".format(e))
 981                 self.logger.debug(logging_text + "RO_action_id={} delete conflict: {}".format(action_id, e))
 982             else:
 983                 failed_detail.append("delete error: {}".format(e))
 984                 self.logger.error(logging_text + "RO_action_id={} delete error: {}".format(action_id, e))
 985
 986         if failed_detail:
 987             stage[2] = "Error deleting from VIM"
 988         else:
 989             stage[2] = "Deleted from VIM"
 990         db_nsr_update["detailed-status"] = " ".join(stage)
 991         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 992         self._write_op_status(nslcmop_id, stage)
 993
 994         if failed_detail:
 995             raise LcmException("; ".join(failed_detail))
 996         return
 997
 998     async def instantiate_RO(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
 999                              n2vc_key_list, stage):
1000         """
1001         Instantiate at RO
1002         :param logging_text: preffix text to use at logging
1003         :param nsr_id: nsr identity
1004         :param nsd: database content of ns descriptor
1005         :param db_nsr: database content of ns record
1006         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
1007         :param db_vnfrs:
1008         :param db_vnfds_ref: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
1009         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
1010         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
1011         :return: None or exception
1012         """
1013         try:
1014             db_nsr_update = {}
1015             RO_descriptor_number = 0   # number of descriptors created at RO
1016             vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
1017             nslcmop_id = db_nslcmop["_id"]
1018             start_deploy = time()
1019             ns_params = db_nslcmop.get("operationParams")
1020             if ns_params and ns_params.get("timeout_ns_deploy"):
1021                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1022             else:
1023                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
1024
1025             # Check for and optionally request placement optimization. Database will be updated if placement activated
1026             stage[2] = "Waiting for Placement."
1027             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
1028                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
1029                 for vnfr in db_vnfrs.values():
1030                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
1031                         break
1032                 else:
1033                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
1034
1035             if self.ng_ro:
1036                 return await self._instantiate_ng_ro(logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs,
1037                                                      db_vnfds_ref, n2vc_key_list, stage, start_deploy,
1038                                                      timeout_ns_deploy)
1039             # deploy RO
1040             # get vnfds, instantiate at RO
1041             for c_vnf in nsd.get("constituent-vnfd", ()):
1042                 member_vnf_index = c_vnf["member-vnf-index"]
1043                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1044                 vnfd_ref = vnfd["id"]
1045
1046                 stage[2] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(vnfd_ref, member_vnf_index)
1047                 db_nsr_update["detailed-status"] = " ".join(stage)
1048                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1049                 self._write_op_status(nslcmop_id, stage)
1050
1051                 # self.logger.debug(logging_text + stage[2])
1052                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1053                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1054                 RO_descriptor_number += 1
1055
1056                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1057                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1058                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1059                         break
1060                 else:
1061                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1062                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1063
1064                 # look if present
1065                 RO_update = {"member-vnf-index": member_vnf_index}
1066                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1067                 if vnfd_list:
1068                     RO_update["id"] = vnfd_list[0]["uuid"]
1069                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1070                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1071                 else:
1072                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1073                                            get("additionalParamsForVnf"), nsr_id)
1074                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1075                     RO_update["id"] = desc["uuid"]
1076                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1077                         vnfd_ref, member_vnf_index, desc["uuid"]))
1078                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1079                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1080
1081             # create nsd at RO
1082             nsd_ref = nsd["id"]
1083
1084             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
1085             db_nsr_update["detailed-status"] = " ".join(stage)
1086             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1087             self._write_op_status(nslcmop_id, stage)
1088
1089             # self.logger.debug(logging_text + stage[2])
1090             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1091             RO_descriptor_number += 1
1092             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1093             if nsd_list:
1094                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1095                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1096                     nsd_ref, RO_nsd_uuid))
1097             else:
1098                 nsd_RO = deepcopy(nsd)
1099                 nsd_RO["id"] = RO_osm_nsd_id
1100                 nsd_RO.pop("_id", None)
1101                 nsd_RO.pop("_admin", None)
1102                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1103                     member_vnf_index = c_vnf["member-vnf-index"]
1104                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1105                 for c_vld in nsd_RO.get("vld", ()):
1106                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1107                         member_vnf_index = cp["member-vnf-index-ref"]
1108                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1109
1110                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1111                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1112                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1113                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1114             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1115
1116             # Crate ns at RO
1117             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
1118             db_nsr_update["detailed-status"] = " ".join(stage)
1119             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1120             self._write_op_status(nslcmop_id, stage)
1121
1122             # if present use it unless in error status
1123             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1124             if RO_nsr_id:
1125                 try:
1126                     stage[2] = "Looking for existing ns at RO"
1127                     db_nsr_update["detailed-status"] = " ".join(stage)
1128                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1129                     self._write_op_status(nslcmop_id, stage)
1130                     # self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
1131                     desc = await self.RO.show("ns", RO_nsr_id)
1132
1133                 except ROclient.ROClientException as e:
1134                     if e.http_code != HTTPStatus.NOT_FOUND:
1135                         raise
1136                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1137                 if RO_nsr_id:
1138                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1139                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1140                     if ns_status == "ERROR":
1141                         stage[2] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1142                         self.logger.debug(logging_text + stage[2])
1143                         await self.RO.delete("ns", RO_nsr_id)
1144                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1145             if not RO_nsr_id:
1146                 stage[2] = "Checking dependencies"
1147                 db_nsr_update["detailed-status"] = " ".join(stage)
1148                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1149                 self._write_op_status(nslcmop_id, stage)
1150                 # self.logger.debug(logging_text + stage[2])
1151
1152                 # check if VIM is creating and wait  look if previous tasks in process
1153                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1154                 if task_dependency:
1155                     stage[2] = "Waiting for related tasks '{}' to be completed".format(task_name)
1156                     self.logger.debug(logging_text + stage[2])
1157                     await asyncio.wait(task_dependency, timeout=3600)
1158                 if ns_params.get("vnf"):
1159                     for vnf in ns_params["vnf"]:
1160                         if "vimAccountId" in vnf:
1161                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1162                                                                                         vnf["vimAccountId"])
1163                         if task_dependency:
1164                             stage[2] = "Waiting for related tasks '{}' to be completed.".format(task_name)
1165                             self.logger.debug(logging_text + stage[2])
1166                             await asyncio.wait(task_dependency, timeout=3600)
1167
1168                 stage[2] = "Checking instantiation parameters."
1169                 RO_ns_params = self._ns_params_2_RO(ns_params, nsd, db_vnfds_ref, db_vnfrs, n2vc_key_list)
1170                 stage[2] = "Deploying ns at VIM."
1171                 db_nsr_update["detailed-status"] = " ".join(stage)
1172                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1173                 self._write_op_status(nslcmop_id, stage)
1174
1175                 desc = await self.RO.create("ns", descriptor=RO_ns_params, name=db_nsr["name"], scenario=RO_nsd_uuid)
1176                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1177                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1178                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1179                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1180
1181             # wait until NS is ready
1182             stage[2] = "Waiting VIM to deploy ns."
1183             db_nsr_update["detailed-status"] = " ".join(stage)
1184             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1185             self._write_op_status(nslcmop_id, stage)
1186             detailed_status_old = None
1187             self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
1188
1189             old_desc = None
1190             while time() <= start_deploy + timeout_ns_deploy:
1191                 desc = await self.RO.show("ns", RO_nsr_id)
1192
1193                 # deploymentStatus
1194                 if desc != old_desc:
1195                     # desc has changed => update db
1196                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
1197                     old_desc = desc
1198
1199                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1200                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1201                     if ns_status == "ERROR":
1202                         raise ROclient.ROClientException(ns_status_info)
1203                     elif ns_status == "BUILD":
1204                         stage[2] = "VIM: ({})".format(ns_status_info)
1205                     elif ns_status == "ACTIVE":
1206                         stage[2] = "Waiting for management IP address reported by the VIM. Updating VNFRs."
1207                         try:
1208                             self.ns_update_vnfr(db_vnfrs, desc)
1209                             break
1210                         except LcmExceptionNoMgmtIP:
1211                             pass
1212                     else:
1213                         assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1214                     if stage[2] != detailed_status_old:
1215                         detailed_status_old = stage[2]
1216                         db_nsr_update["detailed-status"] = " ".join(stage)
1217                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1218                         self._write_op_status(nslcmop_id, stage)
1219                     await asyncio.sleep(5, loop=self.loop)
1220             else:  # timeout_ns_deploy
1221                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1222
1223             # Updating NSR
1224             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1225
1226             db_nsr_update["_admin.deployed.RO.operational-status"] = "running"
1227             # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
1228             stage[2] = "Deployed at VIM"
1229             db_nsr_update["detailed-status"] = " ".join(stage)
1230             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1231             self._write_op_status(nslcmop_id, stage)
1232             # await self._on_update_n2vc_db("nsrs", {"_id": nsr_id}, "_admin.deployed", db_nsr_update)
1233             # self.logger.debug(logging_text + "Deployed at VIM")
1234         except (ROclient.ROClientException, LcmException, DbException, NgRoException) as e:
1235             stage[2] = "ERROR deploying at VIM"
1236             self.set_vnfr_at_error(db_vnfrs, str(e))
1237             raise
1238
1239     async def wait_kdu_up(self, logging_text, nsr_id, vnfr_id, kdu_name):
1240         """
1241         Wait for kdu to be up, get ip address
1242         :param logging_text: prefix use for logging
1243         :param nsr_id:
1244         :param vnfr_id:
1245         :param kdu_name:
1246         :return: IP address
1247         """
1248
1249         # self.logger.debug(logging_text + "Starting wait_kdu_up")
1250         nb_tries = 0
1251
1252         while nb_tries < 360:
1253             db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1254             kdur = next((x for x in get_iterable(db_vnfr, "kdur") if x.get("name") == kdu_name), None)
1255             if not kdur:
1256                 raise LcmException("Not found vnfr_id={}, kdu_name={}".format(vnfr_id, kdu_name))
1257             if kdur.get("status"):
1258                 if kdur["status"] in ("READY", "ENABLED"):
1259                     return kdur.get("ip-address")
1260                 else:
1261                     raise LcmException("target KDU={} is in error state".format(kdu_name))
1262
1263             await asyncio.sleep(10, loop=self.loop)
1264             nb_tries += 1
1265         raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
1266
1267     async def wait_vm_up_insert_key_ro(self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None):
1268         """
1269         Wait for ip addres at RO, and optionally, insert public key in virtual machine
1270         :param logging_text: prefix use for logging
1271         :param nsr_id:
1272         :param vnfr_id:
1273         :param vdu_id:
1274         :param vdu_index:
1275         :param pub_key: public ssh key to inject, None to skip
1276         :param user: user to apply the public ssh key
1277         :return: IP address
1278         """
1279
1280         # self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1281         ro_nsr_id = None
1282         ip_address = None
1283         nb_tries = 0
1284         target_vdu_id = None
1285         ro_retries = 0
1286
1287         while True:
1288
1289             ro_retries += 1
1290             if ro_retries >= 360:  # 1 hour
1291                 raise LcmException("Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id))
1292
1293             await asyncio.sleep(10, loop=self.loop)
1294
1295             # get ip address
1296             if not target_vdu_id:
1297                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1298
1299                 if not vdu_id:  # for the VNF case
1300                     if db_vnfr.get("status") == "ERROR":
1301                         raise LcmException("Cannot inject ssh-key because target VNF is in error state")
1302                     ip_address = db_vnfr.get("ip-address")
1303                     if not ip_address:
1304                         continue
1305                     vdur = next((x for x in get_iterable(db_vnfr, "vdur") if x.get("ip-address") == ip_address), None)
1306                 else:  # VDU case
1307                     vdur = next((x for x in get_iterable(db_vnfr, "vdur")
1308                                  if x.get("vdu-id-ref") == vdu_id and x.get("count-index") == vdu_index), None)
1309
1310                 if not vdur and len(db_vnfr.get("vdur", ())) == 1:  # If only one, this should be the target vdu
1311                     vdur = db_vnfr["vdur"][0]
1312                 if not vdur:
1313                     raise LcmException("Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(vnfr_id, vdu_id,
1314                                                                                               vdu_index))
1315
1316                 if vdur.get("pdu-type") or vdur.get("status") == "ACTIVE":
1317                     ip_address = vdur.get("ip-address")
1318                     if not ip_address:
1319                         continue
1320                     target_vdu_id = vdur["vdu-id-ref"]
1321                 elif vdur.get("status") == "ERROR":
1322                     raise LcmException("Cannot inject ssh-key because target VM is in error state")
1323
1324             if not target_vdu_id:
1325                 continue
1326
1327             # inject public key into machine
1328             if pub_key and user:
1329                 # wait until NS is deployed at RO
1330                 if not ro_nsr_id:
1331                     db_nsrs = self.db.get_one("nsrs", {"_id": nsr_id})
1332                     ro_nsr_id = deep_get(db_nsrs, ("_admin", "deployed", "RO", "nsr_id"))
1333                 if not ro_nsr_id:
1334                     continue
1335
1336                 # self.logger.debug(logging_text + "Inserting RO key")
1337                 if vdur.get("pdu-type"):
1338                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1339                     return ip_address
1340                 try:
1341                     ro_vm_id = "{}-{}".format(db_vnfr["member-vnf-index-ref"], target_vdu_id)  # TODO add vdu_index
1342                     if self.ng_ro:
1343                         target = {"action": "inject_ssh_key", "key": pub_key, "user": user,
1344                                   "vnf": [{"_id": vnfr_id, "vdur": [{"id": vdu_id}]}],
1345                                   }
1346                         await self.RO.deploy(nsr_id, target)
1347                     else:
1348                         result_dict = await self.RO.create_action(
1349                             item="ns",
1350                             item_id_name=ro_nsr_id,
1351                             descriptor={"add_public_key": pub_key, "vms": [ro_vm_id], "user": user}
1352                         )
1353                         # result_dict contains the format {VM-id: {vim_result: 200, description: text}}
1354                         if not result_dict or not isinstance(result_dict, dict):
1355                             raise LcmException("Unknown response from RO when injecting key")
1356                         for result in result_dict.values():
1357                             if result.get("vim_result") == 200:
1358                                 break
1359                             else:
1360                                 raise ROclient.ROClientException("error injecting key: {}".format(
1361                                     result.get("description")))
1362                         break
1363                 except NgRoException as e:
1364                     raise LcmException("Reaching max tries injecting key. Error: {}".format(e))
1365                 except ROclient.ROClientException as e:
1366                     if not nb_tries:
1367                         self.logger.debug(logging_text + "error injecting key: {}. Retrying until {} seconds".
1368                                           format(e, 20*10))
1369                     nb_tries += 1
1370                     if nb_tries >= 20:
1371                         raise LcmException("Reaching max tries injecting key. Error: {}".format(e))
1372             else:
1373                 break
1374
1375         return ip_address
1376
1377     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1378         """
1379         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1380         """
1381         my_vca = vca_deployed_list[vca_index]
1382         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1383             # vdu or kdu: no dependencies
1384             return
1385         timeout = 300
1386         while timeout >= 0:
1387             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1388             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1389             configuration_status_list = db_nsr["configurationStatus"]
1390             for index, vca_deployed in enumerate(configuration_status_list):
1391                 if index == vca_index:
1392                     # myself
1393                     continue
1394                 if not my_vca.get("member-vnf-index") or \
1395                         (vca_deployed.get("member-vnf-index") == my_vca.get("member-vnf-index")):
1396                     internal_status = configuration_status_list[index].get("status")
1397                     if internal_status == 'READY':
1398                         continue
1399                     elif internal_status == 'BROKEN':
1400                         raise LcmException("Configuration aborted because dependent charm/s has failed")
1401                     else:
1402                         break
1403             else:
1404                 # no dependencies, return
1405                 return
1406             await asyncio.sleep(10)
1407             timeout -= 1
1408
1409         raise LcmException("Configuration aborted because dependent charm/s timeout")
1410
1411     async def instantiate_N2VC(self, logging_text, vca_index, nsi_id, db_nsr, db_vnfr, vdu_id, kdu_name, vdu_index,
1412                                config_descriptor, deploy_params, base_folder, nslcmop_id, stage, vca_type, vca_name,
1413                                ee_config_descriptor):
1414         nsr_id = db_nsr["_id"]
1415         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1416         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1417         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1418         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
1419         db_dict = {
1420             'collection': 'nsrs',
1421             'filter': {'_id': nsr_id},
1422             'path': db_update_entry
1423         }
1424         step = ""
1425         try:
1426
1427             element_type = 'NS'
1428             element_under_configuration = nsr_id
1429
1430             vnfr_id = None
1431             if db_vnfr:
1432                 vnfr_id = db_vnfr["_id"]
1433                 osm_config["osm"]["vnf_id"] = vnfr_id
1434
1435             namespace = "{nsi}.{ns}".format(
1436                 nsi=nsi_id if nsi_id else "",
1437                 ns=nsr_id)
1438
1439             if vnfr_id:
1440                 element_type = 'VNF'
1441                 element_under_configuration = vnfr_id
1442                 namespace += ".{}".format(vnfr_id)
1443                 if vdu_id:
1444                     namespace += ".{}-{}".format(vdu_id, vdu_index or 0)
1445                     element_type = 'VDU'
1446                     element_under_configuration = "{}-{}".format(vdu_id, vdu_index or 0)
1447                     osm_config["osm"]["vdu_id"] = vdu_id
1448                 elif kdu_name:
1449                     namespace += ".{}".format(kdu_name)
1450                     element_type = 'KDU'
1451                     element_under_configuration = kdu_name
1452                     osm_config["osm"]["kdu_name"] = kdu_name
1453
1454             # Get artifact path
1455             artifact_path = "{}/{}/{}/{}".format(
1456                 base_folder["folder"],
1457                 base_folder["pkg-dir"],
1458                 "charms" if vca_type in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm") else "helm-charts",
1459                 vca_name
1460             )
1461             # get initial_config_primitive_list that applies to this element
1462             initial_config_primitive_list = config_descriptor.get('initial-config-primitive')
1463
1464             # add config if not present for NS charm
1465             ee_descriptor_id = ee_config_descriptor.get("id")
1466             initial_config_primitive_list = self._get_initial_config_primitive_list(initial_config_primitive_list,
1467                                                                                     vca_deployed, ee_descriptor_id)
1468
1469             # n2vc_redesign STEP 3.1
1470             # find old ee_id if exists
1471             ee_id = vca_deployed.get("ee_id")
1472
1473             # create or register execution environment in VCA
1474             if vca_type in ("lxc_proxy_charm", "k8s_proxy_charm", "helm"):
1475
1476                 self._write_configuration_status(
1477                     nsr_id=nsr_id,
1478                     vca_index=vca_index,
1479                     status='CREATING',
1480                     element_under_configuration=element_under_configuration,
1481                     element_type=element_type
1482                 )
1483
1484                 step = "create execution environment"
1485                 self.logger.debug(logging_text + step)
1486                 ee_id, credentials = await self.vca_map[vca_type].create_execution_environment(
1487                     namespace=namespace,
1488                     reuse_ee_id=ee_id,
1489                     db_dict=db_dict,
1490                     config=osm_config,
1491                     artifact_path=artifact_path,
1492                     vca_type=vca_type)
1493
1494             elif vca_type == "native_charm":
1495                 step = "Waiting to VM being up and getting IP address"
1496                 self.logger.debug(logging_text + step)
1497                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
1498                                                                  user=None, pub_key=None)
1499                 credentials = {"hostname": rw_mgmt_ip}
1500                 # get username
1501                 username = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1502                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1503                 #  merged. Meanwhile let's get username from initial-config-primitive
1504                 if not username and initial_config_primitive_list:
1505                     for config_primitive in initial_config_primitive_list:
1506                         for param in config_primitive.get("parameter", ()):
1507                             if param["name"] == "ssh-username":
1508                                 username = param["value"]
1509                                 break
1510                 if not username:
1511                     raise LcmException("Cannot determine the username neither with 'initial-config-primitive' nor with "
1512                                        "'config-access.ssh-access.default-user'")
1513                 credentials["username"] = username
1514                 # n2vc_redesign STEP 3.2
1515
1516                 self._write_configuration_status(
1517                     nsr_id=nsr_id,
1518                     vca_index=vca_index,
1519                     status='REGISTERING',
1520                     element_under_configuration=element_under_configuration,
1521                     element_type=element_type
1522                 )
1523
1524                 step = "register execution environment {}".format(credentials)
1525                 self.logger.debug(logging_text + step)
1526                 ee_id = await self.vca_map[vca_type].register_execution_environment(
1527                     credentials=credentials, namespace=namespace, db_dict=db_dict)
1528
1529             # for compatibility with MON/POL modules, the need model and application name at database
1530             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
1531             ee_id_parts = ee_id.split('.')
1532             db_nsr_update = {db_update_entry + "ee_id": ee_id}
1533             if len(ee_id_parts) >= 2:
1534                 model_name = ee_id_parts[0]
1535                 application_name = ee_id_parts[1]
1536                 db_nsr_update[db_update_entry + "model"] = model_name
1537                 db_nsr_update[db_update_entry + "application"] = application_name
1538
1539             # n2vc_redesign STEP 3.3
1540             step = "Install configuration Software"
1541
1542             self._write_configuration_status(
1543                 nsr_id=nsr_id,
1544                 vca_index=vca_index,
1545                 status='INSTALLING SW',
1546                 element_under_configuration=element_under_configuration,
1547                 element_type=element_type,
1548                 other_update=db_nsr_update
1549             )
1550
1551             # TODO check if already done
1552             self.logger.debug(logging_text + step)
1553             config = None
1554             if vca_type == "native_charm":
1555                 config_primitive = next((p for p in initial_config_primitive_list if p["name"] == "config"), None)
1556                 if config_primitive:
1557                     config = self._map_primitive_params(
1558                         config_primitive,
1559                         {},
1560                         deploy_params
1561                     )
1562             num_units = 1
1563             if vca_type == "lxc_proxy_charm":
1564                 if element_type == "NS":
1565                     num_units = db_nsr.get("config-units") or 1
1566                 elif element_type == "VNF":
1567                     num_units = db_vnfr.get("config-units") or 1
1568                 elif element_type == "VDU":
1569                     for v in db_vnfr["vdur"]:
1570                         if vdu_id == v["vdu-id-ref"]:
1571                             num_units = v.get("config-units") or 1
1572                             break
1573
1574             await self.vca_map[vca_type].install_configuration_sw(
1575                 ee_id=ee_id,
1576                 artifact_path=artifact_path,
1577                 db_dict=db_dict,
1578                 config=config,
1579                 num_units=num_units,
1580                 vca_type=vca_type
1581             )
1582
1583             # write in db flag of configuration_sw already installed
1584             self.update_db_2("nsrs", nsr_id, {db_update_entry + "config_sw_installed": True})
1585
1586             # add relations for this VCA (wait for other peers related with this VCA)
1587             await self._add_vca_relations(logging_text=logging_text, nsr_id=nsr_id,
1588                                           vca_index=vca_index, vca_type=vca_type)
1589
1590             # if SSH access is required, then get execution environment SSH public
1591             # if native charm we have waited already to VM be UP
1592             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm"):
1593                 pub_key = None
1594                 user = None
1595                 # self.logger.debug("get ssh key block")
1596                 if deep_get(config_descriptor, ("config-access", "ssh-access", "required")):
1597                     # self.logger.debug("ssh key needed")
1598                     # Needed to inject a ssh key
1599                     user = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1600                     step = "Install configuration Software, getting public ssh key"
1601                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(ee_id=ee_id, db_dict=db_dict)
1602
1603                     step = "Insert public key into VM user={} ssh_key={}".format(user, pub_key)
1604                 else:
1605                     # self.logger.debug("no need to get ssh key")
1606                     step = "Waiting to VM being up and getting IP address"
1607                 self.logger.debug(logging_text + step)
1608
1609                 # n2vc_redesign STEP 5.1
1610                 # wait for RO (ip-address) Insert pub_key into VM
1611                 if vnfr_id:
1612                     if kdu_name:
1613                         rw_mgmt_ip = await self.wait_kdu_up(logging_text, nsr_id, vnfr_id, kdu_name)
1614                     else:
1615                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id,
1616                                                                          vdu_index, user=user, pub_key=pub_key)
1617                 else:
1618                     rw_mgmt_ip = None   # This is for a NS configuration
1619
1620                 self.logger.debug(logging_text + ' VM_ip_address={}'.format(rw_mgmt_ip))
1621
1622             # store rw_mgmt_ip in deploy params for later replacement
1623             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
1624
1625             # n2vc_redesign STEP 6  Execute initial config primitive
1626             step = 'execute initial config primitive'
1627
1628             # wait for dependent primitives execution (NS -> VNF -> VDU)
1629             if initial_config_primitive_list:
1630                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
1631
1632             # stage, in function of element type: vdu, kdu, vnf or ns
1633             my_vca = vca_deployed_list[vca_index]
1634             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1635                 # VDU or KDU
1636                 stage[0] = 'Stage 3/5: running Day-1 primitives for VDU.'
1637             elif my_vca.get("member-vnf-index"):
1638                 # VNF
1639                 stage[0] = 'Stage 4/5: running Day-1 primitives for VNF.'
1640             else:
1641                 # NS
1642                 stage[0] = 'Stage 5/5: running Day-1 primitives for NS.'
1643
1644             self._write_configuration_status(
1645                 nsr_id=nsr_id,
1646                 vca_index=vca_index,
1647                 status='EXECUTING PRIMITIVE'
1648             )
1649
1650             self._write_op_status(
1651                 op_id=nslcmop_id,
1652                 stage=stage
1653             )
1654
1655             check_if_terminated_needed = True
1656             for initial_config_primitive in initial_config_primitive_list:
1657                 # adding information on the vca_deployed if it is a NS execution environment
1658                 if not vca_deployed["member-vnf-index"]:
1659                     deploy_params["ns_config_info"] = json.dumps(self._get_ns_config_info(nsr_id))
1660                 # TODO check if already done
1661                 primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, deploy_params)
1662
1663                 step = "execute primitive '{}' params '{}'".format(initial_config_primitive["name"], primitive_params_)
1664                 self.logger.debug(logging_text + step)
1665                 await self.vca_map[vca_type].exec_primitive(
1666                     ee_id=ee_id,
1667                     primitive_name=initial_config_primitive["name"],
1668                     params_dict=primitive_params_,
1669                     db_dict=db_dict
1670                 )
1671                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
1672                 if check_if_terminated_needed:
1673                     if config_descriptor.get('terminate-config-primitive'):
1674                         self.update_db_2("nsrs", nsr_id, {db_update_entry + "needed_terminate": True})
1675                     check_if_terminated_needed = False
1676
1677                 # TODO register in database that primitive is done
1678
1679             # STEP 7 Configure metrics
1680             if vca_type == "helm":
1681                 prometheus_jobs = await self.add_prometheus_metrics(
1682                     ee_id=ee_id,
1683                     artifact_path=artifact_path,
1684                     ee_config_descriptor=ee_config_descriptor,
1685                     vnfr_id=vnfr_id,
1686                     nsr_id=nsr_id,
1687                     target_ip=rw_mgmt_ip,
1688                 )
1689                 if prometheus_jobs:
1690                     self.update_db_2("nsrs", nsr_id, {db_update_entry + "prometheus_jobs": prometheus_jobs})
1691
1692             step = "instantiated at VCA"
1693             self.logger.debug(logging_text + step)
1694
1695             self._write_configuration_status(
1696                 nsr_id=nsr_id,
1697                 vca_index=vca_index,
1698                 status='READY'
1699             )
1700
1701         except Exception as e:  # TODO not use Exception but N2VC exception
1702             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
1703             if not isinstance(e, (DbException, N2VCException, LcmException, asyncio.CancelledError)):
1704                 self.logger.error("Exception while {} : {}".format(step, e), exc_info=True)
1705             self._write_configuration_status(
1706                 nsr_id=nsr_id,
1707                 vca_index=vca_index,
1708                 status='BROKEN'
1709             )
1710             raise LcmException("{} {}".format(step, e)) from e
1711
1712     def _write_ns_status(self, nsr_id: str, ns_state: str, current_operation: str, current_operation_id: str,
1713                          error_description: str = None, error_detail: str = None, other_update: dict = None):
1714         """
1715         Update db_nsr fields.
1716         :param nsr_id:
1717         :param ns_state:
1718         :param current_operation:
1719         :param current_operation_id:
1720         :param error_description:
1721         :param error_detail:
1722         :param other_update: Other required changes at database if provided, will be cleared
1723         :return:
1724         """
1725         try:
1726             db_dict = other_update or {}
1727             db_dict["_admin.nslcmop"] = current_operation_id    # for backward compatibility
1728             db_dict["_admin.current-operation"] = current_operation_id
1729             db_dict["_admin.operation-type"] = current_operation if current_operation != "IDLE" else None
1730             db_dict["currentOperation"] = current_operation
1731             db_dict["currentOperationID"] = current_operation_id
1732             db_dict["errorDescription"] = error_description
1733             db_dict["errorDetail"] = error_detail
1734
1735             if ns_state:
1736                 db_dict["nsState"] = ns_state
1737             self.update_db_2("nsrs", nsr_id, db_dict)
1738         except DbException as e:
1739             self.logger.warn('Error writing NS status, ns={}: {}'.format(nsr_id, e))
1740
1741     def _write_op_status(self, op_id: str, stage: list = None, error_message: str = None, queuePosition: int = 0,
1742                          operation_state: str = None, other_update: dict = None):
1743         try:
1744             db_dict = other_update or {}
1745             db_dict['queuePosition'] = queuePosition
1746             if isinstance(stage, list):
1747                 db_dict['stage'] = stage[0]
1748                 db_dict['detailed-status'] = " ".join(stage)
1749             elif stage is not None:
1750                 db_dict['stage'] = str(stage)
1751
1752             if error_message is not None:
1753                 db_dict['errorMessage'] = error_message
1754             if operation_state is not None:
1755                 db_dict['operationState'] = operation_state
1756                 db_dict["statusEnteredTime"] = time()
1757             self.update_db_2("nslcmops", op_id, db_dict)
1758         except DbException as e:
1759             self.logger.warn('Error writing OPERATION status for op_id: {} -> {}'.format(op_id, e))
1760
1761     def _write_all_config_status(self, db_nsr: dict, status: str):
1762         try:
1763             nsr_id = db_nsr["_id"]
1764             # configurationStatus
1765             config_status = db_nsr.get('configurationStatus')
1766             if config_status:
1767                 db_nsr_update = {"configurationStatus.{}.status".format(index): status for index, v in
1768                                  enumerate(config_status) if v}
1769                 # update status
1770                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1771
1772         except DbException as e:
1773             self.logger.warn('Error writing all configuration status, ns={}: {}'.format(nsr_id, e))
1774
1775     def _write_configuration_status(self, nsr_id: str, vca_index: int, status: str = None,
1776                                     element_under_configuration: str = None, element_type: str = None,
1777                                     other_update: dict = None):
1778
1779         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
1780         #                   .format(vca_index, status))
1781
1782         try:
1783             db_path = 'configurationStatus.{}.'.format(vca_index)
1784             db_dict = other_update or {}
1785             if status:
1786                 db_dict[db_path + 'status'] = status
1787             if element_under_configuration:
1788                 db_dict[db_path + 'elementUnderConfiguration'] = element_under_configuration
1789             if element_type:
1790                 db_dict[db_path + 'elementType'] = element_type
1791             self.update_db_2("nsrs", nsr_id, db_dict)
1792         except DbException as e:
1793             self.logger.warn('Error writing configuration status={}, ns={}, vca_index={}: {}'
1794                              .format(status, nsr_id, vca_index, e))
1795
1796     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
1797         """
1798         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
1799         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
1800         Database is used because the result can be obtained from a different LCM worker in case of HA.
1801         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
1802         :param db_nslcmop: database content of nslcmop
1803         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
1804         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
1805             computed 'vim-account-id'
1806         """
1807         modified = False
1808         nslcmop_id = db_nslcmop['_id']
1809         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
1810         if placement_engine == "PLA":
1811             self.logger.debug(logging_text + "Invoke and wait for placement optimization")
1812             await self.msg.aiowrite("pla", "get_placement", {'nslcmopId': nslcmop_id}, loop=self.loop)
1813             db_poll_interval = 5
1814             wait = db_poll_interval * 10
1815             pla_result = None
1816             while not pla_result and wait >= 0:
1817                 await asyncio.sleep(db_poll_interval)
1818                 wait -= db_poll_interval
1819                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1820                 pla_result = deep_get(db_nslcmop, ('_admin', 'pla'))
1821
1822             if not pla_result:
1823                 raise LcmException("Placement timeout for nslcmopId={}".format(nslcmop_id))
1824
1825             for pla_vnf in pla_result['vnf']:
1826                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
1827                 if not pla_vnf.get('vimAccountId') or not vnfr:
1828                     continue
1829                 modified = True
1830                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})
1831                 # Modifies db_vnfrs
1832                 vnfr["vim-account-id"] = pla_vnf['vimAccountId']
1833         return modified
1834
1835     def update_nsrs_with_pla_result(self, params):
1836         try:
1837             nslcmop_id = deep_get(params, ('placement', 'nslcmopId'))
1838             self.update_db_2("nslcmops", nslcmop_id, {"_admin.pla": params.get('placement')})
1839         except Exception as e:
1840             self.logger.warn('Update failed for nslcmop_id={}:{}'.format(nslcmop_id, e))
1841
1842     async def instantiate(self, nsr_id, nslcmop_id):
1843         """
1844
1845         :param nsr_id: ns instance to deploy
1846         :param nslcmop_id: operation to run
1847         :return:
1848         """
1849
1850         # Try to lock HA task here
1851         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1852         if not task_is_locked_by_me:
1853             self.logger.debug('instantiate() task is not locked by me, ns={}'.format(nsr_id))
1854             return
1855
1856         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
1857         self.logger.debug(logging_text + "Enter")
1858
1859         # get all needed from database
1860
1861         # database nsrs record
1862         db_nsr = None
1863
1864         # database nslcmops record
1865         db_nslcmop = None
1866
1867         # update operation on nsrs
1868         db_nsr_update = {}
1869         # update operation on nslcmops
1870         db_nslcmop_update = {}
1871
1872         nslcmop_operation_state = None
1873         db_vnfrs = {}     # vnf's info indexed by member-index
1874         # n2vc_info = {}
1875         tasks_dict_info = {}  # from task to info text
1876         exc = None
1877         error_list = []
1878         stage = ['Stage 1/5: preparation of the environment.', "Waiting for previous operations to terminate.", ""]
1879         # ^ stage, step, VIM progress
1880         try:
1881             # wait for any previous tasks in process
1882             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
1883
1884             stage[1] = "Sync filesystem from database."
1885             self.fs.sync()  # TODO, make use of partial sync, only for the needed packages
1886
1887             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
1888             stage[1] = "Reading from database."
1889             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
1890             db_nsr_update["detailed-status"] = "creating"
1891             db_nsr_update["operational-status"] = "init"
1892             self._write_ns_status(
1893                 nsr_id=nsr_id,
1894                 ns_state="BUILDING",
1895                 current_operation="INSTANTIATING",
1896                 current_operation_id=nslcmop_id,
1897                 other_update=db_nsr_update
1898             )
1899             self._write_op_status(
1900                 op_id=nslcmop_id,
1901                 stage=stage,
1902                 queuePosition=0
1903             )
1904
1905             # read from db: operation
1906             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
1907             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1908             ns_params = db_nslcmop.get("operationParams")
1909             if ns_params and ns_params.get("timeout_ns_deploy"):
1910                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1911             else:
1912                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
1913
1914             # read from db: ns
1915             stage[1] = "Getting nsr={} from db.".format(nsr_id)
1916             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1917             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
1918             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1919             db_nsr["nsd"] = nsd
1920             # nsr_name = db_nsr["name"]   # TODO short-name??
1921
1922             # read from db: vnf's of this ns
1923             stage[1] = "Getting vnfrs from db."
1924             self.logger.debug(logging_text + stage[1])
1925             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1926
1927             # read from db: vnfd's for every vnf
1928             db_vnfds_ref = {}     # every vnfd data indexed by vnf name
1929             db_vnfds = {}         # every vnfd data indexed by vnf id
1930             db_vnfds_index = {}   # every vnfd data indexed by vnf member-index
1931
1932             # for each vnf in ns, read vnfd
1933             for vnfr in db_vnfrs_list:
1934                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr   # vnf's dict indexed by member-index: '1', '2', etc
1935                 vnfd_id = vnfr["vnfd-id"]                       # vnfd uuid for this vnf
1936                 vnfd_ref = vnfr["vnfd-ref"]                     # vnfd name for this vnf
1937
1938                 # if we haven't this vnfd, read it from db
1939                 if vnfd_id not in db_vnfds:
1940                     # read from db
1941                     stage[1] = "Getting vnfd={} id='{}' from db.".format(vnfd_id, vnfd_ref)
1942                     self.logger.debug(logging_text + stage[1])
1943                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1944
1945                     # store vnfd
1946                     db_vnfds_ref[vnfd_ref] = vnfd     # vnfd's indexed by name
1947                     db_vnfds[vnfd_id] = vnfd          # vnfd's indexed by id
1948                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]  # vnfd's indexed by member-index
1949
1950             # Get or generates the _admin.deployed.VCA list
1951             vca_deployed_list = None
1952             if db_nsr["_admin"].get("deployed"):
1953                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
1954             if vca_deployed_list is None:
1955                 vca_deployed_list = []
1956                 configuration_status_list = []
1957                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1958                 db_nsr_update["configurationStatus"] = configuration_status_list
1959                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
1960                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1961             elif isinstance(vca_deployed_list, dict):
1962                 # maintain backward compatibility. Change a dict to list at database
1963                 vca_deployed_list = list(vca_deployed_list.values())
1964                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1965                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1966
1967             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
1968                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
1969                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
1970
1971             # set state to INSTANTIATED. When instantiated NBI will not delete directly
1972             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1973             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1974             self.db.set_list("vnfrs", {"nsr-id-ref": nsr_id}, {"_admin.nsState": "INSTANTIATED"})
1975
1976             # n2vc_redesign STEP 2 Deploy Network Scenario
1977             stage[0] = 'Stage 2/5: deployment of KDUs, VMs and execution environments.'
1978             self._write_op_status(
1979                 op_id=nslcmop_id,
1980                 stage=stage
1981             )
1982
1983             stage[1] = "Deploying KDUs."
1984             # self.logger.debug(logging_text + "Before deploy_kdus")
1985             # Call to deploy_kdus in case exists the "vdu:kdu" param
1986             await self.deploy_kdus(
1987                 logging_text=logging_text,
1988                 nsr_id=nsr_id,
1989                 nslcmop_id=nslcmop_id,
1990                 db_vnfrs=db_vnfrs,
1991                 db_vnfds=db_vnfds,
1992                 task_instantiation_info=tasks_dict_info,
1993             )
1994
1995             stage[1] = "Getting VCA public key."
1996             # n2vc_redesign STEP 1 Get VCA public ssh-key
1997             # feature 1429. Add n2vc public key to needed VMs
1998             n2vc_key = self.n2vc.get_public_key()
1999             n2vc_key_list = [n2vc_key]
2000             if self.vca_config.get("public_key"):
2001                 n2vc_key_list.append(self.vca_config["public_key"])
2002
2003             stage[1] = "Deploying NS at VIM."
2004             task_ro = asyncio.ensure_future(
2005                 self.instantiate_RO(
2006                     logging_text=logging_text,
2007                     nsr_id=nsr_id,
2008                     nsd=nsd,
2009                     db_nsr=db_nsr,
2010                     db_nslcmop=db_nslcmop,
2011                     db_vnfrs=db_vnfrs,
2012                     db_vnfds_ref=db_vnfds_ref,
2013                     n2vc_key_list=n2vc_key_list,
2014                     stage=stage
2015                 )
2016             )
2017             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
2018             tasks_dict_info[task_ro] = "Deploying at VIM"
2019
2020             # n2vc_redesign STEP 3 to 6 Deploy N2VC
2021             stage[1] = "Deploying Execution Environments."
2022             self.logger.debug(logging_text + stage[1])
2023
2024             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
2025             # get_iterable() returns a value from a dict or empty tuple if key does not exist
2026             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
2027                 vnfd_id = c_vnf["vnfd-id-ref"]
2028                 vnfd = db_vnfds_ref[vnfd_id]
2029                 member_vnf_index = str(c_vnf["member-vnf-index"])
2030                 db_vnfr = db_vnfrs[member_vnf_index]
2031                 base_folder = vnfd["_admin"]["storage"]
2032                 vdu_id = None
2033                 vdu_index = 0
2034                 vdu_name = None
2035                 kdu_name = None
2036
2037                 # Get additional parameters
2038                 deploy_params = {}
2039                 if db_vnfr.get("additionalParamsForVnf"):
2040                     deploy_params = self._format_additional_params(db_vnfr["additionalParamsForVnf"].copy())
2041
2042                 descriptor_config = vnfd.get("vnf-configuration")
2043                 if descriptor_config:
2044                     self._deploy_n2vc(
2045                         logging_text=logging_text + "member_vnf_index={} ".format(member_vnf_index),
2046                         db_nsr=db_nsr,
2047                         db_vnfr=db_vnfr,
2048                         nslcmop_id=nslcmop_id,
2049                         nsr_id=nsr_id,
2050                         nsi_id=nsi_id,
2051                         vnfd_id=vnfd_id,
2052                         vdu_id=vdu_id,
2053                         kdu_name=kdu_name,
2054                         member_vnf_index=member_vnf_index,
2055                         vdu_index=vdu_index,
2056                         vdu_name=vdu_name,
2057                         deploy_params=deploy_params,
2058                         descriptor_config=descriptor_config,
2059                         base_folder=base_folder,
2060                         task_instantiation_info=tasks_dict_info,
2061                         stage=stage
2062                     )
2063
2064                 # Deploy charms for each VDU that supports one.
2065                 for vdud in get_iterable(vnfd, 'vdu'):
2066                     vdu_id = vdud["id"]
2067                     descriptor_config = vdud.get('vdu-configuration')
2068                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
2069                     if vdur.get("additionalParams"):
2070                         deploy_params_vdu = self._format_additional_params(vdur["additionalParams"])
2071                     else:
2072                         deploy_params_vdu = deploy_params
2073                     if descriptor_config:
2074                         # look for vdu index in the db_vnfr["vdu"] section
2075                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
2076                         #     if vdur["vdu-id-ref"] == vdu_id:
2077                         #         break
2078                         # else:
2079                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
2080                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
2081                         # vdu_name = vdur.get("name")
2082                         vdu_name = None
2083                         kdu_name = None
2084                         for vdu_index in range(int(vdud.get("count", 1))):
2085                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
2086                             self._deploy_n2vc(
2087                                 logging_text=logging_text + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
2088                                     member_vnf_index, vdu_id, vdu_index),
2089                                 db_nsr=db_nsr,
2090                                 db_vnfr=db_vnfr,
2091                                 nslcmop_id=nslcmop_id,
2092                                 nsr_id=nsr_id,
2093                                 nsi_id=nsi_id,
2094                                 vnfd_id=vnfd_id,
2095                                 vdu_id=vdu_id,
2096                                 kdu_name=kdu_name,
2097                                 member_vnf_index=member_vnf_index,
2098                                 vdu_index=vdu_index,
2099                                 vdu_name=vdu_name,
2100                                 deploy_params=deploy_params_vdu,
2101                                 descriptor_config=descriptor_config,
2102                                 base_folder=base_folder,
2103                                 task_instantiation_info=tasks_dict_info,
2104                                 stage=stage
2105                             )
2106                 for kdud in get_iterable(vnfd, 'kdu'):
2107                     kdu_name = kdud["name"]
2108                     descriptor_config = kdud.get('kdu-configuration')
2109                     if descriptor_config:
2110                         vdu_id = None
2111                         vdu_index = 0
2112                         vdu_name = None
2113                         # look for vdu index in the db_vnfr["vdu"] section
2114                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
2115                         #     if vdur["vdu-id-ref"] == vdu_id:
2116                         #         break
2117                         # else:
2118                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
2119                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
2120                         # vdu_name = vdur.get("name")
2121                         # vdu_name = None
2122
2123                         self._deploy_n2vc(
2124                             logging_text=logging_text,
2125                             db_nsr=db_nsr,
2126                             db_vnfr=db_vnfr,
2127                             nslcmop_id=nslcmop_id,
2128                             nsr_id=nsr_id,
2129                             nsi_id=nsi_id,
2130                             vnfd_id=vnfd_id,
2131                             vdu_id=vdu_id,
2132                             kdu_name=kdu_name,
2133                             member_vnf_index=member_vnf_index,
2134                             vdu_index=vdu_index,
2135                             vdu_name=vdu_name,
2136                             deploy_params=deploy_params,
2137                             descriptor_config=descriptor_config,
2138                             base_folder=base_folder,
2139                             task_instantiation_info=tasks_dict_info,
2140                             stage=stage
2141                         )
2142
2143             # Check if this NS has a charm configuration
2144             descriptor_config = nsd.get("ns-configuration")
2145             if descriptor_config and descriptor_config.get("juju"):
2146                 vnfd_id = None
2147                 db_vnfr = None
2148                 member_vnf_index = None
2149                 vdu_id = None
2150                 kdu_name = None
2151                 vdu_index = 0
2152                 vdu_name = None
2153
2154                 # Get additional parameters
2155                 deploy_params = {}
2156                 if db_nsr.get("additionalParamsForNs"):
2157                     deploy_params = self._format_additional_params(db_nsr["additionalParamsForNs"].copy())
2158                 base_folder = nsd["_admin"]["storage"]
2159                 self._deploy_n2vc(
2160                     logging_text=logging_text,
2161                     db_nsr=db_nsr,
2162                     db_vnfr=db_vnfr,
2163                     nslcmop_id=nslcmop_id,
2164                     nsr_id=nsr_id,
2165                     nsi_id=nsi_id,
2166                     vnfd_id=vnfd_id,
2167                     vdu_id=vdu_id,
2168                     kdu_name=kdu_name,
2169                     member_vnf_index=member_vnf_index,
2170                     vdu_index=vdu_index,
2171                     vdu_name=vdu_name,
2172                     deploy_params=deploy_params,
2173                     descriptor_config=descriptor_config,
2174                     base_folder=base_folder,
2175                     task_instantiation_info=tasks_dict_info,
2176                     stage=stage
2177                 )
2178
2179             # rest of staff will be done at finally
2180
2181         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
2182             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(stage[1], e))
2183             exc = e
2184         except asyncio.CancelledError:
2185             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
2186             exc = "Operation was cancelled"
2187         except Exception as e:
2188             exc = traceback.format_exc()
2189             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
2190         finally:
2191             if exc:
2192                 error_list.append(str(exc))
2193             try:
2194                 # wait for pending tasks
2195                 if tasks_dict_info:
2196                     stage[1] = "Waiting for instantiate pending tasks."
2197                     self.logger.debug(logging_text + stage[1])
2198                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_deploy,
2199                                                              stage, nslcmop_id, nsr_id=nsr_id)
2200                 stage[1] = stage[2] = ""
2201             except asyncio.CancelledError:
2202                 error_list.append("Cancelled")
2203                 # TODO cancel all tasks
2204             except Exception as exc:
2205                 error_list.append(str(exc))
2206
2207             # update operation-status
2208             db_nsr_update["operational-status"] = "running"
2209             # let's begin with VCA 'configured' status (later we can change it)
2210             db_nsr_update["config-status"] = "configured"
2211             for task, task_name in tasks_dict_info.items():
2212                 if not task.done() or task.cancelled() or task.exception():
2213                     if task_name.startswith(self.task_name_deploy_vca):
2214                         # A N2VC task is pending
2215                         db_nsr_update["config-status"] = "failed"
2216                     else:
2217                         # RO or KDU task is pending
2218                         db_nsr_update["operational-status"] = "failed"
2219
2220             # update status at database
2221             if error_list:
2222                 error_detail = ". ".join(error_list)
2223                 self.logger.error(logging_text + error_detail)
2224                 error_description_nslcmop = '{} Detail: {}'.format(stage[0], error_detail)
2225                 error_description_nsr = 'Operation: INSTANTIATING.{}, {}'.format(nslcmop_id, stage[0])
2226
2227                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
2228                 db_nslcmop_update["detailed-status"] = error_detail
2229                 nslcmop_operation_state = "FAILED"
2230                 ns_state = "BROKEN"
2231             else:
2232                 error_detail = None
2233                 error_description_nsr = error_description_nslcmop = None
2234                 ns_state = "READY"
2235                 db_nsr_update["detailed-status"] = "Done"
2236                 db_nslcmop_update["detailed-status"] = "Done"
2237                 nslcmop_operation_state = "COMPLETED"
2238
2239             if db_nsr:
2240                 self._write_ns_status(
2241                     nsr_id=nsr_id,
2242                     ns_state=ns_state,
2243                     current_operation="IDLE",
2244                     current_operation_id=None,
2245                     error_description=error_description_nsr,
2246                     error_detail=error_detail,
2247                     other_update=db_nsr_update
2248                 )
2249             self._write_op_status(
2250                 op_id=nslcmop_id,
2251                 stage="",
2252                 error_message=error_description_nslcmop,
2253                 operation_state=nslcmop_operation_state,
2254                 other_update=db_nslcmop_update,
2255             )
2256
2257             if nslcmop_operation_state:
2258                 try:
2259                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2260                                                                    "operationState": nslcmop_operation_state},
2261                                             loop=self.loop)
2262                 except Exception as e:
2263                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2264
2265             self.logger.debug(logging_text + "Exit")
2266             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
2267
2268     async def _add_vca_relations(self, logging_text, nsr_id, vca_index: int,
2269                                  timeout: int = 3600, vca_type: str = None) -> bool:
2270
2271         # steps:
2272         # 1. find all relations for this VCA
2273         # 2. wait for other peers related
2274         # 3. add relations
2275
2276         try:
2277             vca_type = vca_type or "lxc_proxy_charm"
2278
2279             # STEP 1: find all relations for this VCA
2280
2281             # read nsr record
2282             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2283             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2284
2285             # this VCA data
2286             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
2287
2288             # read all ns-configuration relations
2289             ns_relations = list()
2290             db_ns_relations = deep_get(nsd, ('ns-configuration', 'relation'))
2291             if db_ns_relations:
2292                 for r in db_ns_relations:
2293                     # check if this VCA is in the relation
2294                     if my_vca.get('member-vnf-index') in\
2295                             (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
2296                         ns_relations.append(r)
2297
2298             # read all vnf-configuration relations
2299             vnf_relations = list()
2300             db_vnfd_list = db_nsr.get('vnfd-id')
2301             if db_vnfd_list:
2302                 for vnfd in db_vnfd_list:
2303                     db_vnfd = self.db.get_one("vnfds", {"_id": vnfd})
2304                     db_vnf_relations = deep_get(db_vnfd, ('vnf-configuration', 'relation'))
2305                     if db_vnf_relations:
2306                         for r in db_vnf_relations:
2307                             # check if this VCA is in the relation
2308                             if my_vca.get('vdu_id') in (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
2309                                 vnf_relations.append(r)
2310
2311             # if no relations, terminate
2312             if not ns_relations and not vnf_relations:
2313                 self.logger.debug(logging_text + ' No relations')
2314                 return True
2315
2316             self.logger.debug(logging_text + ' adding relations\n    {}\n    {}'.format(ns_relations, vnf_relations))
2317
2318             # add all relations
2319             start = time()
2320             while True:
2321                 # check timeout
2322                 now = time()
2323                 if now - start >= timeout:
2324                     self.logger.error(logging_text + ' : timeout adding relations')
2325                     return False
2326
2327                 # reload nsr from database (we need to update record: _admin.deloyed.VCA)
2328                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2329
2330                 # for each defined NS relation, find the VCA's related
2331                 for r in ns_relations.copy():
2332                     from_vca_ee_id = None
2333                     to_vca_ee_id = None
2334                     from_vca_endpoint = None
2335                     to_vca_endpoint = None
2336                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2337                     for vca in vca_list:
2338                         if vca.get('member-vnf-index') == r.get('entities')[0].get('id') \
2339                                 and vca.get('config_sw_installed'):
2340                             from_vca_ee_id = vca.get('ee_id')
2341                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2342                         if vca.get('member-vnf-index') == r.get('entities')[1].get('id') \
2343                                 and vca.get('config_sw_installed'):
2344                             to_vca_ee_id = vca.get('ee_id')
2345                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2346                     if from_vca_ee_id and to_vca_ee_id:
2347                         # add relation
2348                         await self.vca_map[vca_type].add_relation(
2349                             ee_id_1=from_vca_ee_id,
2350                             ee_id_2=to_vca_ee_id,
2351                             endpoint_1=from_vca_endpoint,
2352                             endpoint_2=to_vca_endpoint)
2353                         # remove entry from relations list
2354                         ns_relations.remove(r)
2355                     else:
2356                         # check failed peers
2357                         try:
2358                             vca_status_list = db_nsr.get('configurationStatus')
2359                             if vca_status_list:
2360                                 for i in range(len(vca_list)):
2361                                     vca = vca_list[i]
2362                                     vca_status = vca_status_list[i]
2363                                     if vca.get('member-vnf-index') == r.get('entities')[0].get('id'):
2364                                         if vca_status.get('status') == 'BROKEN':
2365                                             # peer broken: remove relation from list
2366                                             ns_relations.remove(r)
2367                                     if vca.get('member-vnf-index') == r.get('entities')[1].get('id'):
2368                                         if vca_status.get('status') == 'BROKEN':
2369                                             # peer broken: remove relation from list
2370                                             ns_relations.remove(r)
2371                         except Exception:
2372                             # ignore
2373                             pass
2374
2375                 # for each defined VNF relation, find the VCA's related
2376                 for r in vnf_relations.copy():
2377                     from_vca_ee_id = None
2378                     to_vca_ee_id = None
2379                     from_vca_endpoint = None
2380                     to_vca_endpoint = None
2381                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2382                     for vca in vca_list:
2383                         if vca.get('vdu_id') == r.get('entities')[0].get('id') and vca.get('config_sw_installed'):
2384                             from_vca_ee_id = vca.get('ee_id')
2385                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2386                         if vca.get('vdu_id') == r.get('entities')[1].get('id') and vca.get('config_sw_installed'):
2387                             to_vca_ee_id = vca.get('ee_id')
2388                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2389                     if from_vca_ee_id and to_vca_ee_id:
2390                         # add relation
2391                         await self.vca_map[vca_type].add_relation(
2392                             ee_id_1=from_vca_ee_id,
2393                             ee_id_2=to_vca_ee_id,
2394                             endpoint_1=from_vca_endpoint,
2395                             endpoint_2=to_vca_endpoint)
2396                         # remove entry from relations list
2397                         vnf_relations.remove(r)
2398                     else:
2399                         # check failed peers
2400                         try:
2401                             vca_status_list = db_nsr.get('configurationStatus')
2402                             if vca_status_list:
2403                                 for i in range(len(vca_list)):
2404                                     vca = vca_list[i]
2405                                     vca_status = vca_status_list[i]
2406                                     if vca.get('vdu_id') == r.get('entities')[0].get('id'):
2407                                         if vca_status.get('status') == 'BROKEN':
2408                                             # peer broken: remove relation from list
2409                                             vnf_relations.remove(r)
2410                                     if vca.get('vdu_id') == r.get('entities')[1].get('id'):
2411                                         if vca_status.get('status') == 'BROKEN':
2412                                             # peer broken: remove relation from list
2413                                             vnf_relations.remove(r)
2414                         except Exception:
2415                             # ignore
2416                             pass
2417
2418                 # wait for next try
2419                 await asyncio.sleep(5.0)
2420
2421                 if not ns_relations and not vnf_relations:
2422                     self.logger.debug('Relations added')
2423                     break
2424
2425             return True
2426
2427         except Exception as e:
2428             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
2429             return False
2430
2431     async def _install_kdu(self, nsr_id: str, nsr_db_path: str, vnfr_data: dict, kdu_index: int, kdud: dict,
2432                            vnfd: dict, k8s_instance_info: dict, k8params: dict = None, timeout: int = 600):
2433
2434         try:
2435             k8sclustertype = k8s_instance_info["k8scluster-type"]
2436             # Instantiate kdu
2437             db_dict_install = {"collection": "nsrs",
2438                                "filter": {"_id": nsr_id},
2439                                "path": nsr_db_path}
2440
2441             kdu_instance = await self.k8scluster_map[k8sclustertype].install(
2442                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
2443                 kdu_model=k8s_instance_info["kdu-model"],
2444                 atomic=True,
2445                 params=k8params,
2446                 db_dict=db_dict_install,
2447                 timeout=timeout,
2448                 kdu_name=k8s_instance_info["kdu-name"],
2449                 namespace=k8s_instance_info["namespace"])
2450             self.update_db_2("nsrs", nsr_id, {nsr_db_path + ".kdu-instance": kdu_instance})
2451
2452             # Obtain services to obtain management service ip
2453             services = await self.k8scluster_map[k8sclustertype].get_services(
2454                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
2455                 kdu_instance=kdu_instance,
2456                 namespace=k8s_instance_info["namespace"])
2457
2458             # Obtain management service info (if exists)
2459             vnfr_update_dict = {}
2460             if services:
2461                 vnfr_update_dict["kdur.{}.services".format(kdu_index)] = services
2462                 mgmt_services = [service for service in kdud.get("service", []) if service.get("mgmt-service")]
2463                 for mgmt_service in mgmt_services:
2464                     for service in services:
2465                         if service["name"].startswith(mgmt_service["name"]):
2466                             # Mgmt service found, Obtain service ip
2467                             ip = service.get("external_ip", service.get("cluster_ip"))
2468                             if isinstance(ip, list) and len(ip) == 1:
2469                                 ip = ip[0]
2470
2471                             vnfr_update_dict["kdur.{}.ip-address".format(kdu_index)] = ip
2472
2473                             # Check if must update also mgmt ip at the vnf
2474                             service_external_cp = mgmt_service.get("external-connection-point-ref")
2475                             if service_external_cp:
2476                                 if deep_get(vnfd, ("mgmt-interface", "cp")) == service_external_cp:
2477                                     vnfr_update_dict["ip-address"] = ip
2478
2479                             break
2480                     else:
2481                         self.logger.warn("Mgmt service name: {} not found".format(mgmt_service["name"]))
2482
2483             vnfr_update_dict["kdur.{}.status".format(kdu_index)] = "READY"
2484             self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
2485
2486             kdu_config = kdud.get("kdu-configuration")
2487             if kdu_config and kdu_config.get("initial-config-primitive") and kdu_config.get("juju") is None:
2488                 initial_config_primitive_list = kdu_config.get("initial-config-primitive")
2489                 initial_config_primitive_list.sort(key=lambda val: int(val["seq"]))
2490
2491                 for initial_config_primitive in initial_config_primitive_list:
2492                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, {})
2493
2494                     await asyncio.wait_for(
2495                         self.k8scluster_map[k8sclustertype].exec_primitive(
2496                             cluster_uuid=k8s_instance_info["k8scluster-uuid"],
2497                             kdu_instance=kdu_instance,
2498                             primitive_name=initial_config_primitive["name"],
2499                             params=primitive_params_, db_dict={}),
2500                         timeout=timeout)
2501
2502         except Exception as e:
2503             # Prepare update db with error and raise exception
2504             try:
2505                 self.update_db_2("nsrs", nsr_id, {nsr_db_path + ".detailed-status": str(e)})
2506                 self.update_db_2("vnfrs", vnfr_data.get("_id"), {"kdur.{}.status".format(kdu_index): "ERROR"})
2507             except Exception:
2508                 # ignore to keep original exception
2509                 pass
2510             # reraise original error
2511             raise
2512
2513         return kdu_instance
2514
2515     async def deploy_kdus(self, logging_text, nsr_id, nslcmop_id, db_vnfrs, db_vnfds, task_instantiation_info):
2516         # Launch kdus if present in the descriptor
2517
2518         k8scluster_id_2_uuic = {"helm-chart": {}, "juju-bundle": {}}
2519
2520         async def _get_cluster_id(cluster_id, cluster_type):
2521             nonlocal k8scluster_id_2_uuic
2522             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
2523                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
2524
2525             # check if K8scluster is creating and wait look if previous tasks in process
2526             task_name, task_dependency = self.lcm_tasks.lookfor_related("k8scluster", cluster_id)
2527             if task_dependency:
2528                 text = "Waiting for related tasks '{}' on k8scluster {} to be completed".format(task_name, cluster_id)
2529                 self.logger.debug(logging_text + text)
2530                 await asyncio.wait(task_dependency, timeout=3600)
2531
2532             db_k8scluster = self.db.get_one("k8sclusters", {"_id": cluster_id}, fail_on_empty=False)
2533             if not db_k8scluster:
2534                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
2535
2536             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
2537             if not k8s_id:
2538                 raise LcmException("K8s cluster '{}' has not been initialized for '{}'".format(cluster_id,
2539                                                                                                cluster_type))
2540             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
2541             return k8s_id
2542
2543         logging_text += "Deploy kdus: "
2544         step = ""
2545         try:
2546             db_nsr_update = {"_admin.deployed.K8s": []}
2547             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2548
2549             index = 0
2550             updated_cluster_list = []
2551
2552             for vnfr_data in db_vnfrs.values():
2553                 for kdu_index, kdur in enumerate(get_iterable(vnfr_data, "kdur")):
2554                     # Step 0: Prepare and set parameters
2555                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
2556                     vnfd_id = vnfr_data.get('vnfd-id')
2557                     kdud = next(kdud for kdud in db_vnfds[vnfd_id]["kdu"] if kdud["name"] == kdur["kdu-name"])
2558                     namespace = kdur.get("k8s-namespace")
2559                     if kdur.get("helm-chart"):
2560                         kdumodel = kdur["helm-chart"]
2561                         k8sclustertype = "helm-chart"
2562                     elif kdur.get("juju-bundle"):
2563                         kdumodel = kdur["juju-bundle"]
2564                         k8sclustertype = "juju-bundle"
2565                     else:
2566                         raise LcmException("kdu type for kdu='{}.{}' is neither helm-chart nor "
2567                                            "juju-bundle. Maybe an old NBI version is running".
2568                                            format(vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]))
2569                     # check if kdumodel is a file and exists
2570                     try:
2571                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
2572                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
2573                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
2574                             filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["pkg-dir"], k8sclustertype,
2575                                                              kdumodel)
2576                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
2577                                 kdumodel = self.fs.path + filename
2578                     except (asyncio.TimeoutError, asyncio.CancelledError):
2579                         raise
2580                     except Exception:       # it is not a file
2581                         pass
2582
2583                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
2584                     step = "Synchronize repos for k8s cluster '{}'".format(k8s_cluster_id)
2585                     cluster_uuid = await _get_cluster_id(k8s_cluster_id, k8sclustertype)
2586
2587                     # Synchronize  repos
2588                     if k8sclustertype == "helm-chart" and cluster_uuid not in updated_cluster_list:
2589                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
2590                             self.k8sclusterhelm.synchronize_repos(cluster_uuid=cluster_uuid))
2591                         if del_repo_list or added_repo_dict:
2592                             unset = {'_admin.helm_charts_added.' + item: None for item in del_repo_list}
2593                             updated = {'_admin.helm_charts_added.' +
2594                                        item: name for item, name in added_repo_dict.items()}
2595                             self.logger.debug(logging_text + "repos synchronized on k8s cluster '{}' to_delete: {}, "
2596                                                              "to_add: {}".format(k8s_cluster_id, del_repo_list,
2597                                                                                  added_repo_dict))
2598                             self.db.set_one("k8sclusters", {"_id": k8s_cluster_id}, updated, unset=unset)
2599                         updated_cluster_list.append(cluster_uuid)
2600
2601                     # Instantiate kdu
2602                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(vnfr_data["member-vnf-index-ref"],
2603                                                                               kdur["kdu-name"], k8s_cluster_id)
2604                     k8s_instance_info = {"kdu-instance": None,
2605                                          "k8scluster-uuid": cluster_uuid,
2606                                          "k8scluster-type": k8sclustertype,
2607                                          "member-vnf-index": vnfr_data["member-vnf-index-ref"],
2608                                          "kdu-name": kdur["kdu-name"],
2609                                          "kdu-model": kdumodel,
2610                                          "namespace": namespace}
2611                     db_path = "_admin.deployed.K8s.{}".format(index)
2612                     db_nsr_update[db_path] = k8s_instance_info
2613                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2614
2615                     task = asyncio.ensure_future(
2616                         self._install_kdu(nsr_id, db_path, vnfr_data, kdu_index, kdud, db_vnfds[vnfd_id],
2617                                           k8s_instance_info, k8params=desc_params, timeout=600))
2618                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
2619                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
2620
2621                     index += 1
2622
2623         except (LcmException, asyncio.CancelledError):
2624             raise
2625         except Exception as e:
2626             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
2627             if isinstance(e, (N2VCException, DbException)):
2628                 self.logger.error(logging_text + msg)
2629             else:
2630                 self.logger.critical(logging_text + msg, exc_info=True)
2631             raise LcmException(msg)
2632         finally:
2633             if db_nsr_update:
2634                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2635
2636     def _deploy_n2vc(self, logging_text, db_nsr, db_vnfr, nslcmop_id, nsr_id, nsi_id, vnfd_id, vdu_id,
2637                      kdu_name, member_vnf_index, vdu_index, vdu_name, deploy_params, descriptor_config,
2638                      base_folder, task_instantiation_info, stage):
2639         # launch instantiate_N2VC in a asyncio task and register task object
2640         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
2641         # if not found, create one entry and update database
2642         # fill db_nsr._admin.deployed.VCA.<index>
2643
2644         self.logger.debug(logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id))
2645         if descriptor_config.get("juju"):   # There is one execution envioronment of type juju
2646             ee_list = [descriptor_config]
2647         elif descriptor_config.get("execution-environment-list"):
2648             ee_list = descriptor_config.get("execution-environment-list")
2649         else:  # other types as script are not supported
2650             ee_list = []
2651
2652         for ee_item in ee_list:
2653             self.logger.debug(logging_text + "_deploy_n2vc ee_item juju={}, helm={}".format(ee_item.get('juju'),
2654                                                                                             ee_item.get("helm-chart")))
2655             ee_descriptor_id = ee_item.get("id")
2656             if ee_item.get("juju"):
2657                 vca_name = ee_item['juju'].get('charm')
2658                 vca_type = "lxc_proxy_charm" if ee_item['juju'].get('charm') is not None else "native_charm"
2659                 if ee_item['juju'].get('cloud') == "k8s":
2660                     vca_type = "k8s_proxy_charm"
2661                 elif ee_item['juju'].get('proxy') is False:
2662                     vca_type = "native_charm"
2663             elif ee_item.get("helm-chart"):
2664                 vca_name = ee_item['helm-chart']
2665                 vca_type = "helm"
2666             else:
2667                 self.logger.debug(logging_text + "skipping non juju neither charm configuration")
2668                 continue
2669
2670             vca_index = -1
2671             for vca_index, vca_deployed in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
2672                 if not vca_deployed:
2673                     continue
2674                 if vca_deployed.get("member-vnf-index") == member_vnf_index and \
2675                         vca_deployed.get("vdu_id") == vdu_id and \
2676                         vca_deployed.get("kdu_name") == kdu_name and \
2677                         vca_deployed.get("vdu_count_index", 0) == vdu_index and \
2678                         vca_deployed.get("ee_descriptor_id") == ee_descriptor_id:
2679                     break
2680             else:
2681                 # not found, create one.
2682                 target = "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
2683                 if vdu_id:
2684                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
2685                 elif kdu_name:
2686                     target += "/kdu/{}".format(kdu_name)
2687                 vca_deployed = {
2688                     "target_element": target,
2689                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
2690                     "member-vnf-index": member_vnf_index,
2691                     "vdu_id": vdu_id,
2692                     "kdu_name": kdu_name,
2693                     "vdu_count_index": vdu_index,
2694                     "operational-status": "init",  # TODO revise
2695                     "detailed-status": "",  # TODO revise
2696                     "step": "initial-deploy",   # TODO revise
2697                     "vnfd_id": vnfd_id,
2698                     "vdu_name": vdu_name,
2699                     "type": vca_type,
2700                     "ee_descriptor_id": ee_descriptor_id
2701                 }
2702                 vca_index += 1
2703
2704                 # create VCA and configurationStatus in db
2705                 db_dict = {
2706                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
2707                     "configurationStatus.{}".format(vca_index): dict()
2708                 }
2709                 self.update_db_2("nsrs", nsr_id, db_dict)
2710
2711                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
2712
2713             # Launch task
2714             task_n2vc = asyncio.ensure_future(
2715                 self.instantiate_N2VC(
2716                     logging_text=logging_text,
2717                     vca_index=vca_index,
2718                     nsi_id=nsi_id,
2719                     db_nsr=db_nsr,
2720                     db_vnfr=db_vnfr,
2721                     vdu_id=vdu_id,
2722                     kdu_name=kdu_name,
2723                     vdu_index=vdu_index,
2724                     deploy_params=deploy_params,
2725                     config_descriptor=descriptor_config,
2726                     base_folder=base_folder,
2727                     nslcmop_id=nslcmop_id,
2728                     stage=stage,
2729                     vca_type=vca_type,
2730                     vca_name=vca_name,
2731                     ee_config_descriptor=ee_item
2732                 )
2733             )
2734             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_N2VC-{}".format(vca_index), task_n2vc)
2735             task_instantiation_info[task_n2vc] = self.task_name_deploy_vca + " {}.{}".format(
2736                 member_vnf_index or "", vdu_id or "")
2737
2738     @staticmethod
2739     def _get_terminate_config_primitive(primitive_list, vca_deployed):
2740         """ Get a sorted terminate config primitive list. In case ee_descriptor_id is present at vca_deployed,
2741          it get only those primitives for this execution envirom"""
2742
2743         primitive_list = primitive_list or []
2744         # filter primitives by ee_descriptor_id
2745         ee_descriptor_id = vca_deployed.get("ee_descriptor_id")
2746         primitive_list = [p for p in primitive_list if p.get("execution-environment-ref") == ee_descriptor_id]
2747
2748         if primitive_list:
2749             primitive_list.sort(key=lambda val: int(val['seq']))
2750
2751         return primitive_list
2752
2753     @staticmethod
2754     def _create_nslcmop(nsr_id, operation, params):
2755         """
2756         Creates a ns-lcm-opp content to be stored at database.
2757         :param nsr_id: internal id of the instance
2758         :param operation: instantiate, terminate, scale, action, ...
2759         :param params: user parameters for the operation
2760         :return: dictionary following SOL005 format
2761         """
2762         # Raise exception if invalid arguments
2763         if not (nsr_id and operation and params):
2764             raise LcmException(
2765                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
2766         now = time()
2767         _id = str(uuid4())
2768         nslcmop = {
2769             "id": _id,
2770             "_id": _id,
2771             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
2772             "operationState": "PROCESSING",
2773             "statusEnteredTime": now,
2774             "nsInstanceId": nsr_id,
2775             "lcmOperationType": operation,
2776             "startTime": now,
2777             "isAutomaticInvocation": False,
2778             "operationParams": params,
2779             "isCancelPending": False,
2780             "links": {
2781                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
2782                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
2783             }
2784         }
2785         return nslcmop
2786
2787     def _format_additional_params(self, params):
2788         params = params or {}
2789         for key, value in params.items():
2790             if str(value).startswith("!!yaml "):
2791                 params[key] = yaml.safe_load(value[7:])
2792         return params
2793
2794     def _get_terminate_primitive_params(self, seq, vnf_index):
2795         primitive = seq.get('name')
2796         primitive_params = {}
2797         params = {
2798             "member_vnf_index": vnf_index,
2799             "primitive": primitive,
2800             "primitive_params": primitive_params,
2801         }
2802         desc_params = {}
2803         return self._map_primitive_params(seq, params, desc_params)
2804
2805     # sub-operations
2806
2807     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
2808         op = deep_get(db_nslcmop, ('_admin', 'operations'), [])[op_index]
2809         if op.get('operationState') == 'COMPLETED':
2810             # b. Skip sub-operation
2811             # _ns_execute_primitive() or RO.create_action() will NOT be executed
2812             return self.SUBOPERATION_STATUS_SKIP
2813         else:
2814             # c. retry executing sub-operation
2815             # The sub-operation exists, and operationState != 'COMPLETED'
2816             # Update operationState = 'PROCESSING' to indicate a retry.
2817             operationState = 'PROCESSING'
2818             detailed_status = 'In progress'
2819             self._update_suboperation_status(
2820                 db_nslcmop, op_index, operationState, detailed_status)
2821             # Return the sub-operation index
2822             # _ns_execute_primitive() or RO.create_action() will be called from scale()
2823             # with arguments extracted from the sub-operation
2824             return op_index
2825
2826     # Find a sub-operation where all keys in a matching dictionary must match
2827     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
2828     def _find_suboperation(self, db_nslcmop, match):
2829         if db_nslcmop and match:
2830             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
2831             for i, op in enumerate(op_list):
2832                 if all(op.get(k) == match[k] for k in match):
2833                     return i
2834         return self.SUBOPERATION_STATUS_NOT_FOUND
2835
2836     # Update status for a sub-operation given its index
2837     def _update_suboperation_status(self, db_nslcmop, op_index, operationState, detailed_status):
2838         # Update DB for HA tasks
2839         q_filter = {'_id': db_nslcmop['_id']}
2840         update_dict = {'_admin.operations.{}.operationState'.format(op_index): operationState,
2841                        '_admin.operations.{}.detailed-status'.format(op_index): detailed_status}
2842         self.db.set_one("nslcmops",
2843                         q_filter=q_filter,
2844                         update_dict=update_dict,
2845                         fail_on_empty=False)
2846
2847     # Add sub-operation, return the index of the added sub-operation
2848     # Optionally, set operationState, detailed-status, and operationType
2849     # Status and type are currently set for 'scale' sub-operations:
2850     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
2851     # 'detailed-status' : status message
2852     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
2853     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
2854     def _add_suboperation(self, db_nslcmop, vnf_index, vdu_id, vdu_count_index, vdu_name, primitive,
2855                           mapped_primitive_params, operationState=None, detailed_status=None, operationType=None,
2856                           RO_nsr_id=None, RO_scaling_info=None):
2857         if not db_nslcmop:
2858             return self.SUBOPERATION_STATUS_NOT_FOUND
2859         # Get the "_admin.operations" list, if it exists
2860         db_nslcmop_admin = db_nslcmop.get('_admin', {})
2861         op_list = db_nslcmop_admin.get('operations')
2862         # Create or append to the "_admin.operations" list
2863         new_op = {'member_vnf_index': vnf_index,
2864                   'vdu_id': vdu_id,
2865                   'vdu_count_index': vdu_count_index,
2866                   'primitive': primitive,
2867                   'primitive_params': mapped_primitive_params}
2868         if operationState:
2869             new_op['operationState'] = operationState
2870         if detailed_status:
2871             new_op['detailed-status'] = detailed_status
2872         if operationType:
2873             new_op['lcmOperationType'] = operationType
2874         if RO_nsr_id:
2875             new_op['RO_nsr_id'] = RO_nsr_id
2876         if RO_scaling_info:
2877             new_op['RO_scaling_info'] = RO_scaling_info
2878         if not op_list:
2879             # No existing operations, create key 'operations' with current operation as first list element
2880             db_nslcmop_admin.update({'operations': [new_op]})
2881             op_list = db_nslcmop_admin.get('operations')
2882         else:
2883             # Existing operations, append operation to list
2884             op_list.append(new_op)
2885
2886         db_nslcmop_update = {'_admin.operations': op_list}
2887         self.update_db_2("nslcmops", db_nslcmop['_id'], db_nslcmop_update)
2888         op_index = len(op_list) - 1
2889         return op_index
2890
2891     # Helper methods for scale() sub-operations
2892
2893     # pre-scale/post-scale:
2894     # Check for 3 different cases:
2895     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
2896     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
2897     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
2898     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
2899                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
2900         # Find this sub-operation
2901         if RO_nsr_id and RO_scaling_info:
2902             operationType = 'SCALE-RO'
2903             match = {
2904                 'member_vnf_index': vnf_index,
2905                 'RO_nsr_id': RO_nsr_id,
2906                 'RO_scaling_info': RO_scaling_info,
2907             }
2908         else:
2909             match = {
2910                 'member_vnf_index': vnf_index,
2911                 'primitive': vnf_config_primitive,
2912                 'primitive_params': primitive_params,
2913                 'lcmOperationType': operationType
2914             }
2915         op_index = self._find_suboperation(db_nslcmop, match)
2916         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
2917             # a. New sub-operation
2918             # The sub-operation does not exist, add it.
2919             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
2920             # The following parameters are set to None for all kind of scaling:
2921             vdu_id = None
2922             vdu_count_index = None
2923             vdu_name = None
2924             if RO_nsr_id and RO_scaling_info:
2925                 vnf_config_primitive = None
2926                 primitive_params = None
2927             else:
2928                 RO_nsr_id = None
2929                 RO_scaling_info = None
2930             # Initial status for sub-operation
2931             operationState = 'PROCESSING'
2932             detailed_status = 'In progress'
2933             # Add sub-operation for pre/post-scaling (zero or more operations)
2934             self._add_suboperation(db_nslcmop,
2935                                    vnf_index,
2936                                    vdu_id,
2937                                    vdu_count_index,
2938                                    vdu_name,
2939                                    vnf_config_primitive,
2940                                    primitive_params,
2941                                    operationState,
2942                                    detailed_status,
2943                                    operationType,
2944                                    RO_nsr_id,
2945                                    RO_scaling_info)
2946             return self.SUBOPERATION_STATUS_NEW
2947         else:
2948             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
2949             # or op_index (operationState != 'COMPLETED')
2950             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
2951
2952     # Function to return execution_environment id
2953
2954     def _get_ee_id(self, vnf_index, vdu_id, vca_deployed_list):
2955         # TODO vdu_index_count
2956         for vca in vca_deployed_list:
2957             if vca["member-vnf-index"] == vnf_index and vca["vdu_id"] == vdu_id:
2958                 return vca["ee_id"]
2959
2960     async def destroy_N2VC(self, logging_text, db_nslcmop, vca_deployed, config_descriptor,
2961                            vca_index, destroy_ee=True, exec_primitives=True):
2962         """
2963         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
2964         :param logging_text:
2965         :param db_nslcmop:
2966         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
2967         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
2968         :param vca_index: index in the database _admin.deployed.VCA
2969         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
2970         :param exec_primitives: False to do not execute terminate primitives, because the config is not completed or has
2971                             not executed properly
2972         :return: None or exception
2973         """
2974
2975         self.logger.debug(
2976             logging_text + " vca_index: {}, vca_deployed: {}, config_descriptor: {}, destroy_ee: {}".format(
2977                 vca_index, vca_deployed, config_descriptor, destroy_ee
2978             )
2979         )
2980
2981         vca_type = vca_deployed.get("type", "lxc_proxy_charm")
2982
2983         # execute terminate_primitives
2984         if exec_primitives:
2985             terminate_primitives = self._get_terminate_config_primitive(
2986                 config_descriptor.get("terminate-config-primitive"), vca_deployed)
2987             vdu_id = vca_deployed.get("vdu_id")
2988             vdu_count_index = vca_deployed.get("vdu_count_index")
2989             vdu_name = vca_deployed.get("vdu_name")
2990             vnf_index = vca_deployed.get("member-vnf-index")
2991             if terminate_primitives and vca_deployed.get("needed_terminate"):
2992                 for seq in terminate_primitives:
2993                     # For each sequence in list, get primitive and call _ns_execute_primitive()
2994                     step = "Calling terminate action for vnf_member_index={} primitive={}".format(
2995                         vnf_index, seq.get("name"))
2996                     self.logger.debug(logging_text + step)
2997                     # Create the primitive for each sequence, i.e. "primitive": "touch"
2998                     primitive = seq.get('name')
2999                     mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
3000
3001                     # Add sub-operation
3002                     self._add_suboperation(db_nslcmop,
3003                                            vnf_index,
3004                                            vdu_id,
3005                                            vdu_count_index,
3006                                            vdu_name,
3007                                            primitive,
3008                                            mapped_primitive_params)
3009                     # Sub-operations: Call _ns_execute_primitive() instead of action()
3010                     try:
3011                         result, result_detail = await self._ns_execute_primitive(vca_deployed["ee_id"], primitive,
3012                                                                                  mapped_primitive_params,
3013                                                                                  vca_type=vca_type)
3014                     except LcmException:
3015                         # this happens when VCA is not deployed. In this case it is not needed to terminate
3016                         continue
3017                     result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
3018                     if result not in result_ok:
3019                         raise LcmException("terminate_primitive {}  for vnf_member_index={} fails with "
3020                                            "error {}".format(seq.get("name"), vnf_index, result_detail))
3021                 # set that this VCA do not need terminated
3022                 db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(vca_index)
3023                 self.update_db_2("nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False})
3024
3025         if vca_deployed.get("prometheus_jobs") and self.prometheus:
3026             await self.prometheus.update(remove_jobs=vca_deployed["prometheus_jobs"])
3027
3028         if destroy_ee:
3029             await self.vca_map[vca_type].delete_execution_environment(vca_deployed["ee_id"])
3030
3031     async def _delete_all_N2VC(self, db_nsr: dict):
3032         self._write_all_config_status(db_nsr=db_nsr, status='TERMINATING')
3033         namespace = "." + db_nsr["_id"]
3034         try:
3035             await self.n2vc.delete_namespace(namespace=namespace, total_timeout=self.timeout_charm_delete)
3036         except N2VCNotFound:  # already deleted. Skip
3037             pass
3038         self._write_all_config_status(db_nsr=db_nsr, status='DELETED')
3039
3040     async def _terminate_RO(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
3041         """
3042         Terminates a deployment from RO
3043         :param logging_text:
3044         :param nsr_deployed: db_nsr._admin.deployed
3045         :param nsr_id:
3046         :param nslcmop_id:
3047         :param stage: list of string with the content to write on db_nslcmop.detailed-status.
3048             this method will update only the index 2, but it will write on database the concatenated content of the list
3049         :return:
3050         """
3051         db_nsr_update = {}
3052         failed_detail = []
3053         ro_nsr_id = ro_delete_action = None
3054         if nsr_deployed and nsr_deployed.get("RO"):
3055             ro_nsr_id = nsr_deployed["RO"].get("nsr_id")
3056             ro_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
3057         try:
3058             if ro_nsr_id:
3059                 stage[2] = "Deleting ns from VIM."
3060                 db_nsr_update["detailed-status"] = " ".join(stage)
3061                 self._write_op_status(nslcmop_id, stage)
3062                 self.logger.debug(logging_text + stage[2])
3063                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3064                 self._write_op_status(nslcmop_id, stage)
3065                 desc = await self.RO.delete("ns", ro_nsr_id)
3066                 ro_delete_action = desc["action_id"]
3067                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = ro_delete_action
3068                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
3069                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
3070             if ro_delete_action:
3071                 # wait until NS is deleted from VIM
3072                 stage[2] = "Waiting ns deleted from VIM."
3073                 detailed_status_old = None
3074                 self.logger.debug(logging_text + stage[2] + " RO_id={} ro_delete_action={}".format(ro_nsr_id,
3075                                                                                                    ro_delete_action))
3076                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3077                 self._write_op_status(nslcmop_id, stage)
3078
3079                 delete_timeout = 20 * 60  # 20 minutes
3080                 while delete_timeout > 0:
3081                     desc = await self.RO.show(
3082                         "ns",
3083                         item_id_name=ro_nsr_id,
3084                         extra_item="action",
3085                         extra_item_id=ro_delete_action)
3086
3087                     # deploymentStatus
3088                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
3089
3090                     ns_status, ns_status_info = self.RO.check_action_status(desc)
3091                     if ns_status == "ERROR":
3092                         raise ROclient.ROClientException(ns_status_info)
3093                     elif ns_status == "BUILD":
3094                         stage[2] = "Deleting from VIM {}".format(ns_status_info)
3095                     elif ns_status == "ACTIVE":
3096                         db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
3097                         db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
3098                         break
3099                     else:
3100                         assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
3101                     if stage[2] != detailed_status_old:
3102                         detailed_status_old = stage[2]
3103                         db_nsr_update["detailed-status"] = " ".join(stage)
3104                         self._write_op_status(nslcmop_id, stage)
3105                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
3106                     await asyncio.sleep(5, loop=self.loop)
3107                     delete_timeout -= 5
3108                 else:  # delete_timeout <= 0:
3109                     raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
3110
3111         except Exception as e:
3112             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3113             if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
3114                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
3115                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
3116                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
3117                 self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(ro_nsr_id))
3118             elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
3119                 failed_detail.append("delete conflict: {}".format(e))
3120                 self.logger.debug(logging_text + "RO_ns_id={} delete conflict: {}".format(ro_nsr_id, e))
3121             else:
3122                 failed_detail.append("delete error: {}".format(e))
3123                 self.logger.error(logging_text + "RO_ns_id={} delete error: {}".format(ro_nsr_id, e))
3124
3125         # Delete nsd
3126         if not failed_detail and deep_get(nsr_deployed, ("RO", "nsd_id")):
3127             ro_nsd_id = nsr_deployed["RO"]["nsd_id"]
3128             try:
3129                 stage[2] = "Deleting nsd from RO."
3130                 db_nsr_update["detailed-status"] = " ".join(stage)
3131                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3132                 self._write_op_status(nslcmop_id, stage)
3133                 await self.RO.delete("nsd", ro_nsd_id)
3134                 self.logger.debug(logging_text + "ro_nsd_id={} deleted".format(ro_nsd_id))
3135                 db_nsr_update["_admin.deployed.RO.nsd_id"] = None
3136             except Exception as e:
3137                 if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
3138                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
3139                     self.logger.debug(logging_text + "ro_nsd_id={} already deleted".format(ro_nsd_id))
3140                 elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
3141                     failed_detail.append("ro_nsd_id={} delete conflict: {}".format(ro_nsd_id, e))
3142                     self.logger.debug(logging_text + failed_detail[-1])
3143                 else:
3144                     failed_detail.append("ro_nsd_id={} delete error: {}".format(ro_nsd_id, e))
3145                     self.logger.error(logging_text + failed_detail[-1])
3146
3147         if not failed_detail and deep_get(nsr_deployed, ("RO", "vnfd")):
3148             for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
3149                 if not vnf_deployed or not vnf_deployed["id"]:
3150                     continue
3151                 try:
3152                     ro_vnfd_id = vnf_deployed["id"]
3153                     stage[2] = "Deleting member_vnf_index={} ro_vnfd_id={} from RO.".format(
3154                         vnf_deployed["member-vnf-index"], ro_vnfd_id)
3155                     db_nsr_update["detailed-status"] = " ".join(stage)
3156                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
3157                     self._write_op_status(nslcmop_id, stage)
3158                     await self.RO.delete("vnfd", ro_vnfd_id)
3159                     self.logger.debug(logging_text + "ro_vnfd_id={} deleted".format(ro_vnfd_id))
3160                     db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
3161                 except Exception as e:
3162                     if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
3163                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
3164                         self.logger.debug(logging_text + "ro_vnfd_id={} already deleted ".format(ro_vnfd_id))
3165                     elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
3166                         failed_detail.append("ro_vnfd_id={} delete conflict: {}".format(ro_vnfd_id, e))
3167                         self.logger.debug(logging_text + failed_detail[-1])
3168                     else:
3169                         failed_detail.append("ro_vnfd_id={} delete error: {}".format(ro_vnfd_id, e))
3170                         self.logger.error(logging_text + failed_detail[-1])
3171
3172         if failed_detail:
3173             stage[2] = "Error deleting from VIM"
3174         else:
3175             stage[2] = "Deleted from VIM"
3176         db_nsr_update["detailed-status"] = " ".join(stage)
3177         self.update_db_2("nsrs", nsr_id, db_nsr_update)
3178         self._write_op_status(nslcmop_id, stage)
3179
3180         if failed_detail:
3181             raise LcmException("; ".join(failed_detail))
3182
3183     async def terminate(self, nsr_id, nslcmop_id):
3184         # Try to lock HA task here
3185         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3186         if not task_is_locked_by_me:
3187             return
3188
3189         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
3190         self.logger.debug(logging_text + "Enter")
3191         timeout_ns_terminate = self.timeout_ns_terminate
3192         db_nsr = None
3193         db_nslcmop = None
3194         operation_params = None
3195         exc = None
3196         error_list = []   # annotates all failed error messages
3197         db_nslcmop_update = {}
3198         autoremove = False  # autoremove after terminated
3199         tasks_dict_info = {}
3200         db_nsr_update = {}
3201         stage = ["Stage 1/3: Preparing task.", "Waiting for previous operations to terminate.", ""]
3202         # ^ contains [stage, step, VIM-status]
3203         try:
3204             # wait for any previous tasks in process
3205             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
3206
3207             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
3208             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3209             operation_params = db_nslcmop.get("operationParams") or {}
3210             if operation_params.get("timeout_ns_terminate"):
3211                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
3212             stage[1] = "Getting nsr={} from db.".format(nsr_id)
3213             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3214
3215             db_nsr_update["operational-status"] = "terminating"
3216             db_nsr_update["config-status"] = "terminating"
3217             self._write_ns_status(
3218                 nsr_id=nsr_id,
3219                 ns_state="TERMINATING",
3220                 current_operation="TERMINATING",
3221                 current_operation_id=nslcmop_id,
3222                 other_update=db_nsr_update
3223             )
3224             self._write_op_status(
3225                 op_id=nslcmop_id,
3226                 queuePosition=0,
3227                 stage=stage
3228             )
3229             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
3230             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
3231                 return
3232
3233             stage[1] = "Getting vnf descriptors from db."
3234             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
3235             db_vnfds_from_id = {}
3236             db_vnfds_from_member_index = {}
3237             # Loop over VNFRs
3238             for vnfr in db_vnfrs_list:
3239                 vnfd_id = vnfr["vnfd-id"]
3240                 if vnfd_id not in db_vnfds_from_id:
3241                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
3242                     db_vnfds_from_id[vnfd_id] = vnfd
3243                 db_vnfds_from_member_index[vnfr["member-vnf-index-ref"]] = db_vnfds_from_id[vnfd_id]
3244
3245             # Destroy individual execution environments when there are terminating primitives.
3246             # Rest of EE will be deleted at once
3247             # TODO - check before calling _destroy_N2VC
3248             # if not operation_params.get("skip_terminate_primitives"):#
3249             # or not vca.get("needed_terminate"):
3250             stage[0] = "Stage 2/3 execute terminating primitives."
3251             self.logger.debug(logging_text + stage[0])
3252             stage[1] = "Looking execution environment that needs terminate."
3253             self.logger.debug(logging_text + stage[1])
3254             # self.logger.debug("nsr_deployed: {}".format(nsr_deployed))
3255             for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
3256                 config_descriptor = None
3257                 if not vca or not vca.get("ee_id"):
3258                     continue
3259                 if not vca.get("member-vnf-index"):
3260                     # ns
3261                     config_descriptor = db_nsr.get("ns-configuration")
3262                 elif vca.get("vdu_id"):
3263                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
3264                     vdud = next((vdu for vdu in db_vnfd.get("vdu", ()) if vdu["id"] == vca.get("vdu_id")), None)
3265                     if vdud:
3266                         config_descriptor = vdud.get("vdu-configuration")
3267                 elif vca.get("kdu_name"):
3268                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
3269                     kdud = next((kdu for kdu in db_vnfd.get("kdu", ()) if kdu["name"] == vca.get("kdu_name")), None)
3270                     if kdud:
3271                         config_descriptor = kdud.get("kdu-configuration")
3272                 else:
3273                     config_descriptor = db_vnfds_from_member_index[vca["member-vnf-index"]].get("vnf-configuration")
3274                 vca_type = vca.get("type")
3275                 exec_terminate_primitives = (not operation_params.get("skip_terminate_primitives") and
3276                                              vca.get("needed_terminate"))
3277                 # For helm we must destroy_ee. Also for native_charm, as juju_model cannot be deleted if there are
3278                 # pending native charms
3279                 destroy_ee = True if vca_type in ("helm", "native_charm") else False
3280                 # self.logger.debug(logging_text + "vca_index: {}, ee_id: {}, vca_type: {} destroy_ee: {}".format(
3281                 #     vca_index, vca.get("ee_id"), vca_type, destroy_ee))
3282                 task = asyncio.ensure_future(
3283                     self.destroy_N2VC(logging_text, db_nslcmop, vca, config_descriptor, vca_index,
3284                                       destroy_ee, exec_terminate_primitives))
3285                 tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
3286
3287             # wait for pending tasks of terminate primitives
3288             if tasks_dict_info:
3289                 self.logger.debug(logging_text + 'Waiting for tasks {}'.format(list(tasks_dict_info.keys())))
3290                 error_list = await self._wait_for_tasks(logging_text, tasks_dict_info,
3291                                                         min(self.timeout_charm_delete, timeout_ns_terminate),
3292                                                         stage, nslcmop_id)
3293                 tasks_dict_info.clear()
3294                 if error_list:
3295                     return   # raise LcmException("; ".join(error_list))
3296
3297             # remove All execution environments at once
3298             stage[0] = "Stage 3/3 delete all."
3299
3300             if nsr_deployed.get("VCA"):
3301                 stage[1] = "Deleting all execution environments."
3302                 self.logger.debug(logging_text + stage[1])
3303                 task_delete_ee = asyncio.ensure_future(asyncio.wait_for(self._delete_all_N2VC(db_nsr=db_nsr),
3304                                                                         timeout=self.timeout_charm_delete))
3305                 # task_delete_ee = asyncio.ensure_future(self.n2vc.delete_namespace(namespace="." + nsr_id))
3306                 tasks_dict_info[task_delete_ee] = "Terminating all VCA"
3307
3308             # Delete from k8scluster
3309             stage[1] = "Deleting KDUs."
3310             self.logger.debug(logging_text + stage[1])
3311             # print(nsr_deployed)
3312             for kdu in get_iterable(nsr_deployed, "K8s"):
3313                 if not kdu or not kdu.get("kdu-instance"):
3314                     continue
3315                 kdu_instance = kdu.get("kdu-instance")
3316                 if kdu.get("k8scluster-type") in self.k8scluster_map:
3317                     task_delete_kdu_instance = asyncio.ensure_future(
3318                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
3319                             cluster_uuid=kdu.get("k8scluster-uuid"),
3320                             kdu_instance=kdu_instance))
3321                 else:
3322                     self.logger.error(logging_text + "Unknown k8s deployment type {}".
3323                                       format(kdu.get("k8scluster-type")))
3324                     continue
3325                 tasks_dict_info[task_delete_kdu_instance] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
3326
3327             # remove from RO
3328             stage[1] = "Deleting ns from VIM."
3329             if self.ng_ro:
3330                 task_delete_ro = asyncio.ensure_future(
3331                     self._terminate_ng_ro(logging_text, nsr_deployed, nsr_id, nslcmop_id, stage))
3332             else:
3333                 task_delete_ro = asyncio.ensure_future(
3334                     self._terminate_RO(logging_text, nsr_deployed, nsr_id, nslcmop_id, stage))
3335             tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
3336
3337             # rest of staff will be done at finally
3338
3339         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
3340             self.logger.error(logging_text + "Exit Exception {}".format(e))
3341             exc = e
3342         except asyncio.CancelledError:
3343             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
3344             exc = "Operation was cancelled"
3345         except Exception as e:
3346             exc = traceback.format_exc()
3347             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
3348         finally:
3349             if exc:
3350                 error_list.append(str(exc))
3351             try:
3352                 # wait for pending tasks
3353                 if tasks_dict_info:
3354                     stage[1] = "Waiting for terminate pending tasks."
3355                     self.logger.debug(logging_text + stage[1])
3356                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_terminate,
3357                                                              stage, nslcmop_id)
3358                 stage[1] = stage[2] = ""
3359             except asyncio.CancelledError:
3360                 error_list.append("Cancelled")
3361                 # TODO cancell all tasks
3362             except Exception as exc:
3363                 error_list.append(str(exc))
3364             # update status at database
3365             if error_list:
3366                 error_detail = "; ".join(error_list)
3367                 # self.logger.error(logging_text + error_detail)
3368                 error_description_nslcmop = '{} Detail: {}'.format(stage[0], error_detail)
3369                 error_description_nsr = 'Operation: TERMINATING.{}, {}.'.format(nslcmop_id, stage[0])
3370
3371                 db_nsr_update["operational-status"] = "failed"
3372                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
3373                 db_nslcmop_update["detailed-status"] = error_detail
3374                 nslcmop_operation_state = "FAILED"
3375                 ns_state = "BROKEN"
3376             else:
3377                 error_detail = None
3378                 error_description_nsr = error_description_nslcmop = None
3379                 ns_state = "NOT_INSTANTIATED"
3380                 db_nsr_update["operational-status"] = "terminated"
3381                 db_nsr_update["detailed-status"] = "Done"
3382                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
3383                 db_nslcmop_update["detailed-status"] = "Done"
3384                 nslcmop_operation_state = "COMPLETED"
3385
3386             if db_nsr:
3387                 self._write_ns_status(
3388                     nsr_id=nsr_id,
3389                     ns_state=ns_state,
3390                     current_operation="IDLE",
3391                     current_operation_id=None,
3392                     error_description=error_description_nsr,
3393                     error_detail=error_detail,
3394                     other_update=db_nsr_update
3395                 )
3396             self._write_op_status(
3397                 op_id=nslcmop_id,
3398                 stage="",
3399                 error_message=error_description_nslcmop,
3400                 operation_state=nslcmop_operation_state,
3401                 other_update=db_nslcmop_update,
3402             )
3403             if ns_state == "NOT_INSTANTIATED":
3404                 try:
3405                     self.db.set_list("vnfrs", {"nsr-id-ref": nsr_id}, {"_admin.nsState": "NOT_INSTANTIATED"})
3406                 except DbException as e:
3407                     self.logger.warn(logging_text + 'Error writing VNFR status for nsr-id-ref: {} -> {}'.
3408                                      format(nsr_id, e))
3409             if operation_params:
3410                 autoremove = operation_params.get("autoremove", False)
3411             if nslcmop_operation_state:
3412                 try:
3413                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3414                                                                  "operationState": nslcmop_operation_state,
3415                                                                  "autoremove": autoremove},
3416                                             loop=self.loop)
3417                 except Exception as e:
3418                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3419
3420             self.logger.debug(logging_text + "Exit")
3421             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
3422
3423     async def _wait_for_tasks(self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None):
3424         time_start = time()
3425         error_detail_list = []
3426         error_list = []
3427         pending_tasks = list(created_tasks_info.keys())
3428         num_tasks = len(pending_tasks)
3429         num_done = 0
3430         stage[1] = "{}/{}.".format(num_done, num_tasks)
3431         self._write_op_status(nslcmop_id, stage)
3432         while pending_tasks:
3433             new_error = None
3434             _timeout = timeout + time_start - time()
3435             done, pending_tasks = await asyncio.wait(pending_tasks, timeout=_timeout,
3436                                                      return_when=asyncio.FIRST_COMPLETED)
3437             num_done += len(done)
3438             if not done:   # Timeout
3439                 for task in pending_tasks:
3440                     new_error = created_tasks_info[task] + ": Timeout"
3441                     error_detail_list.append(new_error)
3442                     error_list.append(new_error)
3443                 break
3444             for task in done:
3445                 if task.cancelled():
3446                     exc = "Cancelled"
3447                 else:
3448                     exc = task.exception()
3449                 if exc:
3450                     if isinstance(exc, asyncio.TimeoutError):
3451                         exc = "Timeout"
3452                     new_error = created_tasks_info[task] + ": {}".format(exc)
3453                     error_list.append(created_tasks_info[task])
3454                     error_detail_list.append(new_error)
3455                     if isinstance(exc, (str, DbException, N2VCException, ROclient.ROClientException, LcmException,
3456                                         K8sException)):
3457                         self.logger.error(logging_text + new_error)
3458                     else:
3459                         exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__))
3460                         self.logger.error(logging_text + created_tasks_info[task] + exc_traceback)
3461                 else:
3462                     self.logger.debug(logging_text + created_tasks_info[task] + ": Done")
3463             stage[1] = "{}/{}.".format(num_done, num_tasks)
3464             if new_error:
3465                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
3466                 if nsr_id:  # update also nsr
3467                     self.update_db_2("nsrs", nsr_id, {"errorDescription": "Error at: " + ", ".join(error_list),
3468                                                       "errorDetail": ". ".join(error_detail_list)})
3469             self._write_op_status(nslcmop_id, stage)
3470         return error_detail_list
3471
3472     @staticmethod
3473     def _map_primitive_params(primitive_desc, params, instantiation_params):
3474         """
3475         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
3476         The default-value is used. If it is between < > it look for a value at instantiation_params
3477         :param primitive_desc: portion of VNFD/NSD that describes primitive
3478         :param params: Params provided by user
3479         :param instantiation_params: Instantiation params provided by user
3480         :return: a dictionary with the calculated params
3481         """
3482         calculated_params = {}
3483         for parameter in primitive_desc.get("parameter", ()):
3484             param_name = parameter["name"]
3485             if param_name in params:
3486                 calculated_params[param_name] = params[param_name]
3487             elif "default-value" in parameter or "value" in parameter:
3488                 if "value" in parameter:
3489                     calculated_params[param_name] = parameter["value"]
3490                 else:
3491                     calculated_params[param_name] = parameter["default-value"]
3492                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
3493                         and calculated_params[param_name].endswith(">"):
3494                     if calculated_params[param_name][1:-1] in instantiation_params:
3495                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
3496                     else:
3497                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
3498                                            format(calculated_params[param_name], primitive_desc["name"]))
3499             else:
3500                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
3501                                    format(param_name, primitive_desc["name"]))
3502
3503             if isinstance(calculated_params[param_name], (dict, list, tuple)):
3504                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
3505                                                                width=256)
3506             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
3507                 calculated_params[param_name] = calculated_params[param_name][7:]
3508
3509         # add always ns_config_info if primitive name is config
3510         if primitive_desc["name"] == "config":
3511             if "ns_config_info" in instantiation_params:
3512                 calculated_params["ns_config_info"] = instantiation_params["ns_config_info"]
3513         return calculated_params
3514
3515     def _look_for_deployed_vca(self, deployed_vca, member_vnf_index, vdu_id, vdu_count_index, kdu_name=None,
3516                                ee_descriptor_id=None):
3517         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
3518         for vca in deployed_vca:
3519             if not vca:
3520                 continue
3521             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
3522                 continue
3523             if vdu_count_index is not None and vdu_count_index != vca["vdu_count_index"]:
3524                 continue
3525             if kdu_name and kdu_name != vca["kdu_name"]:
3526                 continue
3527             if ee_descriptor_id and ee_descriptor_id != vca["ee_descriptor_id"]:
3528                 continue
3529             break
3530         else:
3531             # vca_deployed not found
3532             raise LcmException("charm for member_vnf_index={} vdu_id={}.{} kdu_name={} execution-environment-list.id={}"
3533                                " is not deployed".format(member_vnf_index, vdu_id, vdu_count_index, kdu_name,
3534                                                          ee_descriptor_id))
3535
3536         # get ee_id
3537         ee_id = vca.get("ee_id")
3538         vca_type = vca.get("type", "lxc_proxy_charm")  # default value for backward compatibility - proxy charm
3539         if not ee_id:
3540             raise LcmException("charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
3541                                "execution environment"
3542                                .format(member_vnf_index, vdu_id, kdu_name, vdu_count_index))
3543         return ee_id, vca_type
3544
3545     async def _ns_execute_primitive(self, ee_id, primitive, primitive_params, retries=0,
3546                                     retries_interval=30, timeout=None,
3547                                     vca_type=None, db_dict=None) -> (str, str):
3548         try:
3549             if primitive == "config":
3550                 primitive_params = {"params": primitive_params}
3551
3552             vca_type = vca_type or "lxc_proxy_charm"
3553
3554             while retries >= 0:
3555                 try:
3556                     output = await asyncio.wait_for(
3557                         self.vca_map[vca_type].exec_primitive(
3558                             ee_id=ee_id,
3559                             primitive_name=primitive,
3560                             params_dict=primitive_params,
3561                             progress_timeout=self.timeout_progress_primitive,
3562                             total_timeout=self.timeout_primitive,
3563                             db_dict=db_dict),
3564                         timeout=timeout or self.timeout_primitive)
3565                     # execution was OK
3566                     break
3567                 except asyncio.CancelledError:
3568                     raise
3569                 except Exception as e:  # asyncio.TimeoutError
3570                     if isinstance(e, asyncio.TimeoutError):
3571                         e = "Timeout"
3572                     retries -= 1
3573                     if retries >= 0:
3574                         self.logger.debug('Error executing action {} on {} -> {}'.format(primitive, ee_id, e))
3575                         # wait and retry
3576                         await asyncio.sleep(retries_interval, loop=self.loop)
3577                     else:
3578                         return 'FAILED', str(e)
3579
3580             return 'COMPLETED', output
3581
3582         except (LcmException, asyncio.CancelledError):
3583             raise
3584         except Exception as e:
3585             return 'FAIL', 'Error executing action {}: {}'.format(primitive, e)
3586
3587     async def action(self, nsr_id, nslcmop_id):
3588
3589         # Try to lock HA task here
3590         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3591         if not task_is_locked_by_me:
3592             return
3593
3594         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
3595         self.logger.debug(logging_text + "Enter")
3596         # get all needed from database
3597         db_nsr = None
3598         db_nslcmop = None
3599         db_nsr_update = {}
3600         db_nslcmop_update = {}
3601         nslcmop_operation_state = None
3602         error_description_nslcmop = None
3603         exc = None
3604         try:
3605             # wait for any previous tasks in process
3606             step = "Waiting for previous operations to terminate"
3607             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3608
3609             self._write_ns_status(
3610                 nsr_id=nsr_id,
3611                 ns_state=None,
3612                 current_operation="RUNNING ACTION",
3613                 current_operation_id=nslcmop_id
3614             )
3615
3616             step = "Getting information from database"
3617             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3618             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3619
3620             nsr_deployed = db_nsr["_admin"].get("deployed")
3621             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3622             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3623             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
3624             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3625             primitive = db_nslcmop["operationParams"]["primitive"]
3626             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
3627             timeout_ns_action = db_nslcmop["operationParams"].get("timeout_ns_action", self.timeout_primitive)
3628
3629             if vnf_index:
3630                 step = "Getting vnfr from database"
3631                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3632                 step = "Getting vnfd from database"
3633                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3634             else:
3635                 step = "Getting nsd from database"
3636                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
3637
3638             # for backward compatibility
3639             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3640                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3641                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3642                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3643
3644             # look for primitive
3645             config_primitive_desc = descriptor_configuration = None
3646             if vdu_id:
3647                 for vdu in get_iterable(db_vnfd, "vdu"):
3648                     if vdu_id == vdu["id"]:
3649                         descriptor_configuration = vdu.get("vdu-configuration")
3650                         break
3651             elif kdu_name:
3652                 for kdu in get_iterable(db_vnfd, "kdu"):
3653                     if kdu_name == kdu["name"]:
3654                         descriptor_configuration = kdu.get("kdu-configuration")
3655                         break
3656             elif vnf_index:
3657                 descriptor_configuration = db_vnfd.get("vnf-configuration")
3658             else:
3659                 descriptor_configuration = db_nsd.get("ns-configuration")
3660
3661             if descriptor_configuration and descriptor_configuration.get("config-primitive"):
3662                 for config_primitive in descriptor_configuration["config-primitive"]:
3663                     if config_primitive["name"] == primitive:
3664                         config_primitive_desc = config_primitive
3665                         break
3666
3667             if not config_primitive_desc:
3668                 if not (kdu_name and primitive in ("upgrade", "rollback", "status")):
3669                     raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
3670                                        format(primitive))
3671                 primitive_name = primitive
3672                 ee_descriptor_id = None
3673             else:
3674                 primitive_name = config_primitive_desc.get("execution-environment-primitive", primitive)
3675                 ee_descriptor_id = config_primitive_desc.get("execution-environment-ref")
3676
3677             if vnf_index:
3678                 if vdu_id:
3679                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
3680                     desc_params = self._format_additional_params(vdur.get("additionalParams"))
3681                 elif kdu_name:
3682                     kdur = next((x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None)
3683                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
3684                 else:
3685                     desc_params = self._format_additional_params(db_vnfr.get("additionalParamsForVnf"))
3686             else:
3687                 desc_params = self._format_additional_params(db_nsr.get("additionalParamsForNs"))
3688
3689             if kdu_name:
3690                 kdu_action = True if not deep_get(kdu, ("kdu-configuration", "juju")) else False
3691
3692             # TODO check if ns is in a proper status
3693             if kdu_name and (primitive_name in ("upgrade", "rollback", "status") or kdu_action):
3694                 # kdur and desc_params already set from before
3695                 if primitive_params:
3696                     desc_params.update(primitive_params)
3697                 # TODO Check if we will need something at vnf level
3698                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
3699                     if kdu_name == kdu["kdu-name"] and kdu["member-vnf-index"] == vnf_index:
3700                         break
3701                 else:
3702                     raise LcmException("KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index))
3703
3704                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
3705                     msg = "unknown k8scluster-type '{}'".format(kdu.get("k8scluster-type"))
3706                     raise LcmException(msg)
3707
3708                 db_dict = {"collection": "nsrs",
3709                            "filter": {"_id": nsr_id},
3710                            "path": "_admin.deployed.K8s.{}".format(index)}
3711                 self.logger.debug(logging_text + "Exec k8s {} on {}.{}".format(primitive_name, vnf_index, kdu_name))
3712                 step = "Executing kdu {}".format(primitive_name)
3713                 if primitive_name == "upgrade":
3714                     if desc_params.get("kdu_model"):
3715                         kdu_model = desc_params.get("kdu_model")
3716                         del desc_params["kdu_model"]
3717                     else:
3718                         kdu_model = kdu.get("kdu-model")
3719                         parts = kdu_model.split(sep=":")
3720                         if len(parts) == 2:
3721                             kdu_model = parts[0]
3722
3723                     detailed_status = await asyncio.wait_for(
3724                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
3725                             cluster_uuid=kdu.get("k8scluster-uuid"),
3726                             kdu_instance=kdu.get("kdu-instance"),
3727                             atomic=True, kdu_model=kdu_model,
3728                             params=desc_params, db_dict=db_dict,
3729                             timeout=timeout_ns_action),
3730                         timeout=timeout_ns_action + 10)
3731                     self.logger.debug(logging_text + " Upgrade of kdu {} done".format(detailed_status))
3732                 elif primitive_name == "rollback":
3733                     detailed_status = await asyncio.wait_for(
3734                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
3735                             cluster_uuid=kdu.get("k8scluster-uuid"),
3736                             kdu_instance=kdu.get("kdu-instance"),
3737                             db_dict=db_dict),
3738                         timeout=timeout_ns_action)
3739                 elif primitive_name == "status":
3740                     detailed_status = await asyncio.wait_for(
3741                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
3742                             cluster_uuid=kdu.get("k8scluster-uuid"),
3743                             kdu_instance=kdu.get("kdu-instance")),
3744                         timeout=timeout_ns_action)
3745                 else:
3746                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(kdu["kdu-name"], nsr_id)
3747                     params = self._map_primitive_params(config_primitive_desc, primitive_params, desc_params)
3748
3749                     detailed_status = await asyncio.wait_for(
3750                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
3751                             cluster_uuid=kdu.get("k8scluster-uuid"),
3752                             kdu_instance=kdu_instance,
3753                             primitive_name=primitive_name,
3754                             params=params, db_dict=db_dict,
3755                             timeout=timeout_ns_action),
3756                         timeout=timeout_ns_action)
3757
3758                 if detailed_status:
3759                     nslcmop_operation_state = 'COMPLETED'
3760                 else:
3761                     detailed_status = ''
3762                     nslcmop_operation_state = 'FAILED'
3763             else:
3764                 ee_id, vca_type = self._look_for_deployed_vca(nsr_deployed["VCA"],
3765                                                               member_vnf_index=vnf_index,
3766                                                               vdu_id=vdu_id,
3767                                                               vdu_count_index=vdu_count_index,
3768                                                               ee_descriptor_id=ee_descriptor_id)
3769                 db_nslcmop_notif = {"collection": "nslcmops",
3770                                     "filter": {"_id": nslcmop_id},
3771                                     "path": "admin.VCA"}
3772                 nslcmop_operation_state, detailed_status = await self._ns_execute_primitive(
3773                     ee_id,
3774                     primitive=primitive_name,
3775                     primitive_params=self._map_primitive_params(config_primitive_desc, primitive_params, desc_params),
3776                     timeout=timeout_ns_action,
3777                     vca_type=vca_type,
3778                     db_dict=db_nslcmop_notif)
3779
3780             db_nslcmop_update["detailed-status"] = detailed_status
3781             error_description_nslcmop = detailed_status if nslcmop_operation_state == "FAILED" else ""
3782             self.logger.debug(logging_text + " task Done with result {} {}".format(nslcmop_operation_state,
3783                                                                                    detailed_status))
3784             return  # database update is called inside finally
3785
3786         except (DbException, LcmException, N2VCException, K8sException) as e:
3787             self.logger.error(logging_text + "Exit Exception {}".format(e))
3788             exc = e
3789         except asyncio.CancelledError:
3790             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
3791             exc = "Operation was cancelled"
3792         except asyncio.TimeoutError:
3793             self.logger.error(logging_text + "Timeout while '{}'".format(step))
3794             exc = "Timeout"
3795         except Exception as e:
3796             exc = traceback.format_exc()
3797             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
3798         finally:
3799             if exc:
3800                 db_nslcmop_update["detailed-status"] = detailed_status = error_description_nslcmop = \
3801                     "FAILED {}: {}".format(step, exc)
3802                 nslcmop_operation_state = "FAILED"
3803             if db_nsr:
3804                 self._write_ns_status(
3805                     nsr_id=nsr_id,
3806                     ns_state=db_nsr["nsState"],   # TODO check if degraded. For the moment use previous status
3807                     current_operation="IDLE",
3808                     current_operation_id=None,
3809                     # error_description=error_description_nsr,
3810                     # error_detail=error_detail,
3811                     other_update=db_nsr_update
3812                 )
3813
3814             self._write_op_status(
3815                 op_id=nslcmop_id,
3816                 stage="",
3817                 error_message=error_description_nslcmop,
3818                 operation_state=nslcmop_operation_state,
3819                 other_update=db_nslcmop_update,
3820             )
3821
3822             if nslcmop_operation_state:
3823                 try:
3824                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3825                                                                "operationState": nslcmop_operation_state},
3826                                             loop=self.loop)
3827                 except Exception as e:
3828                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3829             self.logger.debug(logging_text + "Exit")
3830             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
3831             return nslcmop_operation_state, detailed_status
3832
3833     async def scale(self, nsr_id, nslcmop_id):
3834
3835         # Try to lock HA task here
3836         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3837         if not task_is_locked_by_me:
3838             return
3839
3840         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
3841         self.logger.debug(logging_text + "Enter")
3842         # get all needed from database
3843         db_nsr = None
3844         db_nslcmop = None
3845         db_nslcmop_update = {}
3846         nslcmop_operation_state = None
3847         db_nsr_update = {}
3848         exc = None
3849         # in case of error, indicates what part of scale was failed to put nsr at error status
3850         scale_process = None
3851         old_operational_status = ""
3852         old_config_status = ""
3853         vnfr_scaled = False
3854         try:
3855             # wait for any previous tasks in process
3856             step = "Waiting for previous operations to terminate"
3857             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3858
3859             self._write_ns_status(
3860                 nsr_id=nsr_id,
3861                 ns_state=None,
3862                 current_operation="SCALING",
3863                 current_operation_id=nslcmop_id
3864             )
3865
3866             step = "Getting nslcmop from database"
3867             self.logger.debug(step + " after having waited for previous tasks to be completed")
3868             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3869             step = "Getting nsr from database"
3870             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3871
3872             old_operational_status = db_nsr["operational-status"]
3873             old_config_status = db_nsr["config-status"]
3874             step = "Parsing scaling parameters"
3875             # self.logger.debug(step)
3876             db_nsr_update["operational-status"] = "scaling"
3877             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3878             nsr_deployed = db_nsr["_admin"].get("deployed")
3879
3880             #######
3881             nsr_deployed = db_nsr["_admin"].get("deployed")
3882             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3883             # vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3884             # vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3885             # vdu_name = db_nslcmop["operationParams"].get("vdu_name")
3886             #######
3887
3888             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
3889             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
3890             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
3891             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
3892             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
3893
3894             # for backward compatibility
3895             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3896                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3897                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3898                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3899
3900             step = "Getting vnfr from database"
3901             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3902             step = "Getting vnfd from database"
3903             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3904
3905             step = "Getting scaling-group-descriptor"
3906             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
3907                 if scaling_descriptor["name"] == scaling_group:
3908                     break
3909             else:
3910                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
3911                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
3912
3913             # cooldown_time = 0
3914             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
3915             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
3916             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
3917             #         break
3918
3919             # TODO check if ns is in a proper status
3920             step = "Sending scale order to VIM"
3921             nb_scale_op = 0
3922             if not db_nsr["_admin"].get("scaling-group"):
3923                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
3924                 admin_scale_index = 0
3925             else:
3926                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
3927                     if admin_scale_info["name"] == scaling_group:
3928                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
3929                         break
3930                 else:  # not found, set index one plus last element and add new entry with the name
3931                     admin_scale_index += 1
3932                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
3933             RO_scaling_info = []
3934             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
3935             if scaling_type == "SCALE_OUT":
3936                 # count if max-instance-count is reached
3937                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
3938                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
3939                 if nb_scale_op >= max_instance_count:
3940                     raise LcmException("reached the limit of {} (max-instance-count) "
3941                                        "scaling-out operations for the "
3942                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3943
3944                 nb_scale_op += 1
3945                 vdu_scaling_info["scaling_direction"] = "OUT"
3946                 vdu_scaling_info["vdu-create"] = {}
3947                 for vdu_scale_info in scaling_descriptor["vdu"]:
3948                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3949                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
3950                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3951
3952             elif scaling_type == "SCALE_IN":
3953                 # count if min-instance-count is reached
3954                 min_instance_count = 0
3955                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
3956                     min_instance_count = int(scaling_descriptor["min-instance-count"])
3957                 if nb_scale_op <= min_instance_count:
3958                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
3959                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3960                 nb_scale_op -= 1
3961                 vdu_scaling_info["scaling_direction"] = "IN"
3962                 vdu_scaling_info["vdu-delete"] = {}
3963                 for vdu_scale_info in scaling_descriptor["vdu"]:
3964                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3965                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
3966                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3967
3968             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
3969             vdu_create = vdu_scaling_info.get("vdu-create")
3970             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
3971             if vdu_scaling_info["scaling_direction"] == "IN":
3972                 for vdur in reversed(db_vnfr["vdur"]):
3973                     if vdu_delete.get(vdur["vdu-id-ref"]):
3974                         vdu_delete[vdur["vdu-id-ref"]] -= 1
3975                         vdu_scaling_info["vdu"].append({
3976                             "name": vdur["name"],
3977                             "vdu_id": vdur["vdu-id-ref"],
3978                             "interface": []
3979                         })
3980                         for interface in vdur["interfaces"]:
3981                             vdu_scaling_info["vdu"][-1]["interface"].append({
3982                                 "name": interface["name"],
3983                                 "ip_address": interface["ip-address"],
3984                                 "mac_address": interface.get("mac-address"),
3985                             })
3986                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
3987
3988             # PRE-SCALE BEGIN
3989             step = "Executing pre-scale vnf-config-primitive"
3990             if scaling_descriptor.get("scaling-config-action"):
3991                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
3992                     if (scaling_config_action.get("trigger") == "pre-scale-in" and scaling_type == "SCALE_IN") \
3993                        or (scaling_config_action.get("trigger") == "pre-scale-out" and scaling_type == "SCALE_OUT"):
3994                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
3995                         step = db_nslcmop_update["detailed-status"] = \
3996                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
3997
3998                         # look for primitive
3999                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
4000                             if config_primitive["name"] == vnf_config_primitive:
4001                                 break
4002                         else:
4003                             raise LcmException(
4004                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
4005                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
4006                                 "primitive".format(scaling_group, vnf_config_primitive))
4007
4008                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
4009                         if db_vnfr.get("additionalParamsForVnf"):
4010                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
4011
4012                         scale_process = "VCA"
4013                         db_nsr_update["config-status"] = "configuring pre-scaling"
4014                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
4015
4016                         # Pre-scale retry check: Check if this sub-operation has been executed before
4017                         op_index = self._check_or_add_scale_suboperation(
4018                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
4019                         if op_index == self.SUBOPERATION_STATUS_SKIP:
4020                             # Skip sub-operation
4021                             result = 'COMPLETED'
4022                             result_detail = 'Done'
4023                             self.logger.debug(logging_text +
4024                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
4025                                                   vnf_config_primitive, result, result_detail))
4026                         else:
4027                             if op_index == self.SUBOPERATION_STATUS_NEW:
4028                                 # New sub-operation: Get index of this sub-operation
4029                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
4030                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
4031                                                   format(vnf_config_primitive))
4032                             else:
4033                                 # retry:  Get registered params for this existing sub-operation
4034                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
4035                                 vnf_index = op.get('member_vnf_index')
4036                                 vnf_config_primitive = op.get('primitive')
4037                                 primitive_params = op.get('primitive_params')
4038                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
4039                                                   format(vnf_config_primitive))
4040                             # Execute the primitive, either with new (first-time) or registered (reintent) args
4041                             ee_descriptor_id = config_primitive.get("execution-environment-ref")
4042                             primitive_name = config_primitive.get("execution-environment-primitive",
4043                                                                   vnf_config_primitive)
4044                             ee_id, vca_type = self._look_for_deployed_vca(nsr_deployed["VCA"],
4045                                                                           member_vnf_index=vnf_index,
4046                                                                           vdu_id=None,
4047                                                                           vdu_count_index=None,
4048                                                                           ee_descriptor_id=ee_descriptor_id)
4049                             result, result_detail = await self._ns_execute_primitive(
4050                                 ee_id, primitive_name, primitive_params, vca_type)
4051                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
4052                                 vnf_config_primitive, result, result_detail))
4053                             # Update operationState = COMPLETED | FAILED
4054                             self._update_suboperation_status(
4055                                 db_nslcmop, op_index, result, result_detail)
4056
4057                         if result == "FAILED":
4058                             raise LcmException(result_detail)
4059                         db_nsr_update["config-status"] = old_config_status
4060                         scale_process = None
4061             # PRE-SCALE END
4062
4063             # SCALE RO - BEGIN
4064             # Should this block be skipped if 'RO_nsr_id' == None ?
4065             # if (RO_nsr_id and RO_scaling_info):
4066             if RO_scaling_info:
4067                 scale_process = "RO"
4068                 # Scale RO retry check: Check if this sub-operation has been executed before
4069                 op_index = self._check_or_add_scale_suboperation(
4070                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
4071                 if op_index == self.SUBOPERATION_STATUS_SKIP:
4072                     # Skip sub-operation
4073                     result = 'COMPLETED'
4074                     result_detail = 'Done'
4075                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
4076                         result, result_detail))
4077                 else:
4078                     if op_index == self.SUBOPERATION_STATUS_NEW:
4079                         # New sub-operation: Get index of this sub-operation
4080                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
4081                         self.logger.debug(logging_text + "New sub-operation RO")
4082                     else:
4083                         # retry:  Get registered params for this existing sub-operation
4084                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
4085                         RO_nsr_id = op.get('RO_nsr_id')
4086                         RO_scaling_info = op.get('RO_scaling_info')
4087                         self.logger.debug(logging_text + "Sub-operation RO retry for primitive {}".format(
4088                             vnf_config_primitive))
4089
4090                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
4091                     db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
4092                     db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
4093                     # wait until ready
4094                     RO_nslcmop_id = RO_desc["instance_action_id"]
4095                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
4096
4097                     RO_task_done = False
4098                     step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
4099                     detailed_status_old = None
4100                     self.logger.debug(logging_text + step)
4101
4102                     deployment_timeout = 1 * 3600   # One hour
4103                     while deployment_timeout > 0:
4104                         if not RO_task_done:
4105                             desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
4106                                                       extra_item_id=RO_nslcmop_id)
4107
4108                             # deploymentStatus
4109                             self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
4110
4111                             ns_status, ns_status_info = self.RO.check_action_status(desc)
4112                             if ns_status == "ERROR":
4113                                 raise ROclient.ROClientException(ns_status_info)
4114                             elif ns_status == "BUILD":
4115                                 detailed_status = step + "; {}".format(ns_status_info)
4116                             elif ns_status == "ACTIVE":
4117                                 RO_task_done = True
4118                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
4119                                 self.logger.debug(logging_text + step)
4120                             else:
4121                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
4122                         else:
4123
4124                             if ns_status == "ERROR":
4125                                 raise ROclient.ROClientException(ns_status_info)
4126                             elif ns_status == "BUILD":
4127                                 detailed_status = step + "; {}".format(ns_status_info)
4128                             elif ns_status == "ACTIVE":
4129                                 step = detailed_status = \
4130                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
4131                                 if not vnfr_scaled:
4132                                     self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
4133                                     vnfr_scaled = True
4134                                 try:
4135                                     desc = await self.RO.show("ns", RO_nsr_id)
4136
4137                                     # deploymentStatus
4138                                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
4139
4140                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
4141                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
4142                                     break
4143                                 except LcmExceptionNoMgmtIP:
4144                                     pass
4145                             else:
4146                                 assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
4147                         if detailed_status != detailed_status_old:
4148                             self._update_suboperation_status(
4149                                 db_nslcmop, op_index, 'COMPLETED', detailed_status)
4150                             detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
4151                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
4152
4153                         await asyncio.sleep(5, loop=self.loop)
4154                         deployment_timeout -= 5
4155                     if deployment_timeout <= 0:
4156                         self._update_suboperation_status(
4157                             db_nslcmop, nslcmop_id, op_index, 'FAILED', "Timeout when waiting for ns to get ready")
4158                         raise ROclient.ROClientException("Timeout waiting ns to be ready")
4159
4160                     # update VDU_SCALING_INFO with the obtained ip_addresses
4161                     if vdu_scaling_info["scaling_direction"] == "OUT":
4162                         for vdur in reversed(db_vnfr["vdur"]):
4163                             if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
4164                                 vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
4165                                 vdu_scaling_info["vdu"].append({
4166                                     "name": vdur["name"],
4167                                     "vdu_id": vdur["vdu-id-ref"],
4168                                     "interface": []
4169                                 })
4170                                 for interface in vdur["interfaces"]:
4171                                     vdu_scaling_info["vdu"][-1]["interface"].append({
4172                                         "name": interface["name"],
4173                                         "ip_address": interface["ip-address"],
4174                                         "mac_address": interface.get("mac-address"),
4175                                     })
4176                         del vdu_scaling_info["vdu-create"]
4177
4178                     self._update_suboperation_status(db_nslcmop, op_index, 'COMPLETED', 'Done')
4179             # SCALE RO - END
4180
4181             scale_process = None
4182             if db_nsr_update:
4183                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
4184
4185             # POST-SCALE BEGIN
4186             # execute primitive service POST-SCALING
4187             step = "Executing post-scale vnf-config-primitive"
4188             if scaling_descriptor.get("scaling-config-action"):
4189                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
4190                     if (scaling_config_action.get("trigger") == "post-scale-in" and scaling_type == "SCALE_IN") \
4191                        or (scaling_config_action.get("trigger") == "post-scale-out" and scaling_type == "SCALE_OUT"):
4192                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
4193                         step = db_nslcmop_update["detailed-status"] = \
4194                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
4195
4196                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
4197                         if db_vnfr.get("additionalParamsForVnf"):
4198                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
4199
4200                         # look for primitive
4201                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
4202                             if config_primitive["name"] == vnf_config_primitive:
4203                                 break
4204                         else:
4205                             raise LcmException(
4206                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-"
4207                                 "action[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:"
4208                                 "config-primitive".format(scaling_group, vnf_config_primitive))
4209                         scale_process = "VCA"
4210                         db_nsr_update["config-status"] = "configuring post-scaling"
4211                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
4212
4213                         # Post-scale retry check: Check if this sub-operation has been executed before
4214                         op_index = self._check_or_add_scale_suboperation(
4215                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
4216                         if op_index == self.SUBOPERATION_STATUS_SKIP:
4217                             # Skip sub-operation
4218                             result = 'COMPLETED'
4219                             result_detail = 'Done'
4220                             self.logger.debug(logging_text +
4221                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".
4222                                               format(vnf_config_primitive, result, result_detail))
4223                         else:
4224                             if op_index == self.SUBOPERATION_STATUS_NEW:
4225                                 # New sub-operation: Get index of this sub-operation
4226                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
4227                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
4228                                                   format(vnf_config_primitive))
4229                             else:
4230                                 # retry:  Get registered params for this existing sub-operation
4231                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
4232                                 vnf_index = op.get('member_vnf_index')
4233                                 vnf_config_primitive = op.get('primitive')
4234                                 primitive_params = op.get('primitive_params')
4235                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
4236                                                   format(vnf_config_primitive))
4237                             # Execute the primitive, either with new (first-time) or registered (reintent) args
4238                             ee_descriptor_id = config_primitive.get("execution-environment-ref")
4239                             primitive_name = config_primitive.get("execution-environment-primitive",
4240                                                                   vnf_config_primitive)
4241                             ee_id, vca_type = self._look_for_deployed_vca(nsr_deployed["VCA"],
4242                                                                           member_vnf_index=vnf_index,
4243                                                                           vdu_id=None,
4244                                                                           vdu_count_index=None,
4245                                                                           ee_descriptor_id=ee_descriptor_id)
4246                             result, result_detail = await self._ns_execute_primitive(
4247                                 ee_id, primitive_name, primitive_params, vca_type)
4248                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
4249                                 vnf_config_primitive, result, result_detail))
4250                             # Update operationState = COMPLETED | FAILED
4251                             self._update_suboperation_status(
4252                                 db_nslcmop, op_index, result, result_detail)
4253
4254                         if result == "FAILED":
4255                             raise LcmException(result_detail)
4256                         db_nsr_update["config-status"] = old_config_status
4257                         scale_process = None
4258             # POST-SCALE END
4259
4260             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
4261             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
4262                 else old_operational_status
4263             db_nsr_update["config-status"] = old_config_status
4264             return
4265         except (ROclient.ROClientException, DbException, LcmException) as e:
4266             self.logger.error(logging_text + "Exit Exception {}".format(e))
4267             exc = e
4268         except asyncio.CancelledError:
4269             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
4270             exc = "Operation was cancelled"
4271         except Exception as e:
4272             exc = traceback.format_exc()
4273             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
4274         finally:
4275             self._write_ns_status(
4276                 nsr_id=nsr_id,
4277                 ns_state=None,
4278                 current_operation="IDLE",
4279                 current_operation_id=None
4280             )
4281             if exc:
4282                 db_nslcmop_update["detailed-status"] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
4283                 nslcmop_operation_state = "FAILED"
4284                 if db_nsr:
4285                     db_nsr_update["operational-status"] = old_operational_status
4286                     db_nsr_update["config-status"] = old_config_status
4287                     db_nsr_update["detailed-status"] = ""
4288                     if scale_process:
4289                         if "VCA" in scale_process:
4290                             db_nsr_update["config-status"] = "failed"
4291                         if "RO" in scale_process:
4292                             db_nsr_update["operational-status"] = "failed"
4293                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
4294                                                                                                      exc)
4295             else:
4296                 error_description_nslcmop = None
4297                 nslcmop_operation_state = "COMPLETED"
4298                 db_nslcmop_update["detailed-status"] = "Done"
4299
4300             self._write_op_status(
4301                 op_id=nslcmop_id,
4302                 stage="",
4303                 error_message=error_description_nslcmop,
4304                 operation_state=nslcmop_operation_state,
4305                 other_update=db_nslcmop_update,
4306             )
4307             if db_nsr:
4308                 self._write_ns_status(
4309                     nsr_id=nsr_id,
4310                     ns_state=None,
4311                     current_operation="IDLE",
4312                     current_operation_id=None,
4313                     other_update=db_nsr_update
4314                 )
4315
4316             if nslcmop_operation_state:
4317                 try:
4318                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
4319                                                              "operationState": nslcmop_operation_state},
4320                                             loop=self.loop)
4321                     # if cooldown_time:
4322                     #     await asyncio.sleep(cooldown_time, loop=self.loop)
4323                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
4324                 except Exception as e:
4325                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
4326             self.logger.debug(logging_text + "Exit")
4327             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")
4328
4329     async def add_prometheus_metrics(self, ee_id, artifact_path, ee_config_descriptor, vnfr_id, nsr_id, target_ip):
4330         if not self.prometheus:
4331             return
4332         # look if exist a file called 'prometheus*.j2' and
4333         artifact_content = self.fs.dir_ls(artifact_path)
4334         job_file = next((f for f in artifact_content if f.startswith("prometheus") and f.endswith(".j2")), None)
4335         if not job_file:
4336             return
4337         with self.fs.file_open((artifact_path, job_file), "r") as f:
4338             job_data = f.read()
4339
4340         # TODO get_service
4341         _, _, service = ee_id.partition(".")   # remove prefix   "namespace."
4342         host_name = "{}-{}".format(service, ee_config_descriptor["metric-service"])
4343         host_port = "80"
4344         vnfr_id = vnfr_id.replace("-", "")
4345         variables = {
4346             "JOB_NAME": vnfr_id,
4347             "TARGET_IP": target_ip,
4348             "EXPORTER_POD_IP": host_name,
4349             "EXPORTER_POD_PORT": host_port,
4350         }
4351         job_list = self.prometheus.parse_job(job_data, variables)
4352         # ensure job_name is using the vnfr_id. Adding the metadata nsr_id
4353         for job in job_list:
4354             if not isinstance(job.get("job_name"), str) or vnfr_id not in job["job_name"]:
4355                 job["job_name"] = vnfr_id + "_" + str(randint(1, 10000))
4356             job["nsr_id"] = nsr_id
4357         job_dict = {jl["job_name"]: jl for jl in job_list}
4358         if await self.prometheus.update(job_dict):
4359             return list(job_dict.keys())