osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import traceback
  24 import json
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase, deep_get, get_iterable, populate_dict
  29 from n2vc.k8s_helm_conn import K8sHelmConnector
  30 from n2vc.k8s_juju_conn import K8sJujuConnector
  31
  32 from osm_common.dbbase import DbException
  33 from osm_common.fsbase import FsException
  34
  35 from n2vc.n2vc_juju_conn import N2VCJujuConnector
  36 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
  37
  38 from copy import copy, deepcopy
  39 from http import HTTPStatus
  40 from time import time
  41 from uuid import uuid4
  42 from functools import partial
  43
  44 __author__ = "Alfonso Tierno"
  45
  46
  47 class NsLcm(LcmBase):
  48     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  49     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
  50     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
  51     timeout_charm_delete = 10 * 60
  52     timeout_primitive = 30 * 60  # timeout for primitive execution
  53     timeout_progress_primitive = 10 * 60  # timeout for some progress in a primitive execution
  54
  55     SUBOPERATION_STATUS_NOT_FOUND = -1
  56     SUBOPERATION_STATUS_NEW = -2
  57     SUBOPERATION_STATUS_SKIP = -3
  58     task_name_deploy_vca = "Deploying VCA"
  59
  60     def __init__(self, db, msg, fs, lcm_tasks, config, loop):
  61         """
  62         Init, Connect to database, filesystem storage, and messaging
  63         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  64         :return: None
  65         """
  66         super().__init__(
  67             db=db,
  68             msg=msg,
  69             fs=fs,
  70             logger=logging.getLogger('lcm.ns')
  71         )
  72
  73         self.loop = loop
  74         self.lcm_tasks = lcm_tasks
  75         self.timeout = config["timeout"]
  76         self.ro_config = config["ro_config"]
  77         self.vca_config = config["VCA"].copy()
  78
  79         # create N2VC connector
  80         self.n2vc = N2VCJujuConnector(
  81             db=self.db,
  82             fs=self.fs,
  83             log=self.logger,
  84             loop=self.loop,
  85             url='{}:{}'.format(self.vca_config['host'], self.vca_config['port']),
  86             username=self.vca_config.get('user', None),
  87             vca_config=self.vca_config,
  88             on_update_db=self._on_update_n2vc_db
  89         )
  90
  91         self.k8sclusterhelm = K8sHelmConnector(
  92             kubectl_command=self.vca_config.get("kubectlpath"),
  93             helm_command=self.vca_config.get("helmpath"),
  94             fs=self.fs,
  95             log=self.logger,
  96             db=self.db,
  97             on_update_db=None,
  98         )
  99
 100         self.k8sclusterjuju = K8sJujuConnector(
 101             kubectl_command=self.vca_config.get("kubectlpath"),
 102             juju_command=self.vca_config.get("jujupath"),
 103             fs=self.fs,
 104             log=self.logger,
 105             db=self.db,
 106             on_update_db=None,
 107         )
 108
 109         self.k8scluster_map = {
 110             "helm-chart": self.k8sclusterhelm,
 111             "chart": self.k8sclusterhelm,
 112             "juju-bundle": self.k8sclusterjuju,
 113             "juju": self.k8sclusterjuju,
 114         }
 115         # create RO client
 116         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 117
 118     def _on_update_ro_db(self, nsrs_id, ro_descriptor):
 119
 120         # self.logger.debug('_on_update_ro_db(nsrs_id={}'.format(nsrs_id))
 121
 122         try:
 123             # TODO filter RO descriptor fields...
 124
 125             # write to database
 126             db_dict = dict()
 127             # db_dict['deploymentStatus'] = yaml.dump(ro_descriptor, default_flow_style=False, indent=2)
 128             db_dict['deploymentStatus'] = ro_descriptor
 129             self.update_db_2("nsrs", nsrs_id, db_dict)
 130
 131         except Exception as e:
 132             self.logger.warn('Cannot write database RO deployment for ns={} -> {}'.format(nsrs_id, e))
 133
 134     async def _on_update_n2vc_db(self, table, filter, path, updated_data):
 135
 136         # remove last dot from path (if exists)
 137         if path.endswith('.'):
 138             path = path[:-1]
 139
 140         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 141         #                   .format(table, filter, path, updated_data))
 142
 143         try:
 144
 145             nsr_id = filter.get('_id')
 146
 147             # read ns record from database
 148             nsr = self.db.get_one(table='nsrs', q_filter=filter)
 149             current_ns_status = nsr.get('nsState')
 150
 151             # get vca status for NS
 152             status_dict = await self.n2vc.get_status(namespace='.' + nsr_id, yaml_format=False)
 153
 154             # vcaStatus
 155             db_dict = dict()
 156             db_dict['vcaStatus'] = status_dict
 157
 158             # update configurationStatus for this VCA
 159             try:
 160                 vca_index = int(path[path.rfind(".")+1:])
 161
 162                 vca_list = deep_get(target_dict=nsr, key_list=('_admin', 'deployed', 'VCA'))
 163                 vca_status = vca_list[vca_index].get('status')
 164
 165                 configuration_status_list = nsr.get('configurationStatus')
 166                 config_status = configuration_status_list[vca_index].get('status')
 167
 168                 if config_status == 'BROKEN' and vca_status != 'failed':
 169                     db_dict['configurationStatus'][vca_index] = 'READY'
 170                 elif config_status != 'BROKEN' and vca_status == 'failed':
 171                     db_dict['configurationStatus'][vca_index] = 'BROKEN'
 172             except Exception as e:
 173                 # not update configurationStatus
 174                 self.logger.debug('Error updating vca_index (ignore): {}'.format(e))
 175
 176             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 177             # if nsState = 'DEGRADED' check if all is OK
 178             is_degraded = False
 179             if current_ns_status in ('READY', 'DEGRADED'):
 180                 error_description = ''
 181                 # check machines
 182                 if status_dict.get('machines'):
 183                     for machine_id in status_dict.get('machines'):
 184                         machine = status_dict.get('machines').get(machine_id)
 185                         # check machine agent-status
 186                         if machine.get('agent-status'):
 187                             s = machine.get('agent-status').get('status')
 188                             if s != 'started':
 189                                 is_degraded = True
 190                                 error_description += 'machine {} agent-status={} ; '.format(machine_id, s)
 191                         # check machine instance status
 192                         if machine.get('instance-status'):
 193                             s = machine.get('instance-status').get('status')
 194                             if s != 'running':
 195                                 is_degraded = True
 196                                 error_description += 'machine {} instance-status={} ; '.format(machine_id, s)
 197                 # check applications
 198                 if status_dict.get('applications'):
 199                     for app_id in status_dict.get('applications'):
 200                         app = status_dict.get('applications').get(app_id)
 201                         # check application status
 202                         if app.get('status'):
 203                             s = app.get('status').get('status')
 204                             if s != 'active':
 205                                 is_degraded = True
 206                                 error_description += 'application {} status={} ; '.format(app_id, s)
 207
 208                 if error_description:
 209                     db_dict['errorDescription'] = error_description
 210                 if current_ns_status == 'READY' and is_degraded:
 211                     db_dict['nsState'] = 'DEGRADED'
 212                 if current_ns_status == 'DEGRADED' and not is_degraded:
 213                     db_dict['nsState'] = 'READY'
 214
 215             # write to database
 216             self.update_db_2("nsrs", nsr_id, db_dict)
 217
 218         except (asyncio.CancelledError, asyncio.TimeoutError):
 219             raise
 220         except Exception as e:
 221             self.logger.warn('Error updating NS state for ns={}: {}'.format(nsr_id, e))
 222
 223     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 224         """
 225         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 226         :param vnfd: input vnfd
 227         :param new_id: overrides vnf id if provided
 228         :param additionalParams: Instantiation params for VNFs provided
 229         :param nsrId: Id of the NSR
 230         :return: copy of vnfd
 231         """
 232         try:
 233             vnfd_RO = deepcopy(vnfd)
 234             # remove unused by RO configuration, monitoring, scaling and internal keys
 235             vnfd_RO.pop("_id", None)
 236             vnfd_RO.pop("_admin", None)
 237             vnfd_RO.pop("vnf-configuration", None)
 238             vnfd_RO.pop("monitoring-param", None)
 239             vnfd_RO.pop("scaling-group-descriptor", None)
 240             vnfd_RO.pop("kdu", None)
 241             vnfd_RO.pop("k8s-cluster", None)
 242             if new_id:
 243                 vnfd_RO["id"] = new_id
 244
 245             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 246             for vdu in get_iterable(vnfd_RO, "vdu"):
 247                 cloud_init_file = None
 248                 if vdu.get("cloud-init-file"):
 249                     base_folder = vnfd["_admin"]["storage"]
 250                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 251                                                                    vdu["cloud-init-file"])
 252                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 253                         cloud_init_content = ci_file.read()
 254                     vdu.pop("cloud-init-file", None)
 255                 elif vdu.get("cloud-init"):
 256                     cloud_init_content = vdu["cloud-init"]
 257                 else:
 258                     continue
 259
 260                 env = Environment()
 261                 ast = env.parse(cloud_init_content)
 262                 mandatory_vars = meta.find_undeclared_variables(ast)
 263                 if mandatory_vars:
 264                     for var in mandatory_vars:
 265                         if not additionalParams or var not in additionalParams.keys():
 266                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 267                                                "file, must be provided in the instantiation parameters inside the "
 268                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 269                 template = Template(cloud_init_content)
 270                 cloud_init_content = template.render(additionalParams or {})
 271                 vdu["cloud-init"] = cloud_init_content
 272
 273             return vnfd_RO
 274         except FsException as e:
 275             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 276                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 277         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 278             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 279                                format(vnfd["id"], vdu["id"], e))
 280
 281     def _ns_params_2_RO(self, ns_params, nsd, vnfd_dict, db_vnfrs, n2vc_key_list):
 282         """
 283         Creates a RO ns descriptor from OSM ns_instantiate params
 284         :param ns_params: OSM instantiate params
 285         :param vnfd_dict: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
 286         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index. {member-vnf-index: {vnfr_object}, ...}
 287         :return: The RO ns descriptor
 288         """
 289         vim_2_RO = {}
 290         wim_2_RO = {}
 291         # TODO feature 1417: Check that no instantiation is set over PDU
 292         # check if PDU forces a concrete vim-network-id and add it
 293         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 294
 295         def vim_account_2_RO(vim_account):
 296             if vim_account in vim_2_RO:
 297                 return vim_2_RO[vim_account]
 298
 299             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 300             if db_vim["_admin"]["operationalState"] != "ENABLED":
 301                 raise LcmException("VIM={} is not available. operationalState={}".format(
 302                     vim_account, db_vim["_admin"]["operationalState"]))
 303             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 304             vim_2_RO[vim_account] = RO_vim_id
 305             return RO_vim_id
 306
 307         def wim_account_2_RO(wim_account):
 308             if isinstance(wim_account, str):
 309                 if wim_account in wim_2_RO:
 310                     return wim_2_RO[wim_account]
 311
 312                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 313                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 314                     raise LcmException("WIM={} is not available. operationalState={}".format(
 315                         wim_account, db_wim["_admin"]["operationalState"]))
 316                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 317                 wim_2_RO[wim_account] = RO_wim_id
 318                 return RO_wim_id
 319             else:
 320                 return wim_account
 321
 322         def ip_profile_2_RO(ip_profile):
 323             RO_ip_profile = deepcopy((ip_profile))
 324             if "dns-server" in RO_ip_profile:
 325                 if isinstance(RO_ip_profile["dns-server"], list):
 326                     RO_ip_profile["dns-address"] = []
 327                     for ds in RO_ip_profile.pop("dns-server"):
 328                         RO_ip_profile["dns-address"].append(ds['address'])
 329                 else:
 330                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 331             if RO_ip_profile.get("ip-version") == "ipv4":
 332                 RO_ip_profile["ip-version"] = "IPv4"
 333             if RO_ip_profile.get("ip-version") == "ipv6":
 334                 RO_ip_profile["ip-version"] = "IPv6"
 335             if "dhcp-params" in RO_ip_profile:
 336                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 337             return RO_ip_profile
 338
 339         if not ns_params:
 340             return None
 341         RO_ns_params = {
 342             # "name": ns_params["nsName"],
 343             # "description": ns_params.get("nsDescription"),
 344             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 345             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 346             # "scenario": ns_params["nsdId"],
 347         }
 348         # set vim_account of each vnf if different from general vim_account.
 349         # Get this information from <vnfr> database content, key vim-account-id
 350         # Vim account can be set by placement_engine and it may be different from
 351         # the instantiate parameters (vnfs.member-vnf-index.datacenter).
 352         for vnf_index, vnfr in db_vnfrs.items():
 353             if vnfr.get("vim-account-id") and vnfr["vim-account-id"] != ns_params["vimAccountId"]:
 354                 populate_dict(RO_ns_params, ("vnfs", vnf_index, "datacenter"), vim_account_2_RO(vnfr["vim-account-id"]))
 355
 356         n2vc_key_list = n2vc_key_list or []
 357         for vnfd_ref, vnfd in vnfd_dict.items():
 358             vdu_needed_access = []
 359             mgmt_cp = None
 360             if vnfd.get("vnf-configuration"):
 361                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 362                 if ssh_required and vnfd.get("mgmt-interface"):
 363                     if vnfd["mgmt-interface"].get("vdu-id"):
 364                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 365                     elif vnfd["mgmt-interface"].get("cp"):
 366                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 367
 368             for vdu in vnfd.get("vdu", ()):
 369                 if vdu.get("vdu-configuration"):
 370                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 371                     if ssh_required:
 372                         vdu_needed_access.append(vdu["id"])
 373                 elif mgmt_cp:
 374                     for vdu_interface in vdu.get("interface"):
 375                         if vdu_interface.get("external-connection-point-ref") and \
 376                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 377                             vdu_needed_access.append(vdu["id"])
 378                             mgmt_cp = None
 379                             break
 380
 381             if vdu_needed_access:
 382                 for vnf_member in nsd.get("constituent-vnfd"):
 383                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 384                         continue
 385                     for vdu in vdu_needed_access:
 386                         populate_dict(RO_ns_params,
 387                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 388                                       n2vc_key_list)
 389
 390         if ns_params.get("vduImage"):
 391             RO_ns_params["vduImage"] = ns_params["vduImage"]
 392
 393         if ns_params.get("ssh_keys"):
 394             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 395         for vnf_params in get_iterable(ns_params, "vnf"):
 396             for constituent_vnfd in nsd["constituent-vnfd"]:
 397                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 398                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 399                     break
 400             else:
 401                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 402                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 403
 404             for vdu_params in get_iterable(vnf_params, "vdu"):
 405                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 406                 if vdu_params.get("volume"):
 407                     for volume_params in vdu_params["volume"]:
 408                         if volume_params.get("vim-volume-id"):
 409                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 410                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 411                                           volume_params["vim-volume-id"])
 412                 if vdu_params.get("interface"):
 413                     for interface_params in vdu_params["interface"]:
 414                         if interface_params.get("ip-address"):
 415                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 416                                                          vdu_params["id"], "interfaces", interface_params["name"],
 417                                                          "ip_address"),
 418                                           interface_params["ip-address"])
 419                         if interface_params.get("mac-address"):
 420                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 421                                                          vdu_params["id"], "interfaces", interface_params["name"],
 422                                                          "mac_address"),
 423                                           interface_params["mac-address"])
 424                         if interface_params.get("floating-ip-required"):
 425                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 426                                                          vdu_params["id"], "interfaces", interface_params["name"],
 427                                                          "floating-ip"),
 428                                           interface_params["floating-ip-required"])
 429
 430             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 431                 if internal_vld_params.get("vim-network-name"):
 432                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 433                                                  internal_vld_params["name"], "vim-network-name"),
 434                                   internal_vld_params["vim-network-name"])
 435                 if internal_vld_params.get("vim-network-id"):
 436                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 437                                                  internal_vld_params["name"], "vim-network-id"),
 438                                   internal_vld_params["vim-network-id"])
 439                 if internal_vld_params.get("ip-profile"):
 440                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 441                                                  internal_vld_params["name"], "ip-profile"),
 442                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 443                 if internal_vld_params.get("provider-network"):
 444
 445                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 446                                                  internal_vld_params["name"], "provider-network"),
 447                                   internal_vld_params["provider-network"].copy())
 448
 449                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 450                     # look for interface
 451                     iface_found = False
 452                     for vdu_descriptor in vnf_descriptor["vdu"]:
 453                         for vdu_interface in vdu_descriptor["interface"]:
 454                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 455                                 if icp_params.get("ip-address"):
 456                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 457                                                                  vdu_descriptor["id"], "interfaces",
 458                                                                  vdu_interface["name"], "ip_address"),
 459                                                   icp_params["ip-address"])
 460
 461                                 if icp_params.get("mac-address"):
 462                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 463                                                                  vdu_descriptor["id"], "interfaces",
 464                                                                  vdu_interface["name"], "mac_address"),
 465                                                   icp_params["mac-address"])
 466                                 iface_found = True
 467                                 break
 468                         if iface_found:
 469                             break
 470                     else:
 471                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 472                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 473                                            "connection-point".format(vnf_params["member-vnf-index"],
 474                                                                      icp_params["id-ref"]))
 475
 476         for vld_params in get_iterable(ns_params, "vld"):
 477             if "ip-profile" in vld_params:
 478                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 479                               ip_profile_2_RO(vld_params["ip-profile"]))
 480
 481             if vld_params.get("provider-network"):
 482
 483                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "provider-network"),
 484                               vld_params["provider-network"].copy())
 485
 486             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 487                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 488                               wim_account_2_RO(vld_params["wimAccountId"])),
 489             if vld_params.get("vim-network-name"):
 490                 RO_vld_sites = []
 491                 if isinstance(vld_params["vim-network-name"], dict):
 492                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 493                         RO_vld_sites.append({
 494                             "netmap-use": vim_net,
 495                             "datacenter": vim_account_2_RO(vim_account)
 496                         })
 497                 else:  # isinstance str
 498                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 499                 if RO_vld_sites:
 500                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 501
 502             if vld_params.get("vim-network-id"):
 503                 RO_vld_sites = []
 504                 if isinstance(vld_params["vim-network-id"], dict):
 505                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 506                         RO_vld_sites.append({
 507                             "netmap-use": vim_net,
 508                             "datacenter": vim_account_2_RO(vim_account)
 509                         })
 510                 else:  # isinstance str
 511                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 512                 if RO_vld_sites:
 513                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 514             if vld_params.get("ns-net"):
 515                 if isinstance(vld_params["ns-net"], dict):
 516                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 517                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 518                         populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 519             if "vnfd-connection-point-ref" in vld_params:
 520                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 521                     # look for interface
 522                     for constituent_vnfd in nsd["constituent-vnfd"]:
 523                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 524                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 525                             break
 526                     else:
 527                         raise LcmException(
 528                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 529                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 530                     match_cp = False
 531                     for vdu_descriptor in vnf_descriptor["vdu"]:
 532                         for interface_descriptor in vdu_descriptor["interface"]:
 533                             if interface_descriptor.get("external-connection-point-ref") == \
 534                                     cp_params["vnfd-connection-point-ref"]:
 535                                 match_cp = True
 536                                 break
 537                         if match_cp:
 538                             break
 539                     else:
 540                         raise LcmException(
 541                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 542                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 543                                 cp_params["member-vnf-index-ref"],
 544                                 cp_params["vnfd-connection-point-ref"],
 545                                 vnf_descriptor["id"]))
 546                     if cp_params.get("ip-address"):
 547                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 548                                                      vdu_descriptor["id"], "interfaces",
 549                                                      interface_descriptor["name"], "ip_address"),
 550                                       cp_params["ip-address"])
 551                     if cp_params.get("mac-address"):
 552                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 553                                                      vdu_descriptor["id"], "interfaces",
 554                                                      interface_descriptor["name"], "mac_address"),
 555                                       cp_params["mac-address"])
 556         return RO_ns_params
 557
 558     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 559         # make a copy to do not change
 560         vdu_create = copy(vdu_create)
 561         vdu_delete = copy(vdu_delete)
 562
 563         vdurs = db_vnfr.get("vdur")
 564         if vdurs is None:
 565             vdurs = []
 566         vdu_index = len(vdurs)
 567         while vdu_index:
 568             vdu_index -= 1
 569             vdur = vdurs[vdu_index]
 570             if vdur.get("pdu-type"):
 571                 continue
 572             vdu_id_ref = vdur["vdu-id-ref"]
 573             if vdu_create and vdu_create.get(vdu_id_ref):
 574                 for index in range(0, vdu_create[vdu_id_ref]):
 575                     vdur = deepcopy(vdur)
 576                     vdur["_id"] = str(uuid4())
 577                     vdur["count-index"] += 1
 578                     vdurs.insert(vdu_index+1+index, vdur)
 579                 del vdu_create[vdu_id_ref]
 580             if vdu_delete and vdu_delete.get(vdu_id_ref):
 581                 del vdurs[vdu_index]
 582                 vdu_delete[vdu_id_ref] -= 1
 583                 if not vdu_delete[vdu_id_ref]:
 584                     del vdu_delete[vdu_id_ref]
 585         # check all operations are done
 586         if vdu_create or vdu_delete:
 587             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 588                 vdu_create))
 589         if vdu_delete:
 590             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 591                 vdu_delete))
 592
 593         vnfr_update = {"vdur": vdurs}
 594         db_vnfr["vdur"] = vdurs
 595         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 596
 597     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 598         """
 599         Updates database nsr with the RO info for the created vld
 600         :param ns_update_nsr: dictionary to be filled with the updated info
 601         :param db_nsr: content of db_nsr. This is also modified
 602         :param nsr_desc_RO: nsr descriptor from RO
 603         :return: Nothing, LcmException is raised on errors
 604         """
 605
 606         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 607             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 608                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 609                     continue
 610                 vld["vim-id"] = net_RO.get("vim_net_id")
 611                 vld["name"] = net_RO.get("vim_name")
 612                 vld["status"] = net_RO.get("status")
 613                 vld["status-detailed"] = net_RO.get("error_msg")
 614                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 615                 break
 616             else:
 617                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 618
 619     def set_vnfr_at_error(self, db_vnfrs, error_text):
 620         try:
 621             for db_vnfr in db_vnfrs.values():
 622                 vnfr_update = {"status": "ERROR"}
 623                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 624                     if "status" not in vdur:
 625                         vdur["status"] = "ERROR"
 626                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 627                         if error_text:
 628                             vdur["status-detailed"] = str(error_text)
 629                             vnfr_update["vdur.{}.status-detailed".format(vdu_index)] = "ERROR"
 630                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 631         except DbException as e:
 632             self.logger.error("Cannot update vnf. {}".format(e))
 633
 634     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 635         """
 636         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 637         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 638         :param nsr_desc_RO: nsr descriptor from RO
 639         :return: Nothing, LcmException is raised on errors
 640         """
 641         for vnf_index, db_vnfr in db_vnfrs.items():
 642             for vnf_RO in nsr_desc_RO["vnfs"]:
 643                 if vnf_RO["member_vnf_index"] != vnf_index:
 644                     continue
 645                 vnfr_update = {}
 646                 if vnf_RO.get("ip_address"):
 647                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 648                 elif not db_vnfr.get("ip-address"):
 649                     if db_vnfr.get("vdur"):   # if not VDUs, there is not ip_address
 650                         raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 651
 652                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 653                     vdur_RO_count_index = 0
 654                     if vdur.get("pdu-type"):
 655                         continue
 656                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 657                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 658                             continue
 659                         if vdur["count-index"] != vdur_RO_count_index:
 660                             vdur_RO_count_index += 1
 661                             continue
 662                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 663                         if vdur_RO.get("ip_address"):
 664                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 665                         else:
 666                             vdur["ip-address"] = None
 667                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 668                         vdur["name"] = vdur_RO.get("vim_name")
 669                         vdur["status"] = vdur_RO.get("status")
 670                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 671                         for ifacer in get_iterable(vdur, "interfaces"):
 672                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 673                                 if ifacer["name"] == interface_RO.get("internal_name"):
 674                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 675                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 676                                     break
 677                             else:
 678                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 679                                                    "from VIM info"
 680                                                    .format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 681                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 682                         break
 683                     else:
 684                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 685                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 686
 687                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 688                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 689                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 690                             continue
 691                         vld["vim-id"] = net_RO.get("vim_net_id")
 692                         vld["name"] = net_RO.get("vim_name")
 693                         vld["status"] = net_RO.get("status")
 694                         vld["status-detailed"] = net_RO.get("error_msg")
 695                         vnfr_update["vld.{}".format(vld_index)] = vld
 696                         break
 697                     else:
 698                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 699                             vnf_index, vld["id"]))
 700
 701                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 702                 break
 703
 704             else:
 705                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 706
 707     def _get_ns_config_info(self, nsr_id):
 708         """
 709         Generates a mapping between vnf,vdu elements and the N2VC id
 710         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 711         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 712             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 713             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 714         """
 715         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 716         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 717         mapping = {}
 718         ns_config_info = {"osm-config-mapping": mapping}
 719         for vca in vca_deployed_list:
 720             if not vca["member-vnf-index"]:
 721                 continue
 722             if not vca["vdu_id"]:
 723                 mapping[vca["member-vnf-index"]] = vca["application"]
 724             else:
 725                 mapping["{}.{}.{}".format(vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"])] =\
 726                     vca["application"]
 727         return ns_config_info
 728
 729     @staticmethod
 730     def _get_initial_config_primitive_list(desc_primitive_list, vca_deployed):
 731         """
 732         Generates a list of initial-config-primitive based on the list provided by the descriptor. It includes internal
 733         primitives as verify-ssh-credentials, or config when needed
 734         :param desc_primitive_list: information of the descriptor
 735         :param vca_deployed: information of the deployed, needed for known if it is related to an NS, VNF, VDU and if
 736             this element contains a ssh public key
 737         :return: The modified list. Can ba an empty list, but always a list
 738         """
 739         if desc_primitive_list:
 740             primitive_list = desc_primitive_list.copy()
 741         else:
 742             primitive_list = []
 743         # look for primitive config, and get the position. None if not present
 744         config_position = None
 745         for index, primitive in enumerate(primitive_list):
 746             if primitive["name"] == "config":
 747                 config_position = index
 748                 break
 749
 750         # for NS, add always a config primitive if not present (bug 874)
 751         if not vca_deployed["member-vnf-index"] and config_position is None:
 752             primitive_list.insert(0, {"name": "config", "parameter": []})
 753             config_position = 0
 754         # for VNF/VDU add verify-ssh-credentials after config
 755         if vca_deployed["member-vnf-index"] and config_position is not None and vca_deployed.get("ssh-public-key"):
 756             primitive_list.insert(config_position + 1, {"name": "verify-ssh-credentials", "parameter": []})
 757         return primitive_list
 758
 759     async def instantiate_RO(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
 760                              n2vc_key_list, stage):
 761         """
 762         Instantiate at RO
 763         :param logging_text: preffix text to use at logging
 764         :param nsr_id: nsr identity
 765         :param nsd: database content of ns descriptor
 766         :param db_nsr: database content of ns record
 767         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
 768         :param db_vnfrs:
 769         :param db_vnfds_ref: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
 770         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
 771         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
 772         :return: None or exception
 773         """
 774         try:
 775             db_nsr_update = {}
 776             RO_descriptor_number = 0   # number of descriptors created at RO
 777             vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 778             nslcmop_id = db_nslcmop["_id"]
 779             start_deploy = time()
 780             ns_params = db_nslcmop.get("operationParams")
 781             if ns_params and ns_params.get("timeout_ns_deploy"):
 782                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
 783             else:
 784                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
 785
 786             # Check for and optionally request placement optimization. Database will be updated if placement activated
 787             stage[2] = "Waiting for Placement."
 788             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
 789                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
 790                 for vnfr in db_vnfrs.values():
 791                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
 792                         break
 793                 else:
 794                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
 795
 796             # deploy RO
 797             # get vnfds, instantiate at RO
 798             for c_vnf in nsd.get("constituent-vnfd", ()):
 799                 member_vnf_index = c_vnf["member-vnf-index"]
 800                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 801                 vnfd_ref = vnfd["id"]
 802
 803                 stage[2] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(vnfd_ref, member_vnf_index)
 804                 db_nsr_update["detailed-status"] = " ".join(stage)
 805                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 806                 self._write_op_status(nslcmop_id, stage)
 807
 808                 # self.logger.debug(logging_text + stage[2])
 809                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 810                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 811                 RO_descriptor_number += 1
 812
 813                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 814                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 815                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 816                         break
 817                 else:
 818                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
 819                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
 820
 821                 # look if present
 822                 RO_update = {"member-vnf-index": member_vnf_index}
 823                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
 824                 if vnfd_list:
 825                     RO_update["id"] = vnfd_list[0]["uuid"]
 826                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
 827                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
 828                 else:
 829                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
 830                                            get("additionalParamsForVnf"), nsr_id)
 831                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
 832                     RO_update["id"] = desc["uuid"]
 833                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
 834                         vnfd_ref, member_vnf_index, desc["uuid"]))
 835                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
 836                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
 837
 838             # create nsd at RO
 839             nsd_ref = nsd["id"]
 840
 841             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
 842             db_nsr_update["detailed-status"] = " ".join(stage)
 843             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 844             self._write_op_status(nslcmop_id, stage)
 845
 846             # self.logger.debug(logging_text + stage[2])
 847             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
 848             RO_descriptor_number += 1
 849             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
 850             if nsd_list:
 851                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
 852                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
 853                     nsd_ref, RO_nsd_uuid))
 854             else:
 855                 nsd_RO = deepcopy(nsd)
 856                 nsd_RO["id"] = RO_osm_nsd_id
 857                 nsd_RO.pop("_id", None)
 858                 nsd_RO.pop("_admin", None)
 859                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
 860                     member_vnf_index = c_vnf["member-vnf-index"]
 861                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 862                 for c_vld in nsd_RO.get("vld", ()):
 863                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
 864                         member_vnf_index = cp["member-vnf-index-ref"]
 865                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 866
 867                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
 868                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 869                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
 870                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
 871             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 872
 873             # Crate ns at RO
 874             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
 875             db_nsr_update["detailed-status"] = " ".join(stage)
 876             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 877             self._write_op_status(nslcmop_id, stage)
 878
 879             # if present use it unless in error status
 880             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
 881             if RO_nsr_id:
 882                 try:
 883                     stage[2] = "Looking for existing ns at RO"
 884                     db_nsr_update["detailed-status"] = " ".join(stage)
 885                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 886                     self._write_op_status(nslcmop_id, stage)
 887                     # self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
 888                     desc = await self.RO.show("ns", RO_nsr_id)
 889
 890                 except ROclient.ROClientException as e:
 891                     if e.http_code != HTTPStatus.NOT_FOUND:
 892                         raise
 893                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 894                 if RO_nsr_id:
 895                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
 896                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 897                     if ns_status == "ERROR":
 898                         stage[2] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
 899                         self.logger.debug(logging_text + stage[2])
 900                         await self.RO.delete("ns", RO_nsr_id)
 901                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 902             if not RO_nsr_id:
 903                 stage[2] = "Checking dependencies"
 904                 db_nsr_update["detailed-status"] = " ".join(stage)
 905                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 906                 self._write_op_status(nslcmop_id, stage)
 907                 # self.logger.debug(logging_text + stage[2])
 908
 909                 # check if VIM is creating and wait  look if previous tasks in process
 910                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
 911                 if task_dependency:
 912                     stage[2] = "Waiting for related tasks '{}' to be completed".format(task_name)
 913                     self.logger.debug(logging_text + stage[2])
 914                     await asyncio.wait(task_dependency, timeout=3600)
 915                 if ns_params.get("vnf"):
 916                     for vnf in ns_params["vnf"]:
 917                         if "vimAccountId" in vnf:
 918                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 919                                                                                         vnf["vimAccountId"])
 920                         if task_dependency:
 921                             stage[2] = "Waiting for related tasks '{}' to be completed.".format(task_name)
 922                             self.logger.debug(logging_text + stage[2])
 923                             await asyncio.wait(task_dependency, timeout=3600)
 924
 925                 stage[2] = "Checking instantiation parameters."
 926                 RO_ns_params = self._ns_params_2_RO(ns_params, nsd, db_vnfds_ref, db_vnfrs, n2vc_key_list)
 927                 stage[2] = "Deploying ns at VIM."
 928                 db_nsr_update["detailed-status"] = " ".join(stage)
 929                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 930                 self._write_op_status(nslcmop_id, stage)
 931
 932                 desc = await self.RO.create("ns", descriptor=RO_ns_params, name=db_nsr["name"], scenario=RO_nsd_uuid)
 933                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
 934                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 935                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
 936                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 937
 938             # wait until NS is ready
 939             stage[2] = "Waiting VIM to deploy ns."
 940             db_nsr_update["detailed-status"] = " ".join(stage)
 941             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 942             self._write_op_status(nslcmop_id, stage)
 943             detailed_status_old = None
 944             self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
 945
 946             old_desc = None
 947             while time() <= start_deploy + timeout_ns_deploy:
 948                 desc = await self.RO.show("ns", RO_nsr_id)
 949
 950                 # deploymentStatus
 951                 if desc != old_desc:
 952                     # desc has changed => update db
 953                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
 954                     old_desc = desc
 955
 956                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
 957                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 958                     if ns_status == "ERROR":
 959                         raise ROclient.ROClientException(ns_status_info)
 960                     elif ns_status == "BUILD":
 961                         stage[2] = "VIM: ({})".format(ns_status_info)
 962                     elif ns_status == "ACTIVE":
 963                         stage[2] = "Waiting for management IP address reported by the VIM. Updating VNFRs."
 964                         try:
 965                             self.ns_update_vnfr(db_vnfrs, desc)
 966                             break
 967                         except LcmExceptionNoMgmtIP:
 968                             pass
 969                     else:
 970                         assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 971                     if stage[2] != detailed_status_old:
 972                         detailed_status_old = stage[2]
 973                         db_nsr_update["detailed-status"] = " ".join(stage)
 974                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 975                         self._write_op_status(nslcmop_id, stage)
 976                     await asyncio.sleep(5, loop=self.loop)
 977             else:  # timeout_ns_deploy
 978                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 979
 980             # Updating NSR
 981             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 982
 983             db_nsr_update["_admin.deployed.RO.operational-status"] = "running"
 984             # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
 985             stage[2] = "Deployed at VIM"
 986             db_nsr_update["detailed-status"] = " ".join(stage)
 987             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 988             self._write_op_status(nslcmop_id, stage)
 989             # await self._on_update_n2vc_db("nsrs", {"_id": nsr_id}, "_admin.deployed", db_nsr_update)
 990             # self.logger.debug(logging_text + "Deployed at VIM")
 991         except (ROclient.ROClientException, LcmException, DbException) as e:
 992             stage[2] = "ERROR deploying at VIM"
 993             self.set_vnfr_at_error(db_vnfrs, str(e))
 994             raise
 995
 996     async def wait_vm_up_insert_key_ro(self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None):
 997         """
 998         Wait for ip addres at RO, and optionally, insert public key in virtual machine
 999         :param logging_text: prefix use for logging
1000         :param nsr_id:
1001         :param vnfr_id:
1002         :param vdu_id:
1003         :param vdu_index:
1004         :param pub_key: public ssh key to inject, None to skip
1005         :param user: user to apply the public ssh key
1006         :return: IP address
1007         """
1008
1009         # self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1010         ro_nsr_id = None
1011         ip_address = None
1012         nb_tries = 0
1013         target_vdu_id = None
1014         ro_retries = 0
1015
1016         while True:
1017
1018             ro_retries += 1
1019             if ro_retries >= 360:  # 1 hour
1020                 raise LcmException("Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id))
1021
1022             await asyncio.sleep(10, loop=self.loop)
1023
1024             # get ip address
1025             if not target_vdu_id:
1026                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1027
1028                 if not vdu_id:  # for the VNF case
1029                     if db_vnfr.get("status") == "ERROR":
1030                         raise LcmException("Cannot inject ssh-key because target VNF is in error state")
1031                     ip_address = db_vnfr.get("ip-address")
1032                     if not ip_address:
1033                         continue
1034                     vdur = next((x for x in get_iterable(db_vnfr, "vdur") if x.get("ip-address") == ip_address), None)
1035                 else:  # VDU case
1036                     vdur = next((x for x in get_iterable(db_vnfr, "vdur")
1037                                  if x.get("vdu-id-ref") == vdu_id and x.get("count-index") == vdu_index), None)
1038
1039                 if not vdur and len(db_vnfr.get("vdur", ())) == 1:  # If only one, this should be the target vdu
1040                     vdur = db_vnfr["vdur"][0]
1041                 if not vdur:
1042                     raise LcmException("Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(vnfr_id, vdu_id,
1043                                                                                               vdu_index))
1044
1045                 if vdur.get("pdu-type") or vdur.get("status") == "ACTIVE":
1046                     ip_address = vdur.get("ip-address")
1047                     if not ip_address:
1048                         continue
1049                     target_vdu_id = vdur["vdu-id-ref"]
1050                 elif vdur.get("status") == "ERROR":
1051                     raise LcmException("Cannot inject ssh-key because target VM is in error state")
1052
1053             if not target_vdu_id:
1054                 continue
1055
1056             # inject public key into machine
1057             if pub_key and user:
1058                 # wait until NS is deployed at RO
1059                 if not ro_nsr_id:
1060                     db_nsrs = self.db.get_one("nsrs", {"_id": nsr_id})
1061                     ro_nsr_id = deep_get(db_nsrs, ("_admin", "deployed", "RO", "nsr_id"))
1062                 if not ro_nsr_id:
1063                     continue
1064
1065                 # self.logger.debug(logging_text + "Inserting RO key")
1066                 if vdur.get("pdu-type"):
1067                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1068                     return ip_address
1069                 try:
1070                     ro_vm_id = "{}-{}".format(db_vnfr["member-vnf-index-ref"], target_vdu_id)  # TODO add vdu_index
1071                     result_dict = await self.RO.create_action(
1072                         item="ns",
1073                         item_id_name=ro_nsr_id,
1074                         descriptor={"add_public_key": pub_key, "vms": [ro_vm_id], "user": user}
1075                     )
1076                     # result_dict contains the format {VM-id: {vim_result: 200, description: text}}
1077                     if not result_dict or not isinstance(result_dict, dict):
1078                         raise LcmException("Unknown response from RO when injecting key")
1079                     for result in result_dict.values():
1080                         if result.get("vim_result") == 200:
1081                             break
1082                         else:
1083                             raise ROclient.ROClientException("error injecting key: {}".format(
1084                                 result.get("description")))
1085                     break
1086                 except ROclient.ROClientException as e:
1087                     if not nb_tries:
1088                         self.logger.debug(logging_text + "error injecting key: {}. Retrying until {} seconds".
1089                                           format(e, 20*10))
1090                     nb_tries += 1
1091                     if nb_tries >= 20:
1092                         raise LcmException("Reaching max tries injecting key. Error: {}".format(e))
1093             else:
1094                 break
1095
1096         return ip_address
1097
1098     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1099         """
1100         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1101         """
1102         my_vca = vca_deployed_list[vca_index]
1103         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1104             # vdu or kdu: no dependencies
1105             return
1106         timeout = 300
1107         while timeout >= 0:
1108             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1109             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1110             configuration_status_list = db_nsr["configurationStatus"]
1111             for index, vca_deployed in enumerate(configuration_status_list):
1112                 if index == vca_index:
1113                     # myself
1114                     continue
1115                 if not my_vca.get("member-vnf-index") or \
1116                         (vca_deployed.get("member-vnf-index") == my_vca.get("member-vnf-index")):
1117                     internal_status = configuration_status_list[index].get("status")
1118                     if internal_status == 'READY':
1119                         continue
1120                     elif internal_status == 'BROKEN':
1121                         raise LcmException("Configuration aborted because dependent charm/s has failed")
1122                     else:
1123                         break
1124             else:
1125                 # no dependencies, return
1126                 return
1127             await asyncio.sleep(10)
1128             timeout -= 1
1129
1130         raise LcmException("Configuration aborted because dependent charm/s timeout")
1131
1132     async def instantiate_N2VC(self, logging_text, vca_index, nsi_id, db_nsr, db_vnfr, vdu_id, kdu_name, vdu_index,
1133                                config_descriptor, deploy_params, base_folder, nslcmop_id, stage):
1134         nsr_id = db_nsr["_id"]
1135         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1136         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1137         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1138         db_dict = {
1139             'collection': 'nsrs',
1140             'filter': {'_id': nsr_id},
1141             'path': db_update_entry
1142         }
1143         step = ""
1144         try:
1145
1146             element_type = 'NS'
1147             element_under_configuration = nsr_id
1148
1149             vnfr_id = None
1150             if db_vnfr:
1151                 vnfr_id = db_vnfr["_id"]
1152
1153             namespace = "{nsi}.{ns}".format(
1154                 nsi=nsi_id if nsi_id else "",
1155                 ns=nsr_id)
1156
1157             if vnfr_id:
1158                 element_type = 'VNF'
1159                 element_under_configuration = vnfr_id
1160                 namespace += ".{}".format(vnfr_id)
1161                 if vdu_id:
1162                     namespace += ".{}-{}".format(vdu_id, vdu_index or 0)
1163                     element_type = 'VDU'
1164                     element_under_configuration = "{}-{}".format(vdu_id, vdu_index or 0)
1165                 elif kdu_name:
1166                     namespace += ".{}".format(kdu_name)
1167                     element_type = 'KDU'
1168                     element_under_configuration = kdu_name
1169
1170             # Get artifact path
1171             artifact_path = "{}/{}/charms/{}".format(
1172                 base_folder["folder"],
1173                 base_folder["pkg-dir"],
1174                 config_descriptor["juju"]["charm"]
1175             )
1176
1177             is_proxy_charm = deep_get(config_descriptor, ('juju', 'charm')) is not None
1178             if deep_get(config_descriptor, ('juju', 'proxy')) is False:
1179                 is_proxy_charm = False
1180
1181             # n2vc_redesign STEP 3.1
1182
1183             # find old ee_id if exists
1184             ee_id = vca_deployed.get("ee_id")
1185
1186             # create or register execution environment in VCA
1187             if is_proxy_charm:
1188
1189                 self._write_configuration_status(
1190                     nsr_id=nsr_id,
1191                     vca_index=vca_index,
1192                     status='CREATING',
1193                     element_under_configuration=element_under_configuration,
1194                     element_type=element_type
1195                 )
1196
1197                 step = "create execution environment"
1198                 self.logger.debug(logging_text + step)
1199                 ee_id, credentials = await self.n2vc.create_execution_environment(namespace=namespace,
1200                                                                                   reuse_ee_id=ee_id,
1201                                                                                   db_dict=db_dict)
1202
1203             else:
1204                 step = "Waiting to VM being up and getting IP address"
1205                 self.logger.debug(logging_text + step)
1206                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
1207                                                                  user=None, pub_key=None)
1208                 credentials = {"hostname": rw_mgmt_ip}
1209                 # get username
1210                 username = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1211                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1212                 #  merged. Meanwhile let's get username from initial-config-primitive
1213                 if not username and config_descriptor.get("initial-config-primitive"):
1214                     for config_primitive in config_descriptor["initial-config-primitive"]:
1215                         for param in config_primitive.get("parameter", ()):
1216                             if param["name"] == "ssh-username":
1217                                 username = param["value"]
1218                                 break
1219                 if not username:
1220                     raise LcmException("Cannot determine the username neither with 'initial-config-promitive' nor with "
1221                                        "'config-access.ssh-access.default-user'")
1222                 credentials["username"] = username
1223                 # n2vc_redesign STEP 3.2
1224
1225                 self._write_configuration_status(
1226                     nsr_id=nsr_id,
1227                     vca_index=vca_index,
1228                     status='REGISTERING',
1229                     element_under_configuration=element_under_configuration,
1230                     element_type=element_type
1231                 )
1232
1233                 step = "register execution environment {}".format(credentials)
1234                 self.logger.debug(logging_text + step)
1235                 ee_id = await self.n2vc.register_execution_environment(credentials=credentials, namespace=namespace,
1236                                                                        db_dict=db_dict)
1237
1238             # for compatibility with MON/POL modules, the need model and application name at database
1239             # TODO ask to N2VC instead of assuming the format "model_name.application_name"
1240             ee_id_parts = ee_id.split('.')
1241             model_name = ee_id_parts[0]
1242             application_name = ee_id_parts[1]
1243             db_nsr_update = {db_update_entry + "model": model_name,
1244                              db_update_entry + "application": application_name,
1245                              db_update_entry + "ee_id": ee_id}
1246
1247             # n2vc_redesign STEP 3.3
1248
1249             step = "Install configuration Software"
1250
1251             self._write_configuration_status(
1252                 nsr_id=nsr_id,
1253                 vca_index=vca_index,
1254                 status='INSTALLING SW',
1255                 element_under_configuration=element_under_configuration,
1256                 element_type=element_type,
1257                 other_update=db_nsr_update
1258             )
1259
1260             # TODO check if already done
1261             self.logger.debug(logging_text + step)
1262             config = None
1263             if not is_proxy_charm:
1264                 initial_config_primitive_list = config_descriptor.get('initial-config-primitive')
1265                 if initial_config_primitive_list:
1266                     for primitive in initial_config_primitive_list:
1267                         if primitive["name"] == "config":
1268                             config = self._map_primitive_params(
1269                                 primitive,
1270                                 {},
1271                                 deploy_params
1272                             )
1273                             break
1274             num_units = 1
1275             if is_proxy_charm:
1276                 if element_type == "NS":
1277                     num_units = db_nsr.get("config-units") or 1
1278                 elif element_type == "VNF":
1279                     num_units = db_vnfr.get("config-units") or 1
1280                 elif element_type == "VDU":
1281                     for v in db_vnfr["vdur"]:
1282                         if vdu_id == v["vdu-id-ref"]:
1283                             num_units = v.get("config-units") or 1
1284                             break
1285
1286             await self.n2vc.install_configuration_sw(
1287                 ee_id=ee_id,
1288                 artifact_path=artifact_path,
1289                 db_dict=db_dict,
1290                 config=config,
1291                 num_units=num_units
1292             )
1293
1294             # write in db flag of configuration_sw already installed
1295             self.update_db_2("nsrs", nsr_id, {db_update_entry + "config_sw_installed": True})
1296
1297             # add relations for this VCA (wait for other peers related with this VCA)
1298             await self._add_vca_relations(logging_text=logging_text, nsr_id=nsr_id, vca_index=vca_index)
1299
1300             # if SSH access is required, then get execution environment SSH public
1301             if is_proxy_charm:  # if native charm we have waited already to VM be UP
1302                 pub_key = None
1303                 user = None
1304                 if deep_get(config_descriptor, ("config-access", "ssh-access", "required")):
1305                     # Needed to inject a ssh key
1306                     user = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1307                     step = "Install configuration Software, getting public ssh key"
1308                     pub_key = await self.n2vc.get_ee_ssh_public__key(ee_id=ee_id, db_dict=db_dict)
1309
1310                     step = "Insert public key into VM user={} ssh_key={}".format(user, pub_key)
1311                 else:
1312                     step = "Waiting to VM being up and getting IP address"
1313                 self.logger.debug(logging_text + step)
1314
1315                 # n2vc_redesign STEP 5.1
1316                 # wait for RO (ip-address) Insert pub_key into VM
1317                 if vnfr_id:
1318                     rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
1319                                                                      user=user, pub_key=pub_key)
1320                 else:
1321                     rw_mgmt_ip = None   # This is for a NS configuration
1322
1323                 self.logger.debug(logging_text + ' VM_ip_address={}'.format(rw_mgmt_ip))
1324
1325             # store rw_mgmt_ip in deploy params for later replacement
1326             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
1327
1328             # n2vc_redesign STEP 6  Execute initial config primitive
1329             step = 'execute initial config primitive'
1330             initial_config_primitive_list = config_descriptor.get('initial-config-primitive')
1331
1332             # sort initial config primitives by 'seq'
1333             if initial_config_primitive_list:
1334                 try:
1335                     initial_config_primitive_list.sort(key=lambda val: int(val['seq']))
1336                 except Exception as e:
1337                     self.logger.error(logging_text + step + ": " + str(e))
1338             else:
1339                 self.logger.debug(logging_text + step + ": No initial-config-primitive")
1340
1341             # add config if not present for NS charm
1342             initial_config_primitive_list = self._get_initial_config_primitive_list(initial_config_primitive_list,
1343                                                                                     vca_deployed)
1344
1345             # wait for dependent primitives execution (NS -> VNF -> VDU)
1346             if initial_config_primitive_list:
1347                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
1348
1349             # stage, in function of element type: vdu, kdu, vnf or ns
1350             my_vca = vca_deployed_list[vca_index]
1351             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1352                 # VDU or KDU
1353                 stage[0] = 'Stage 3/5: running Day-1 primitives for VDU.'
1354             elif my_vca.get("member-vnf-index"):
1355                 # VNF
1356                 stage[0] = 'Stage 4/5: running Day-1 primitives for VNF.'
1357             else:
1358                 # NS
1359                 stage[0] = 'Stage 5/5: running Day-1 primitives for NS.'
1360
1361             self._write_configuration_status(
1362                 nsr_id=nsr_id,
1363                 vca_index=vca_index,
1364                 status='EXECUTING PRIMITIVE'
1365             )
1366
1367             self._write_op_status(
1368                 op_id=nslcmop_id,
1369                 stage=stage
1370             )
1371
1372             check_if_terminated_needed = True
1373             for initial_config_primitive in initial_config_primitive_list:
1374                 # adding information on the vca_deployed if it is a NS execution environment
1375                 if not vca_deployed["member-vnf-index"]:
1376                     deploy_params["ns_config_info"] = json.dumps(self._get_ns_config_info(nsr_id))
1377                 # TODO check if already done
1378                 primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, deploy_params)
1379
1380                 step = "execute primitive '{}' params '{}'".format(initial_config_primitive["name"], primitive_params_)
1381                 self.logger.debug(logging_text + step)
1382                 await self.n2vc.exec_primitive(
1383                     ee_id=ee_id,
1384                     primitive_name=initial_config_primitive["name"],
1385                     params_dict=primitive_params_,
1386                     db_dict=db_dict
1387                 )
1388                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
1389                 if check_if_terminated_needed:
1390                     if config_descriptor.get('terminate-config-primitive'):
1391                         self.update_db_2("nsrs", nsr_id, {db_update_entry + "needed_terminate": True})
1392                     check_if_terminated_needed = False
1393
1394                 # TODO register in database that primitive is done
1395
1396             step = "instantiated at VCA"
1397             self.logger.debug(logging_text + step)
1398
1399             self._write_configuration_status(
1400                 nsr_id=nsr_id,
1401                 vca_index=vca_index,
1402                 status='READY'
1403             )
1404
1405         except Exception as e:  # TODO not use Exception but N2VC exception
1406             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
1407             if not isinstance(e, (DbException, N2VCException, LcmException, asyncio.CancelledError)):
1408                 self.logger.error("Exception while {} : {}".format(step, e), exc_info=True)
1409             self._write_configuration_status(
1410                 nsr_id=nsr_id,
1411                 vca_index=vca_index,
1412                 status='BROKEN'
1413             )
1414             raise LcmException("{} {}".format(step, e)) from e
1415
1416     def _write_ns_status(self, nsr_id: str, ns_state: str, current_operation: str, current_operation_id: str,
1417                          error_description: str = None, error_detail: str = None, other_update: dict = None):
1418         """
1419         Update db_nsr fields.
1420         :param nsr_id:
1421         :param ns_state:
1422         :param current_operation:
1423         :param current_operation_id:
1424         :param error_description:
1425         :param error_detail:
1426         :param other_update: Other required changes at database if provided, will be cleared
1427         :return:
1428         """
1429         try:
1430             db_dict = other_update or {}
1431             db_dict["_admin.nslcmop"] = current_operation_id    # for backward compatibility
1432             db_dict["_admin.current-operation"] = current_operation_id
1433             db_dict["_admin.operation-type"] = current_operation if current_operation != "IDLE" else None
1434             db_dict["currentOperation"] = current_operation
1435             db_dict["currentOperationID"] = current_operation_id
1436             db_dict["errorDescription"] = error_description
1437             db_dict["errorDetail"] = error_detail
1438
1439             if ns_state:
1440                 db_dict["nsState"] = ns_state
1441             self.update_db_2("nsrs", nsr_id, db_dict)
1442         except DbException as e:
1443             self.logger.warn('Error writing NS status, ns={}: {}'.format(nsr_id, e))
1444
1445     def _write_op_status(self, op_id: str, stage: list = None, error_message: str = None, queuePosition: int = 0,
1446                          operation_state: str = None, other_update: dict = None):
1447         try:
1448             db_dict = other_update or {}
1449             db_dict['queuePosition'] = queuePosition
1450             if isinstance(stage, list):
1451                 db_dict['stage'] = stage[0]
1452                 db_dict['detailed-status'] = " ".join(stage)
1453             elif stage is not None:
1454                 db_dict['stage'] = str(stage)
1455
1456             if error_message is not None:
1457                 db_dict['errorMessage'] = error_message
1458             if operation_state is not None:
1459                 db_dict['operationState'] = operation_state
1460                 db_dict["statusEnteredTime"] = time()
1461             self.update_db_2("nslcmops", op_id, db_dict)
1462         except DbException as e:
1463             self.logger.warn('Error writing OPERATION status for op_id: {} -> {}'.format(op_id, e))
1464
1465     def _write_all_config_status(self, db_nsr: dict, status: str):
1466         try:
1467             nsr_id = db_nsr["_id"]
1468             # configurationStatus
1469             config_status = db_nsr.get('configurationStatus')
1470             if config_status:
1471                 db_nsr_update = {"configurationStatus.{}.status".format(index): status for index, v in
1472                                  enumerate(config_status) if v}
1473                 # update status
1474                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1475
1476         except DbException as e:
1477             self.logger.warn('Error writing all configuration status, ns={}: {}'.format(nsr_id, e))
1478
1479     def _write_configuration_status(self, nsr_id: str, vca_index: int, status: str = None,
1480                                     element_under_configuration: str = None, element_type: str = None,
1481                                     other_update: dict = None):
1482
1483         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
1484         #                   .format(vca_index, status))
1485
1486         try:
1487             db_path = 'configurationStatus.{}.'.format(vca_index)
1488             db_dict = other_update or {}
1489             if status:
1490                 db_dict[db_path + 'status'] = status
1491             if element_under_configuration:
1492                 db_dict[db_path + 'elementUnderConfiguration'] = element_under_configuration
1493             if element_type:
1494                 db_dict[db_path + 'elementType'] = element_type
1495             self.update_db_2("nsrs", nsr_id, db_dict)
1496         except DbException as e:
1497             self.logger.warn('Error writing configuration status={}, ns={}, vca_index={}: {}'
1498                              .format(status, nsr_id, vca_index, e))
1499
1500     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
1501         """
1502         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
1503         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
1504         Database is used because the result can be obtained from a different LCM worker in case of HA.
1505         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
1506         :param db_nslcmop: database content of nslcmop
1507         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
1508         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
1509             computed 'vim-account-id'
1510         """
1511         modified = False
1512         nslcmop_id = db_nslcmop['_id']
1513         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
1514         if placement_engine == "PLA":
1515             self.logger.debug(logging_text + "Invoke and wait for placement optimization")
1516             await self.msg.aiowrite("pla", "get_placement", {'nslcmopId': nslcmop_id}, loop=self.loop)
1517             db_poll_interval = 5
1518             wait = db_poll_interval * 10
1519             pla_result = None
1520             while not pla_result and wait >= 0:
1521                 await asyncio.sleep(db_poll_interval)
1522                 wait -= db_poll_interval
1523                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1524                 pla_result = deep_get(db_nslcmop, ('_admin', 'pla'))
1525
1526             if not pla_result:
1527                 raise LcmException("Placement timeout for nslcmopId={}".format(nslcmop_id))
1528
1529             for pla_vnf in pla_result['vnf']:
1530                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
1531                 if not pla_vnf.get('vimAccountId') or not vnfr:
1532                     continue
1533                 modified = True
1534                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})
1535                 # Modifies db_vnfrs
1536                 vnfr["vim-account-id"] = pla_vnf['vimAccountId']
1537         return modified
1538
1539     def update_nsrs_with_pla_result(self, params):
1540         try:
1541             nslcmop_id = deep_get(params, ('placement', 'nslcmopId'))
1542             self.update_db_2("nslcmops", nslcmop_id, {"_admin.pla": params.get('placement')})
1543         except Exception as e:
1544             self.logger.warn('Update failed for nslcmop_id={}:{}'.format(nslcmop_id, e))
1545
1546     async def instantiate(self, nsr_id, nslcmop_id):
1547         """
1548
1549         :param nsr_id: ns instance to deploy
1550         :param nslcmop_id: operation to run
1551         :return:
1552         """
1553
1554         # Try to lock HA task here
1555         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1556         if not task_is_locked_by_me:
1557             self.logger.debug('instantiate() task is not locked by me, ns={}'.format(nsr_id))
1558             return
1559
1560         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
1561         self.logger.debug(logging_text + "Enter")
1562
1563         # Sync from FSMongo
1564         self.fs.sync()
1565
1566         # get all needed from database
1567
1568         # database nsrs record
1569         db_nsr = None
1570
1571         # database nslcmops record
1572         db_nslcmop = None
1573
1574         # update operation on nsrs
1575         db_nsr_update = {}
1576         # update operation on nslcmops
1577         db_nslcmop_update = {}
1578
1579         nslcmop_operation_state = None
1580         db_vnfrs = {}     # vnf's info indexed by member-index
1581         # n2vc_info = {}
1582         tasks_dict_info = {}  # from task to info text
1583         exc = None
1584         error_list = []
1585         stage = ['Stage 1/5: preparation of the environment.', "Waiting for previous operations to terminate.", ""]
1586         # ^ stage, step, VIM progress
1587         try:
1588             # wait for any previous tasks in process
1589             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
1590
1591             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
1592             stage[1] = "Reading from database,"
1593             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
1594             db_nsr_update["detailed-status"] = "creating"
1595             db_nsr_update["operational-status"] = "init"
1596             self._write_ns_status(
1597                 nsr_id=nsr_id,
1598                 ns_state="BUILDING",
1599                 current_operation="INSTANTIATING",
1600                 current_operation_id=nslcmop_id,
1601                 other_update=db_nsr_update
1602             )
1603             self._write_op_status(
1604                 op_id=nslcmop_id,
1605                 stage=stage,
1606                 queuePosition=0
1607             )
1608
1609             # read from db: operation
1610             stage[1] = "Getting nslcmop={} from db".format(nslcmop_id)
1611             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1612             ns_params = db_nslcmop.get("operationParams")
1613             if ns_params and ns_params.get("timeout_ns_deploy"):
1614                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1615             else:
1616                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
1617
1618             # read from db: ns
1619             stage[1] = "Getting nsr={} from db".format(nsr_id)
1620             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1621             stage[1] = "Getting nsd={} from db".format(db_nsr["nsd-id"])
1622             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1623             db_nsr["nsd"] = nsd
1624             # nsr_name = db_nsr["name"]   # TODO short-name??
1625
1626             # read from db: vnf's of this ns
1627             stage[1] = "Getting vnfrs from db"
1628             self.logger.debug(logging_text + stage[1])
1629             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1630
1631             # read from db: vnfd's for every vnf
1632             db_vnfds_ref = {}     # every vnfd data indexed by vnf name
1633             db_vnfds = {}         # every vnfd data indexed by vnf id
1634             db_vnfds_index = {}   # every vnfd data indexed by vnf member-index
1635
1636             # for each vnf in ns, read vnfd
1637             for vnfr in db_vnfrs_list:
1638                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr   # vnf's dict indexed by member-index: '1', '2', etc
1639                 vnfd_id = vnfr["vnfd-id"]                       # vnfd uuid for this vnf
1640                 vnfd_ref = vnfr["vnfd-ref"]                     # vnfd name for this vnf
1641                 # if we haven't this vnfd, read it from db
1642                 if vnfd_id not in db_vnfds:
1643                     # read from db
1644                     stage[1] = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
1645                     self.logger.debug(logging_text + stage[1])
1646                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1647
1648                     # store vnfd
1649                     db_vnfds_ref[vnfd_ref] = vnfd     # vnfd's indexed by name
1650                     db_vnfds[vnfd_id] = vnfd          # vnfd's indexed by id
1651                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]  # vnfd's indexed by member-index
1652
1653             # Get or generates the _admin.deployed.VCA list
1654             vca_deployed_list = None
1655             if db_nsr["_admin"].get("deployed"):
1656                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
1657             if vca_deployed_list is None:
1658                 vca_deployed_list = []
1659                 configuration_status_list = []
1660                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1661                 db_nsr_update["configurationStatus"] = configuration_status_list
1662                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
1663                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1664             elif isinstance(vca_deployed_list, dict):
1665                 # maintain backward compatibility. Change a dict to list at database
1666                 vca_deployed_list = list(vca_deployed_list.values())
1667                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1668                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1669
1670             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
1671                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
1672                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
1673
1674             # set state to INSTANTIATED. When instantiated NBI will not delete directly
1675             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1676             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1677
1678             # n2vc_redesign STEP 2 Deploy Network Scenario
1679             stage[0] = 'Stage 2/5: deployment of KDUs, VMs and execution environments.'
1680             self._write_op_status(
1681                 op_id=nslcmop_id,
1682                 stage=stage
1683             )
1684
1685             stage[1] = "Deploying KDUs,"
1686             # self.logger.debug(logging_text + "Before deploy_kdus")
1687             # Call to deploy_kdus in case exists the "vdu:kdu" param
1688             await self.deploy_kdus(
1689                 logging_text=logging_text,
1690                 nsr_id=nsr_id,
1691                 nslcmop_id=nslcmop_id,
1692                 db_vnfrs=db_vnfrs,
1693                 db_vnfds=db_vnfds,
1694                 task_instantiation_info=tasks_dict_info,
1695             )
1696
1697             stage[1] = "Getting VCA public key."
1698             # n2vc_redesign STEP 1 Get VCA public ssh-key
1699             # feature 1429. Add n2vc public key to needed VMs
1700             n2vc_key = self.n2vc.get_public_key()
1701             n2vc_key_list = [n2vc_key]
1702             if self.vca_config.get("public_key"):
1703                 n2vc_key_list.append(self.vca_config["public_key"])
1704
1705             stage[1] = "Deploying NS at VIM."
1706             task_ro = asyncio.ensure_future(
1707                 self.instantiate_RO(
1708                     logging_text=logging_text,
1709                     nsr_id=nsr_id,
1710                     nsd=nsd,
1711                     db_nsr=db_nsr,
1712                     db_nslcmop=db_nslcmop,
1713                     db_vnfrs=db_vnfrs,
1714                     db_vnfds_ref=db_vnfds_ref,
1715                     n2vc_key_list=n2vc_key_list,
1716                     stage=stage
1717                 )
1718             )
1719             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
1720             tasks_dict_info[task_ro] = "Deploying at VIM"
1721
1722             # n2vc_redesign STEP 3 to 6 Deploy N2VC
1723             stage[1] = "Deploying Execution Environments."
1724             self.logger.debug(logging_text + stage[1])
1725
1726             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
1727             # get_iterable() returns a value from a dict or empty tuple if key does not exist
1728             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1729                 vnfd_id = c_vnf["vnfd-id-ref"]
1730                 vnfd = db_vnfds_ref[vnfd_id]
1731                 member_vnf_index = str(c_vnf["member-vnf-index"])
1732                 db_vnfr = db_vnfrs[member_vnf_index]
1733                 base_folder = vnfd["_admin"]["storage"]
1734                 vdu_id = None
1735                 vdu_index = 0
1736                 vdu_name = None
1737                 kdu_name = None
1738
1739                 # Get additional parameters
1740                 deploy_params = {}
1741                 if db_vnfr.get("additionalParamsForVnf"):
1742                     deploy_params = self._format_additional_params(db_vnfr["additionalParamsForVnf"].copy())
1743
1744                 descriptor_config = vnfd.get("vnf-configuration")
1745                 if descriptor_config and descriptor_config.get("juju"):
1746                     self._deploy_n2vc(
1747                         logging_text=logging_text + "member_vnf_index={} ".format(member_vnf_index),
1748                         db_nsr=db_nsr,
1749                         db_vnfr=db_vnfr,
1750                         nslcmop_id=nslcmop_id,
1751                         nsr_id=nsr_id,
1752                         nsi_id=nsi_id,
1753                         vnfd_id=vnfd_id,
1754                         vdu_id=vdu_id,
1755                         kdu_name=kdu_name,
1756                         member_vnf_index=member_vnf_index,
1757                         vdu_index=vdu_index,
1758                         vdu_name=vdu_name,
1759                         deploy_params=deploy_params,
1760                         descriptor_config=descriptor_config,
1761                         base_folder=base_folder,
1762                         task_instantiation_info=tasks_dict_info,
1763                         stage=stage
1764                     )
1765
1766                 # Deploy charms for each VDU that supports one.
1767                 for vdud in get_iterable(vnfd, 'vdu'):
1768                     vdu_id = vdud["id"]
1769                     descriptor_config = vdud.get('vdu-configuration')
1770                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
1771                     if vdur.get("additionalParams"):
1772                         deploy_params_vdu = self._format_additional_params(vdur["additionalParams"])
1773                     else:
1774                         deploy_params_vdu = deploy_params
1775                     if descriptor_config and descriptor_config.get("juju"):
1776                         # look for vdu index in the db_vnfr["vdu"] section
1777                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
1778                         #     if vdur["vdu-id-ref"] == vdu_id:
1779                         #         break
1780                         # else:
1781                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
1782                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
1783                         # vdu_name = vdur.get("name")
1784                         vdu_name = None
1785                         kdu_name = None
1786                         for vdu_index in range(int(vdud.get("count", 1))):
1787                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1788                             self._deploy_n2vc(
1789                                 logging_text=logging_text + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
1790                                     member_vnf_index, vdu_id, vdu_index),
1791                                 db_nsr=db_nsr,
1792                                 db_vnfr=db_vnfr,
1793                                 nslcmop_id=nslcmop_id,
1794                                 nsr_id=nsr_id,
1795                                 nsi_id=nsi_id,
1796                                 vnfd_id=vnfd_id,
1797                                 vdu_id=vdu_id,
1798                                 kdu_name=kdu_name,
1799                                 member_vnf_index=member_vnf_index,
1800                                 vdu_index=vdu_index,
1801                                 vdu_name=vdu_name,
1802                                 deploy_params=deploy_params_vdu,
1803                                 descriptor_config=descriptor_config,
1804                                 base_folder=base_folder,
1805                                 task_instantiation_info=tasks_dict_info,
1806                                 stage=stage
1807                             )
1808                 for kdud in get_iterable(vnfd, 'kdu'):
1809                     kdu_name = kdud["name"]
1810                     descriptor_config = kdud.get('kdu-configuration')
1811                     if descriptor_config and descriptor_config.get("juju"):
1812                         vdu_id = None
1813                         vdu_index = 0
1814                         vdu_name = None
1815                         # look for vdu index in the db_vnfr["vdu"] section
1816                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
1817                         #     if vdur["vdu-id-ref"] == vdu_id:
1818                         #         break
1819                         # else:
1820                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
1821                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
1822                         # vdu_name = vdur.get("name")
1823                         # vdu_name = None
1824
1825                         self._deploy_n2vc(
1826                             logging_text=logging_text,
1827                             db_nsr=db_nsr,
1828                             db_vnfr=db_vnfr,
1829                             nslcmop_id=nslcmop_id,
1830                             nsr_id=nsr_id,
1831                             nsi_id=nsi_id,
1832                             vnfd_id=vnfd_id,
1833                             vdu_id=vdu_id,
1834                             kdu_name=kdu_name,
1835                             member_vnf_index=member_vnf_index,
1836                             vdu_index=vdu_index,
1837                             vdu_name=vdu_name,
1838                             deploy_params=deploy_params,
1839                             descriptor_config=descriptor_config,
1840                             base_folder=base_folder,
1841                             task_instantiation_info=tasks_dict_info,
1842                             stage=stage
1843                         )
1844
1845             # Check if this NS has a charm configuration
1846             descriptor_config = nsd.get("ns-configuration")
1847             if descriptor_config and descriptor_config.get("juju"):
1848                 vnfd_id = None
1849                 db_vnfr = None
1850                 member_vnf_index = None
1851                 vdu_id = None
1852                 kdu_name = None
1853                 vdu_index = 0
1854                 vdu_name = None
1855
1856                 # Get additional parameters
1857                 deploy_params = {}
1858                 if db_nsr.get("additionalParamsForNs"):
1859                     deploy_params = self._format_additional_params(db_nsr["additionalParamsForNs"].copy())
1860                 base_folder = nsd["_admin"]["storage"]
1861                 self._deploy_n2vc(
1862                     logging_text=logging_text,
1863                     db_nsr=db_nsr,
1864                     db_vnfr=db_vnfr,
1865                     nslcmop_id=nslcmop_id,
1866                     nsr_id=nsr_id,
1867                     nsi_id=nsi_id,
1868                     vnfd_id=vnfd_id,
1869                     vdu_id=vdu_id,
1870                     kdu_name=kdu_name,
1871                     member_vnf_index=member_vnf_index,
1872                     vdu_index=vdu_index,
1873                     vdu_name=vdu_name,
1874                     deploy_params=deploy_params,
1875                     descriptor_config=descriptor_config,
1876                     base_folder=base_folder,
1877                     task_instantiation_info=tasks_dict_info,
1878                     stage=stage
1879                 )
1880
1881             # rest of staff will be done at finally
1882
1883         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
1884             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(stage[1], e))
1885             exc = e
1886         except asyncio.CancelledError:
1887             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
1888             exc = "Operation was cancelled"
1889         except Exception as e:
1890             exc = traceback.format_exc()
1891             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
1892         finally:
1893             if exc:
1894                 error_list.append(str(exc))
1895             try:
1896                 # wait for pending tasks
1897                 if tasks_dict_info:
1898                     stage[1] = "Waiting for instantiate pending tasks."
1899                     self.logger.debug(logging_text + stage[1])
1900                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_deploy,
1901                                                              stage, nslcmop_id, nsr_id=nsr_id)
1902                 stage[1] = stage[2] = ""
1903             except asyncio.CancelledError:
1904                 error_list.append("Cancelled")
1905                 # TODO cancel all tasks
1906             except Exception as exc:
1907                 error_list.append(str(exc))
1908
1909             # update operation-status
1910             db_nsr_update["operational-status"] = "running"
1911             # let's begin with VCA 'configured' status (later we can change it)
1912             db_nsr_update["config-status"] = "configured"
1913             for task, task_name in tasks_dict_info.items():
1914                 if not task.done() or task.cancelled() or task.exception():
1915                     if task_name.startswith(self.task_name_deploy_vca):
1916                         # A N2VC task is pending
1917                         db_nsr_update["config-status"] = "failed"
1918                     else:
1919                         # RO or KDU task is pending
1920                         db_nsr_update["operational-status"] = "failed"
1921
1922             # update status at database
1923             if error_list:
1924                 error_detail = ". ".join(error_list)
1925                 self.logger.error(logging_text + error_detail)
1926                 error_description_nslcmop = 'Stage: {}. Detail: {}'.format(stage[0], error_detail)
1927                 error_description_nsr = 'Operation: INSTANTIATING.{}, Stage {}'.format(nslcmop_id, stage[0])
1928
1929                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
1930                 db_nslcmop_update["detailed-status"] = error_detail
1931                 nslcmop_operation_state = "FAILED"
1932                 ns_state = "BROKEN"
1933             else:
1934                 error_detail = None
1935                 error_description_nsr = error_description_nslcmop = None
1936                 ns_state = "READY"
1937                 db_nsr_update["detailed-status"] = "Done"
1938                 db_nslcmop_update["detailed-status"] = "Done"
1939                 nslcmop_operation_state = "COMPLETED"
1940
1941             if db_nsr:
1942                 self._write_ns_status(
1943                     nsr_id=nsr_id,
1944                     ns_state=ns_state,
1945                     current_operation="IDLE",
1946                     current_operation_id=None,
1947                     error_description=error_description_nsr,
1948                     error_detail=error_detail,
1949                     other_update=db_nsr_update
1950                 )
1951             self._write_op_status(
1952                 op_id=nslcmop_id,
1953                 stage="",
1954                 error_message=error_description_nslcmop,
1955                 operation_state=nslcmop_operation_state,
1956                 other_update=db_nslcmop_update,
1957             )
1958
1959             if nslcmop_operation_state:
1960                 try:
1961                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1962                                                                    "operationState": nslcmop_operation_state},
1963                                             loop=self.loop)
1964                 except Exception as e:
1965                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1966
1967             self.logger.debug(logging_text + "Exit")
1968             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1969
1970     async def _add_vca_relations(self, logging_text, nsr_id, vca_index: int, timeout: int = 3600) -> bool:
1971
1972         # steps:
1973         # 1. find all relations for this VCA
1974         # 2. wait for other peers related
1975         # 3. add relations
1976
1977         try:
1978
1979             # STEP 1: find all relations for this VCA
1980
1981             # read nsr record
1982             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1983             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1984
1985             # this VCA data
1986             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
1987
1988             # read all ns-configuration relations
1989             ns_relations = list()
1990             db_ns_relations = deep_get(nsd, ('ns-configuration', 'relation'))
1991             if db_ns_relations:
1992                 for r in db_ns_relations:
1993                     # check if this VCA is in the relation
1994                     if my_vca.get('member-vnf-index') in\
1995                             (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
1996                         ns_relations.append(r)
1997
1998             # read all vnf-configuration relations
1999             vnf_relations = list()
2000             db_vnfd_list = db_nsr.get('vnfd-id')
2001             if db_vnfd_list:
2002                 for vnfd in db_vnfd_list:
2003                     db_vnfd = self.db.get_one("vnfds", {"_id": vnfd})
2004                     db_vnf_relations = deep_get(db_vnfd, ('vnf-configuration', 'relation'))
2005                     if db_vnf_relations:
2006                         for r in db_vnf_relations:
2007                             # check if this VCA is in the relation
2008                             if my_vca.get('vdu_id') in (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
2009                                 vnf_relations.append(r)
2010
2011             # if no relations, terminate
2012             if not ns_relations and not vnf_relations:
2013                 self.logger.debug(logging_text + ' No relations')
2014                 return True
2015
2016             self.logger.debug(logging_text + ' adding relations\n    {}\n    {}'.format(ns_relations, vnf_relations))
2017
2018             # add all relations
2019             start = time()
2020             while True:
2021                 # check timeout
2022                 now = time()
2023                 if now - start >= timeout:
2024                     self.logger.error(logging_text + ' : timeout adding relations')
2025                     return False
2026
2027                 # reload nsr from database (we need to update record: _admin.deloyed.VCA)
2028                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2029
2030                 # for each defined NS relation, find the VCA's related
2031                 for r in ns_relations:
2032                     from_vca_ee_id = None
2033                     to_vca_ee_id = None
2034                     from_vca_endpoint = None
2035                     to_vca_endpoint = None
2036                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2037                     for vca in vca_list:
2038                         if vca.get('member-vnf-index') == r.get('entities')[0].get('id') \
2039                                 and vca.get('config_sw_installed'):
2040                             from_vca_ee_id = vca.get('ee_id')
2041                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2042                         if vca.get('member-vnf-index') == r.get('entities')[1].get('id') \
2043                                 and vca.get('config_sw_installed'):
2044                             to_vca_ee_id = vca.get('ee_id')
2045                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2046                     if from_vca_ee_id and to_vca_ee_id:
2047                         # add relation
2048                         await self.n2vc.add_relation(
2049                             ee_id_1=from_vca_ee_id,
2050                             ee_id_2=to_vca_ee_id,
2051                             endpoint_1=from_vca_endpoint,
2052                             endpoint_2=to_vca_endpoint)
2053                         # remove entry from relations list
2054                         ns_relations.remove(r)
2055                     else:
2056                         # check failed peers
2057                         try:
2058                             vca_status_list = db_nsr.get('configurationStatus')
2059                             if vca_status_list:
2060                                 for i in range(len(vca_list)):
2061                                     vca = vca_list[i]
2062                                     vca_status = vca_status_list[i]
2063                                     if vca.get('member-vnf-index') == r.get('entities')[0].get('id'):
2064                                         if vca_status.get('status') == 'BROKEN':
2065                                             # peer broken: remove relation from list
2066                                             ns_relations.remove(r)
2067                                     if vca.get('member-vnf-index') == r.get('entities')[1].get('id'):
2068                                         if vca_status.get('status') == 'BROKEN':
2069                                             # peer broken: remove relation from list
2070                                             ns_relations.remove(r)
2071                         except Exception:
2072                             # ignore
2073                             pass
2074
2075                 # for each defined VNF relation, find the VCA's related
2076                 for r in vnf_relations:
2077                     from_vca_ee_id = None
2078                     to_vca_ee_id = None
2079                     from_vca_endpoint = None
2080                     to_vca_endpoint = None
2081                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2082                     for vca in vca_list:
2083                         if vca.get('vdu_id') == r.get('entities')[0].get('id') and vca.get('config_sw_installed'):
2084                             from_vca_ee_id = vca.get('ee_id')
2085                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2086                         if vca.get('vdu_id') == r.get('entities')[1].get('id') and vca.get('config_sw_installed'):
2087                             to_vca_ee_id = vca.get('ee_id')
2088                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2089                     if from_vca_ee_id and to_vca_ee_id:
2090                         # add relation
2091                         await self.n2vc.add_relation(
2092                             ee_id_1=from_vca_ee_id,
2093                             ee_id_2=to_vca_ee_id,
2094                             endpoint_1=from_vca_endpoint,
2095                             endpoint_2=to_vca_endpoint)
2096                         # remove entry from relations list
2097                         vnf_relations.remove(r)
2098                     else:
2099                         # check failed peers
2100                         try:
2101                             vca_status_list = db_nsr.get('configurationStatus')
2102                             if vca_status_list:
2103                                 for i in range(len(vca_list)):
2104                                     vca = vca_list[i]
2105                                     vca_status = vca_status_list[i]
2106                                     if vca.get('vdu_id') == r.get('entities')[0].get('id'):
2107                                         if vca_status.get('status') == 'BROKEN':
2108                                             # peer broken: remove relation from list
2109                                             ns_relations.remove(r)
2110                                     if vca.get('vdu_id') == r.get('entities')[1].get('id'):
2111                                         if vca_status.get('status') == 'BROKEN':
2112                                             # peer broken: remove relation from list
2113                                             ns_relations.remove(r)
2114                         except Exception:
2115                             # ignore
2116                             pass
2117
2118                 # wait for next try
2119                 await asyncio.sleep(5.0)
2120
2121                 if not ns_relations and not vnf_relations:
2122                     self.logger.debug('Relations added')
2123                     break
2124
2125             return True
2126
2127         except Exception as e:
2128             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
2129             return False
2130
2131     def _write_db_callback(self, task, item, _id, on_done=None, on_exc=None):
2132         """
2133         callback for kdu install intended to store the returned kdu_instance at database
2134         :return: None
2135         """
2136         db_update = {}
2137         try:
2138             result = task.result()
2139             if on_done:
2140                 db_update[on_done] = str(result)
2141         except Exception as e:
2142             if on_exc:
2143                 db_update[on_exc] = str(e)
2144         if db_update:
2145             try:
2146                 self.update_db_2(item, _id, db_update)
2147             except Exception:
2148                 pass
2149
2150     async def deploy_kdus(self, logging_text, nsr_id, nslcmop_id, db_vnfrs, db_vnfds, task_instantiation_info):
2151         # Launch kdus if present in the descriptor
2152
2153         k8scluster_id_2_uuic = {"helm-chart": {}, "juju-bundle": {}}
2154
2155         def _get_cluster_id(cluster_id, cluster_type):
2156             nonlocal k8scluster_id_2_uuic
2157             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
2158                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
2159
2160             db_k8scluster = self.db.get_one("k8sclusters", {"_id": cluster_id}, fail_on_empty=False)
2161             if not db_k8scluster:
2162                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
2163             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
2164             if not k8s_id:
2165                 raise LcmException("K8s cluster '{}' has not been initilized for '{}'".format(cluster_id, cluster_type))
2166             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
2167             return k8s_id
2168
2169         logging_text += "Deploy kdus: "
2170         step = ""
2171         try:
2172             db_nsr_update = {"_admin.deployed.K8s": []}
2173             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2174
2175             index = 0
2176             updated_cluster_list = []
2177
2178             for vnfr_data in db_vnfrs.values():
2179                 for kdur in get_iterable(vnfr_data, "kdur"):
2180                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
2181                     vnfd_id = vnfr_data.get('vnfd-id')
2182                     namespace = kdur.get("k8s-namespace")
2183                     if kdur.get("helm-chart"):
2184                         kdumodel = kdur["helm-chart"]
2185                         k8sclustertype = "helm-chart"
2186                     elif kdur.get("juju-bundle"):
2187                         kdumodel = kdur["juju-bundle"]
2188                         k8sclustertype = "juju-bundle"
2189                     else:
2190                         raise LcmException("kdu type for kdu='{}.{}' is neither helm-chart nor "
2191                                            "juju-bundle. Maybe an old NBI version is running".
2192                                            format(vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]))
2193                     # check if kdumodel is a file and exists
2194                     try:
2195                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
2196                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
2197                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
2198                             filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["pkg-dir"], k8sclustertype,
2199                                                              kdumodel)
2200                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
2201                                 kdumodel = self.fs.path + filename
2202                     except (asyncio.TimeoutError, asyncio.CancelledError):
2203                         raise
2204                     except Exception:       # it is not a file
2205                         pass
2206
2207                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
2208                     step = "Synchronize repos for k8s cluster '{}'".format(k8s_cluster_id)
2209                     cluster_uuid = _get_cluster_id(k8s_cluster_id, k8sclustertype)
2210
2211                     if k8sclustertype == "helm-chart" and cluster_uuid not in updated_cluster_list:
2212                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
2213                             self.k8sclusterhelm.synchronize_repos(cluster_uuid=cluster_uuid))
2214                         if del_repo_list or added_repo_dict:
2215                             unset = {'_admin.helm_charts_added.' + item: None for item in del_repo_list}
2216                             updated = {'_admin.helm_charts_added.' +
2217                                        item: name for item, name in added_repo_dict.items()}
2218                             self.logger.debug(logging_text + "repos synchronized on k8s cluster '{}' to_delete: {}, "
2219                                                              "to_add: {}".format(k8s_cluster_id, del_repo_list,
2220                                                                                  added_repo_dict))
2221                             self.db.set_one("k8sclusters", {"_id": k8s_cluster_id}, updated, unset=unset)
2222                         updated_cluster_list.append(cluster_uuid)
2223
2224                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(vnfr_data["member-vnf-index-ref"],
2225                                                                               kdur["kdu-name"], k8s_cluster_id)
2226
2227                     k8s_instace_info = {"kdu-instance": None,
2228                                         "k8scluster-uuid": cluster_uuid,
2229                                         "k8scluster-type": k8sclustertype,
2230                                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
2231                                         "kdu-name": kdur["kdu-name"],
2232                                         "kdu-model": kdumodel,
2233                                         "namespace": namespace}
2234                     db_path = "_admin.deployed.K8s.{}".format(index)
2235                     db_nsr_update[db_path] = k8s_instace_info
2236                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2237
2238                     db_dict = {"collection": "nsrs",
2239                                "filter": {"_id": nsr_id},
2240                                "path": db_path}
2241
2242                     task = asyncio.ensure_future(
2243                         self.k8scluster_map[k8sclustertype].install(cluster_uuid=cluster_uuid, kdu_model=kdumodel,
2244                                                                     atomic=True, params=desc_params,
2245                                                                     db_dict=db_dict, timeout=600,
2246                                                                     kdu_name=kdur["kdu-name"], namespace=namespace))
2247
2248                     task.add_done_callback(partial(self._write_db_callback, item="nsrs", _id=nsr_id,
2249                                                    on_done=db_path + ".kdu-instance",
2250                                                    on_exc=db_path + ".detailed-status"))
2251                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
2252                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
2253
2254                     index += 1
2255
2256         except (LcmException, asyncio.CancelledError):
2257             raise
2258         except Exception as e:
2259             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
2260             if isinstance(e, (N2VCException, DbException)):
2261                 self.logger.error(logging_text + msg)
2262             else:
2263                 self.logger.critical(logging_text + msg, exc_info=True)
2264             raise LcmException(msg)
2265         finally:
2266             if db_nsr_update:
2267                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2268
2269     def _deploy_n2vc(self, logging_text, db_nsr, db_vnfr, nslcmop_id, nsr_id, nsi_id, vnfd_id, vdu_id,
2270                      kdu_name, member_vnf_index, vdu_index, vdu_name, deploy_params, descriptor_config,
2271                      base_folder, task_instantiation_info, stage):
2272         # launch instantiate_N2VC in a asyncio task and register task object
2273         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
2274         # if not found, create one entry and update database
2275
2276         # fill db_nsr._admin.deployed.VCA.<index>
2277         vca_index = -1
2278         for vca_index, vca_deployed in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
2279             if not vca_deployed:
2280                 continue
2281             if vca_deployed.get("member-vnf-index") == member_vnf_index and \
2282                     vca_deployed.get("vdu_id") == vdu_id and \
2283                     vca_deployed.get("kdu_name") == kdu_name and \
2284                     vca_deployed.get("vdu_count_index", 0) == vdu_index:
2285                 break
2286         else:
2287             # not found, create one.
2288             vca_deployed = {
2289                 "member-vnf-index": member_vnf_index,
2290                 "vdu_id": vdu_id,
2291                 "kdu_name": kdu_name,
2292                 "vdu_count_index": vdu_index,
2293                 "operational-status": "init",  # TODO revise
2294                 "detailed-status": "",  # TODO revise
2295                 "step": "initial-deploy",   # TODO revise
2296                 "vnfd_id": vnfd_id,
2297                 "vdu_name": vdu_name,
2298             }
2299             vca_index += 1
2300
2301             # create VCA and configurationStatus in db
2302             db_dict = {
2303                 "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
2304                 "configurationStatus.{}".format(vca_index): dict()
2305             }
2306             self.update_db_2("nsrs", nsr_id, db_dict)
2307
2308             db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
2309
2310         # Launch task
2311         task_n2vc = asyncio.ensure_future(
2312             self.instantiate_N2VC(
2313                 logging_text=logging_text,
2314                 vca_index=vca_index,
2315                 nsi_id=nsi_id,
2316                 db_nsr=db_nsr,
2317                 db_vnfr=db_vnfr,
2318                 vdu_id=vdu_id,
2319                 kdu_name=kdu_name,
2320                 vdu_index=vdu_index,
2321                 deploy_params=deploy_params,
2322                 config_descriptor=descriptor_config,
2323                 base_folder=base_folder,
2324                 nslcmop_id=nslcmop_id,
2325                 stage=stage
2326             )
2327         )
2328         self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_N2VC-{}".format(vca_index), task_n2vc)
2329         task_instantiation_info[task_n2vc] = self.task_name_deploy_vca + " {}.{}".format(
2330             member_vnf_index or "", vdu_id or "")
2331
2332     # Check if this VNFD has a configured terminate action
2333     def _has_terminate_config_primitive(self, vnfd):
2334         vnf_config = vnfd.get("vnf-configuration")
2335         if vnf_config and vnf_config.get("terminate-config-primitive"):
2336             return True
2337         else:
2338             return False
2339
2340     @staticmethod
2341     def _get_terminate_config_primitive_seq_list(vnfd):
2342         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
2343         # No need to check for existing primitive twice, already done before
2344         vnf_config = vnfd.get("vnf-configuration")
2345         seq_list = vnf_config.get("terminate-config-primitive")
2346         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
2347         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
2348         return seq_list_sorted
2349
2350     @staticmethod
2351     def _create_nslcmop(nsr_id, operation, params):
2352         """
2353         Creates a ns-lcm-opp content to be stored at database.
2354         :param nsr_id: internal id of the instance
2355         :param operation: instantiate, terminate, scale, action, ...
2356         :param params: user parameters for the operation
2357         :return: dictionary following SOL005 format
2358         """
2359         # Raise exception if invalid arguments
2360         if not (nsr_id and operation and params):
2361             raise LcmException(
2362                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
2363         now = time()
2364         _id = str(uuid4())
2365         nslcmop = {
2366             "id": _id,
2367             "_id": _id,
2368             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
2369             "operationState": "PROCESSING",
2370             "statusEnteredTime": now,
2371             "nsInstanceId": nsr_id,
2372             "lcmOperationType": operation,
2373             "startTime": now,
2374             "isAutomaticInvocation": False,
2375             "operationParams": params,
2376             "isCancelPending": False,
2377             "links": {
2378                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
2379                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
2380             }
2381         }
2382         return nslcmop
2383
2384     def _format_additional_params(self, params):
2385         params = params or {}
2386         for key, value in params.items():
2387             if str(value).startswith("!!yaml "):
2388                 params[key] = yaml.safe_load(value[7:])
2389         return params
2390
2391     def _get_terminate_primitive_params(self, seq, vnf_index):
2392         primitive = seq.get('name')
2393         primitive_params = {}
2394         params = {
2395             "member_vnf_index": vnf_index,
2396             "primitive": primitive,
2397             "primitive_params": primitive_params,
2398         }
2399         desc_params = {}
2400         return self._map_primitive_params(seq, params, desc_params)
2401
2402     # sub-operations
2403
2404     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
2405         op = deep_get(db_nslcmop, ('_admin', 'operations'), [])[op_index]
2406         if op.get('operationState') == 'COMPLETED':
2407             # b. Skip sub-operation
2408             # _ns_execute_primitive() or RO.create_action() will NOT be executed
2409             return self.SUBOPERATION_STATUS_SKIP
2410         else:
2411             # c. retry executing sub-operation
2412             # The sub-operation exists, and operationState != 'COMPLETED'
2413             # Update operationState = 'PROCESSING' to indicate a retry.
2414             operationState = 'PROCESSING'
2415             detailed_status = 'In progress'
2416             self._update_suboperation_status(
2417                 db_nslcmop, op_index, operationState, detailed_status)
2418             # Return the sub-operation index
2419             # _ns_execute_primitive() or RO.create_action() will be called from scale()
2420             # with arguments extracted from the sub-operation
2421             return op_index
2422
2423     # Find a sub-operation where all keys in a matching dictionary must match
2424     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
2425     def _find_suboperation(self, db_nslcmop, match):
2426         if db_nslcmop and match:
2427             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
2428             for i, op in enumerate(op_list):
2429                 if all(op.get(k) == match[k] for k in match):
2430                     return i
2431         return self.SUBOPERATION_STATUS_NOT_FOUND
2432
2433     # Update status for a sub-operation given its index
2434     def _update_suboperation_status(self, db_nslcmop, op_index, operationState, detailed_status):
2435         # Update DB for HA tasks
2436         q_filter = {'_id': db_nslcmop['_id']}
2437         update_dict = {'_admin.operations.{}.operationState'.format(op_index): operationState,
2438                        '_admin.operations.{}.detailed-status'.format(op_index): detailed_status}
2439         self.db.set_one("nslcmops",
2440                         q_filter=q_filter,
2441                         update_dict=update_dict,
2442                         fail_on_empty=False)
2443
2444     # Add sub-operation, return the index of the added sub-operation
2445     # Optionally, set operationState, detailed-status, and operationType
2446     # Status and type are currently set for 'scale' sub-operations:
2447     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
2448     # 'detailed-status' : status message
2449     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
2450     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
2451     def _add_suboperation(self, db_nslcmop, vnf_index, vdu_id, vdu_count_index, vdu_name, primitive,
2452                           mapped_primitive_params, operationState=None, detailed_status=None, operationType=None,
2453                           RO_nsr_id=None, RO_scaling_info=None):
2454         if not db_nslcmop:
2455             return self.SUBOPERATION_STATUS_NOT_FOUND
2456         # Get the "_admin.operations" list, if it exists
2457         db_nslcmop_admin = db_nslcmop.get('_admin', {})
2458         op_list = db_nslcmop_admin.get('operations')
2459         # Create or append to the "_admin.operations" list
2460         new_op = {'member_vnf_index': vnf_index,
2461                   'vdu_id': vdu_id,
2462                   'vdu_count_index': vdu_count_index,
2463                   'primitive': primitive,
2464                   'primitive_params': mapped_primitive_params}
2465         if operationState:
2466             new_op['operationState'] = operationState
2467         if detailed_status:
2468             new_op['detailed-status'] = detailed_status
2469         if operationType:
2470             new_op['lcmOperationType'] = operationType
2471         if RO_nsr_id:
2472             new_op['RO_nsr_id'] = RO_nsr_id
2473         if RO_scaling_info:
2474             new_op['RO_scaling_info'] = RO_scaling_info
2475         if not op_list:
2476             # No existing operations, create key 'operations' with current operation as first list element
2477             db_nslcmop_admin.update({'operations': [new_op]})
2478             op_list = db_nslcmop_admin.get('operations')
2479         else:
2480             # Existing operations, append operation to list
2481             op_list.append(new_op)
2482
2483         db_nslcmop_update = {'_admin.operations': op_list}
2484         self.update_db_2("nslcmops", db_nslcmop['_id'], db_nslcmop_update)
2485         op_index = len(op_list) - 1
2486         return op_index
2487
2488     # Helper methods for scale() sub-operations
2489
2490     # pre-scale/post-scale:
2491     # Check for 3 different cases:
2492     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
2493     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
2494     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
2495     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
2496                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
2497         # Find this sub-operation
2498         if RO_nsr_id and RO_scaling_info:
2499             operationType = 'SCALE-RO'
2500             match = {
2501                 'member_vnf_index': vnf_index,
2502                 'RO_nsr_id': RO_nsr_id,
2503                 'RO_scaling_info': RO_scaling_info,
2504             }
2505         else:
2506             match = {
2507                 'member_vnf_index': vnf_index,
2508                 'primitive': vnf_config_primitive,
2509                 'primitive_params': primitive_params,
2510                 'lcmOperationType': operationType
2511             }
2512         op_index = self._find_suboperation(db_nslcmop, match)
2513         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
2514             # a. New sub-operation
2515             # The sub-operation does not exist, add it.
2516             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
2517             # The following parameters are set to None for all kind of scaling:
2518             vdu_id = None
2519             vdu_count_index = None
2520             vdu_name = None
2521             if RO_nsr_id and RO_scaling_info:
2522                 vnf_config_primitive = None
2523                 primitive_params = None
2524             else:
2525                 RO_nsr_id = None
2526                 RO_scaling_info = None
2527             # Initial status for sub-operation
2528             operationState = 'PROCESSING'
2529             detailed_status = 'In progress'
2530             # Add sub-operation for pre/post-scaling (zero or more operations)
2531             self._add_suboperation(db_nslcmop,
2532                                    vnf_index,
2533                                    vdu_id,
2534                                    vdu_count_index,
2535                                    vdu_name,
2536                                    vnf_config_primitive,
2537                                    primitive_params,
2538                                    operationState,
2539                                    detailed_status,
2540                                    operationType,
2541                                    RO_nsr_id,
2542                                    RO_scaling_info)
2543             return self.SUBOPERATION_STATUS_NEW
2544         else:
2545             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
2546             # or op_index (operationState != 'COMPLETED')
2547             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
2548
2549     # Function to return execution_environment id
2550
2551     def _get_ee_id(self, vnf_index, vdu_id, vca_deployed_list):
2552         # TODO vdu_index_count
2553         for vca in vca_deployed_list:
2554             if vca["member-vnf-index"] == vnf_index and vca["vdu_id"] == vdu_id:
2555                 return vca["ee_id"]
2556
2557     async def destroy_N2VC(self, logging_text, db_nslcmop, vca_deployed, config_descriptor, vca_index, destroy_ee=True):
2558         """
2559         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
2560         :param logging_text:
2561         :param db_nslcmop:
2562         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
2563         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
2564         :param vca_index: index in the database _admin.deployed.VCA
2565         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
2566         :return: None or exception
2567         """
2568         # execute terminate_primitives
2569         terminate_primitives = config_descriptor.get("terminate-config-primitive")
2570         vdu_id = vca_deployed.get("vdu_id")
2571         vdu_count_index = vca_deployed.get("vdu_count_index")
2572         vdu_name = vca_deployed.get("vdu_name")
2573         vnf_index = vca_deployed.get("member-vnf-index")
2574         if terminate_primitives and vca_deployed.get("needed_terminate"):
2575             # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
2576             terminate_primitives = sorted(terminate_primitives, key=lambda x: int(x['seq']))
2577             for seq in terminate_primitives:
2578                 # For each sequence in list, get primitive and call _ns_execute_primitive()
2579                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
2580                     vnf_index, seq.get("name"))
2581                 self.logger.debug(logging_text + step)
2582                 # Create the primitive for each sequence, i.e. "primitive": "touch"
2583                 primitive = seq.get('name')
2584                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
2585                 # The following 3 parameters are currently set to None for 'terminate':
2586                 # vdu_id, vdu_count_index, vdu_name
2587
2588                 # Add sub-operation
2589                 self._add_suboperation(db_nslcmop,
2590                                        vnf_index,
2591                                        vdu_id,
2592                                        vdu_count_index,
2593                                        vdu_name,
2594                                        primitive,
2595                                        mapped_primitive_params)
2596                 # Sub-operations: Call _ns_execute_primitive() instead of action()
2597                 try:
2598                     result, result_detail = await self._ns_execute_primitive(vca_deployed["ee_id"], primitive,
2599                                                                              mapped_primitive_params)
2600                 except LcmException:
2601                     # this happens when VCA is not deployed. In this case it is not needed to terminate
2602                     continue
2603                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
2604                 if result not in result_ok:
2605                     raise LcmException("terminate_primitive {}  for vnf_member_index={} fails with "
2606                                        "error {}".format(seq.get("name"), vnf_index, result_detail))
2607             # set that this VCA do not need terminated
2608             db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(vca_index)
2609             self.update_db_2("nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False})
2610
2611         if destroy_ee:
2612             await self.n2vc.delete_execution_environment(vca_deployed["ee_id"])
2613
2614     async def _delete_all_N2VC(self, db_nsr: dict):
2615         self._write_all_config_status(db_nsr=db_nsr, status='TERMINATING')
2616         namespace = "." + db_nsr["_id"]
2617         try:
2618             await self.n2vc.delete_namespace(namespace=namespace, total_timeout=self.timeout_charm_delete)
2619         except N2VCNotFound:  # already deleted. Skip
2620             pass
2621         self._write_all_config_status(db_nsr=db_nsr, status='DELETED')
2622
2623     async def _terminate_RO(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
2624         """
2625         Terminates a deployment from RO
2626         :param logging_text:
2627         :param nsr_deployed: db_nsr._admin.deployed
2628         :param nsr_id:
2629         :param nslcmop_id:
2630         :param stage: list of string with the content to write on db_nslcmop.detailed-status.
2631             this method will update only the index 2, but it will write on database the concatenated content of the list
2632         :return:
2633         """
2634         db_nsr_update = {}
2635         failed_detail = []
2636         ro_nsr_id = ro_delete_action = None
2637         if nsr_deployed and nsr_deployed.get("RO"):
2638             ro_nsr_id = nsr_deployed["RO"].get("nsr_id")
2639             ro_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
2640         try:
2641             if ro_nsr_id:
2642                 stage[2] = "Deleting ns from VIM."
2643                 db_nsr_update["detailed-status"] = " ".join(stage)
2644                 self._write_op_status(nslcmop_id, stage)
2645                 self.logger.debug(logging_text + stage[2])
2646                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2647                 self._write_op_status(nslcmop_id, stage)
2648                 desc = await self.RO.delete("ns", ro_nsr_id)
2649                 ro_delete_action = desc["action_id"]
2650                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = ro_delete_action
2651                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2652                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2653             if ro_delete_action:
2654                 # wait until NS is deleted from VIM
2655                 stage[2] = "Waiting ns deleted from VIM."
2656                 detailed_status_old = None
2657                 self.logger.debug(logging_text + stage[2] + " RO_id={} ro_delete_action={}".format(ro_nsr_id,
2658                                                                                                    ro_delete_action))
2659                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2660                 self._write_op_status(nslcmop_id, stage)
2661
2662                 delete_timeout = 20 * 60  # 20 minutes
2663                 while delete_timeout > 0:
2664                     desc = await self.RO.show(
2665                         "ns",
2666                         item_id_name=ro_nsr_id,
2667                         extra_item="action",
2668                         extra_item_id=ro_delete_action)
2669
2670                     # deploymentStatus
2671                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
2672
2673                     ns_status, ns_status_info = self.RO.check_action_status(desc)
2674                     if ns_status == "ERROR":
2675                         raise ROclient.ROClientException(ns_status_info)
2676                     elif ns_status == "BUILD":
2677                         stage[2] = "Deleting from VIM {}".format(ns_status_info)
2678                     elif ns_status == "ACTIVE":
2679                         db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2680                         db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2681                         break
2682                     else:
2683                         assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2684                     if stage[2] != detailed_status_old:
2685                         detailed_status_old = stage[2]
2686                         db_nsr_update["detailed-status"] = " ".join(stage)
2687                         self._write_op_status(nslcmop_id, stage)
2688                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
2689                     await asyncio.sleep(5, loop=self.loop)
2690                     delete_timeout -= 5
2691                 else:  # delete_timeout <= 0:
2692                     raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
2693
2694         except Exception as e:
2695             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2696             if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2697                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2698                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2699                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2700                 self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(ro_nsr_id))
2701             elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2702                 failed_detail.append("delete conflict: {}".format(e))
2703                 self.logger.debug(logging_text + "RO_ns_id={} delete conflict: {}".format(ro_nsr_id, e))
2704             else:
2705                 failed_detail.append("delete error: {}".format(e))
2706                 self.logger.error(logging_text + "RO_ns_id={} delete error: {}".format(ro_nsr_id, e))
2707
2708         # Delete nsd
2709         if not failed_detail and deep_get(nsr_deployed, ("RO", "nsd_id")):
2710             ro_nsd_id = nsr_deployed["RO"]["nsd_id"]
2711             try:
2712                 stage[2] = "Deleting nsd from RO."
2713                 db_nsr_update["detailed-status"] = " ".join(stage)
2714                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2715                 self._write_op_status(nslcmop_id, stage)
2716                 await self.RO.delete("nsd", ro_nsd_id)
2717                 self.logger.debug(logging_text + "ro_nsd_id={} deleted".format(ro_nsd_id))
2718                 db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2719             except Exception as e:
2720                 if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2721                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2722                     self.logger.debug(logging_text + "ro_nsd_id={} already deleted".format(ro_nsd_id))
2723                 elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2724                     failed_detail.append("ro_nsd_id={} delete conflict: {}".format(ro_nsd_id, e))
2725                     self.logger.debug(logging_text + failed_detail[-1])
2726                 else:
2727                     failed_detail.append("ro_nsd_id={} delete error: {}".format(ro_nsd_id, e))
2728                     self.logger.error(logging_text + failed_detail[-1])
2729
2730         if not failed_detail and deep_get(nsr_deployed, ("RO", "vnfd")):
2731             for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
2732                 if not vnf_deployed or not vnf_deployed["id"]:
2733                     continue
2734                 try:
2735                     ro_vnfd_id = vnf_deployed["id"]
2736                     stage[2] = "Deleting member_vnf_index={} ro_vnfd_id={} from RO.".format(
2737                         vnf_deployed["member-vnf-index"], ro_vnfd_id)
2738                     db_nsr_update["detailed-status"] = " ".join(stage)
2739                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2740                     self._write_op_status(nslcmop_id, stage)
2741                     await self.RO.delete("vnfd", ro_vnfd_id)
2742                     self.logger.debug(logging_text + "ro_vnfd_id={} deleted".format(ro_vnfd_id))
2743                     db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2744                 except Exception as e:
2745                     if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2746                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2747                         self.logger.debug(logging_text + "ro_vnfd_id={} already deleted ".format(ro_vnfd_id))
2748                     elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2749                         failed_detail.append("ro_vnfd_id={} delete conflict: {}".format(ro_vnfd_id, e))
2750                         self.logger.debug(logging_text + failed_detail[-1])
2751                     else:
2752                         failed_detail.append("ro_vnfd_id={} delete error: {}".format(ro_vnfd_id, e))
2753                         self.logger.error(logging_text + failed_detail[-1])
2754
2755         if failed_detail:
2756             stage[2] = "Error deleting from VIM"
2757         else:
2758             stage[2] = "Deleted from VIM"
2759         db_nsr_update["detailed-status"] = " ".join(stage)
2760         self.update_db_2("nsrs", nsr_id, db_nsr_update)
2761         self._write_op_status(nslcmop_id, stage)
2762
2763         if failed_detail:
2764             raise LcmException("; ".join(failed_detail))
2765
2766     async def terminate(self, nsr_id, nslcmop_id):
2767         # Try to lock HA task here
2768         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2769         if not task_is_locked_by_me:
2770             return
2771
2772         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
2773         self.logger.debug(logging_text + "Enter")
2774         timeout_ns_terminate = self.timeout_ns_terminate
2775         db_nsr = None
2776         db_nslcmop = None
2777         operation_params = None
2778         exc = None
2779         error_list = []   # annotates all failed error messages
2780         db_nslcmop_update = {}
2781         autoremove = False  # autoremove after terminated
2782         tasks_dict_info = {}
2783         db_nsr_update = {}
2784         stage = ["Stage 1/3: Preparing task.", "Waiting for previous operations to terminate.", ""]
2785         # ^ contains [stage, step, VIM-status]
2786         try:
2787             # wait for any previous tasks in process
2788             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
2789
2790             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
2791             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2792             operation_params = db_nslcmop.get("operationParams") or {}
2793             if operation_params.get("timeout_ns_terminate"):
2794                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
2795             stage[1] = "Getting nsr={} from db.".format(nsr_id)
2796             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2797
2798             db_nsr_update["operational-status"] = "terminating"
2799             db_nsr_update["config-status"] = "terminating"
2800             self._write_ns_status(
2801                 nsr_id=nsr_id,
2802                 ns_state="TERMINATING",
2803                 current_operation="TERMINATING",
2804                 current_operation_id=nslcmop_id,
2805                 other_update=db_nsr_update
2806             )
2807             self._write_op_status(
2808                 op_id=nslcmop_id,
2809                 queuePosition=0,
2810                 stage=stage
2811             )
2812             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
2813             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
2814                 return
2815
2816             stage[1] = "Getting vnf descriptors from db."
2817             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
2818             db_vnfds_from_id = {}
2819             db_vnfds_from_member_index = {}
2820             # Loop over VNFRs
2821             for vnfr in db_vnfrs_list:
2822                 vnfd_id = vnfr["vnfd-id"]
2823                 if vnfd_id not in db_vnfds_from_id:
2824                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
2825                     db_vnfds_from_id[vnfd_id] = vnfd
2826                 db_vnfds_from_member_index[vnfr["member-vnf-index-ref"]] = db_vnfds_from_id[vnfd_id]
2827
2828             # Destroy individual execution environments when there are terminating primitives.
2829             # Rest of EE will be deleted at once
2830             if not operation_params.get("skip_terminate_primitives"):
2831                 stage[0] = "Stage 2/3 execute terminating primitives."
2832                 stage[1] = "Looking execution environment that needs terminate."
2833                 self.logger.debug(logging_text + stage[1])
2834                 for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
2835                     config_descriptor = None
2836                     if not vca or not vca.get("ee_id") or not vca.get("needed_terminate"):
2837                         continue
2838                     if not vca.get("member-vnf-index"):
2839                         # ns
2840                         config_descriptor = db_nsr.get("ns-configuration")
2841                     elif vca.get("vdu_id"):
2842                         db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
2843                         vdud = next((vdu for vdu in db_vnfd.get("vdu", ()) if vdu["id"] == vca.get("vdu_id")), None)
2844                         if vdud:
2845                             config_descriptor = vdud.get("vdu-configuration")
2846                     elif vca.get("kdu_name"):
2847                         db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
2848                         kdud = next((kdu for kdu in db_vnfd.get("kdu", ()) if kdu["name"] == vca.get("kdu_name")), None)
2849                         if kdud:
2850                             config_descriptor = kdud.get("kdu-configuration")
2851                     else:
2852                         config_descriptor = db_vnfds_from_member_index[vca["member-vnf-index"]].get("vnf-configuration")
2853                     task = asyncio.ensure_future(self.destroy_N2VC(logging_text, db_nslcmop, vca, config_descriptor,
2854                                                                    vca_index, False))
2855                     tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
2856
2857                 # wait for pending tasks of terminate primitives
2858                 if tasks_dict_info:
2859                     self.logger.debug(logging_text + 'Waiting for terminate primitive pending tasks...')
2860                     error_list = await self._wait_for_tasks(logging_text, tasks_dict_info,
2861                                                             min(self.timeout_charm_delete, timeout_ns_terminate),
2862                                                             stage, nslcmop_id)
2863                     if error_list:
2864                         return   # raise LcmException("; ".join(error_list))
2865                     tasks_dict_info.clear()
2866
2867             # remove All execution environments at once
2868             stage[0] = "Stage 3/3 delete all."
2869
2870             if nsr_deployed.get("VCA"):
2871                 stage[1] = "Deleting all execution environments."
2872                 self.logger.debug(logging_text + stage[1])
2873                 task_delete_ee = asyncio.ensure_future(asyncio.wait_for(self._delete_all_N2VC(db_nsr=db_nsr),
2874                                                                         timeout=self.timeout_charm_delete))
2875                 # task_delete_ee = asyncio.ensure_future(self.n2vc.delete_namespace(namespace="." + nsr_id))
2876                 tasks_dict_info[task_delete_ee] = "Terminating all VCA"
2877
2878             # Delete from k8scluster
2879             stage[1] = "Deleting KDUs."
2880             self.logger.debug(logging_text + stage[1])
2881             # print(nsr_deployed)
2882             for kdu in get_iterable(nsr_deployed, "K8s"):
2883                 if not kdu or not kdu.get("kdu-instance"):
2884                     continue
2885                 kdu_instance = kdu.get("kdu-instance")
2886                 if kdu.get("k8scluster-type") in self.k8scluster_map:
2887                     task_delete_kdu_instance = asyncio.ensure_future(
2888                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
2889                             cluster_uuid=kdu.get("k8scluster-uuid"),
2890                             kdu_instance=kdu_instance))
2891                 else:
2892                     self.logger.error(logging_text + "Unknown k8s deployment type {}".
2893                                       format(kdu.get("k8scluster-type")))
2894                     continue
2895                 tasks_dict_info[task_delete_kdu_instance] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
2896
2897             # remove from RO
2898             stage[1] = "Deleting ns from VIM."
2899             task_delete_ro = asyncio.ensure_future(
2900                 self._terminate_RO(logging_text, nsr_deployed, nsr_id, nslcmop_id, stage))
2901             tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
2902
2903             # rest of staff will be done at finally
2904
2905         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
2906             self.logger.error(logging_text + "Exit Exception {}".format(e))
2907             exc = e
2908         except asyncio.CancelledError:
2909             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
2910             exc = "Operation was cancelled"
2911         except Exception as e:
2912             exc = traceback.format_exc()
2913             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
2914         finally:
2915             if exc:
2916                 error_list.append(str(exc))
2917             try:
2918                 # wait for pending tasks
2919                 if tasks_dict_info:
2920                     stage[1] = "Waiting for terminate pending tasks."
2921                     self.logger.debug(logging_text + stage[1])
2922                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_terminate,
2923                                                              stage, nslcmop_id)
2924                 stage[1] = stage[2] = ""
2925             except asyncio.CancelledError:
2926                 error_list.append("Cancelled")
2927                 # TODO cancell all tasks
2928             except Exception as exc:
2929                 error_list.append(str(exc))
2930             # update status at database
2931             if error_list:
2932                 error_detail = "; ".join(error_list)
2933                 # self.logger.error(logging_text + error_detail)
2934                 error_description_nslcmop = 'Stage: {}. Detail: {}'.format(stage[0], error_detail)
2935                 error_description_nsr = 'Operation: TERMINATING.{}, Stage {}.'.format(nslcmop_id, stage[0])
2936
2937                 db_nsr_update["operational-status"] = "failed"
2938                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
2939                 db_nslcmop_update["detailed-status"] = error_detail
2940                 nslcmop_operation_state = "FAILED"
2941                 ns_state = "BROKEN"
2942             else:
2943                 error_detail = None
2944                 error_description_nsr = error_description_nslcmop = None
2945                 ns_state = "NOT_INSTANTIATED"
2946                 db_nsr_update["operational-status"] = "terminated"
2947                 db_nsr_update["detailed-status"] = "Done"
2948                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
2949                 db_nslcmop_update["detailed-status"] = "Done"
2950                 nslcmop_operation_state = "COMPLETED"
2951
2952             if db_nsr:
2953                 self._write_ns_status(
2954                     nsr_id=nsr_id,
2955                     ns_state=ns_state,
2956                     current_operation="IDLE",
2957                     current_operation_id=None,
2958                     error_description=error_description_nsr,
2959                     error_detail=error_detail,
2960                     other_update=db_nsr_update
2961                 )
2962             self._write_op_status(
2963                 op_id=nslcmop_id,
2964                 stage="",
2965                 error_message=error_description_nslcmop,
2966                 operation_state=nslcmop_operation_state,
2967                 other_update=db_nslcmop_update,
2968             )
2969             if operation_params:
2970                 autoremove = operation_params.get("autoremove", False)
2971             if nslcmop_operation_state:
2972                 try:
2973                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2974                                                                  "operationState": nslcmop_operation_state,
2975                                                                  "autoremove": autoremove},
2976                                             loop=self.loop)
2977                 except Exception as e:
2978                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2979
2980             self.logger.debug(logging_text + "Exit")
2981             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
2982
2983     async def _wait_for_tasks(self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None):
2984         time_start = time()
2985         error_detail_list = []
2986         error_list = []
2987         pending_tasks = list(created_tasks_info.keys())
2988         num_tasks = len(pending_tasks)
2989         num_done = 0
2990         stage[1] = "{}/{}.".format(num_done, num_tasks)
2991         self._write_op_status(nslcmop_id, stage)
2992         while pending_tasks:
2993             new_error = None
2994             _timeout = timeout + time_start - time()
2995             done, pending_tasks = await asyncio.wait(pending_tasks, timeout=_timeout,
2996                                                      return_when=asyncio.FIRST_COMPLETED)
2997             num_done += len(done)
2998             if not done:   # Timeout
2999                 for task in pending_tasks:
3000                     new_error = created_tasks_info[task] + ": Timeout"
3001                     error_detail_list.append(new_error)
3002                     error_list.append(new_error)
3003                 break
3004             for task in done:
3005                 if task.cancelled():
3006                     exc = "Cancelled"
3007                 else:
3008                     exc = task.exception()
3009                 if exc:
3010                     if isinstance(exc, asyncio.TimeoutError):
3011                         exc = "Timeout"
3012                     new_error = created_tasks_info[task] + ": {}".format(exc)
3013                     error_list.append(created_tasks_info[task])
3014                     error_detail_list.append(new_error)
3015                     if isinstance(exc, (str, DbException, N2VCException, ROclient.ROClientException, LcmException,
3016                                         K8sException)):
3017                         self.logger.error(logging_text + new_error)
3018                     else:
3019                         exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__))
3020                         self.logger.error(logging_text + created_tasks_info[task] + exc_traceback)
3021                 else:
3022                     self.logger.debug(logging_text + created_tasks_info[task] + ": Done")
3023             stage[1] = "{}/{}.".format(num_done, num_tasks)
3024             if new_error:
3025                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
3026                 if nsr_id:  # update also nsr
3027                     self.update_db_2("nsrs", nsr_id, {"errorDescription": "Error at: " + ", ".join(error_list),
3028                                                       "errorDetail": ". ".join(error_detail_list)})
3029             self._write_op_status(nslcmop_id, stage)
3030         return error_detail_list
3031
3032     @staticmethod
3033     def _map_primitive_params(primitive_desc, params, instantiation_params):
3034         """
3035         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
3036         The default-value is used. If it is between < > it look for a value at instantiation_params
3037         :param primitive_desc: portion of VNFD/NSD that describes primitive
3038         :param params: Params provided by user
3039         :param instantiation_params: Instantiation params provided by user
3040         :return: a dictionary with the calculated params
3041         """
3042         calculated_params = {}
3043         for parameter in primitive_desc.get("parameter", ()):
3044             param_name = parameter["name"]
3045             if param_name in params:
3046                 calculated_params[param_name] = params[param_name]
3047             elif "default-value" in parameter or "value" in parameter:
3048                 if "value" in parameter:
3049                     calculated_params[param_name] = parameter["value"]
3050                 else:
3051                     calculated_params[param_name] = parameter["default-value"]
3052                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
3053                         and calculated_params[param_name].endswith(">"):
3054                     if calculated_params[param_name][1:-1] in instantiation_params:
3055                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
3056                     else:
3057                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
3058                                            format(calculated_params[param_name], primitive_desc["name"]))
3059             else:
3060                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
3061                                    format(param_name, primitive_desc["name"]))
3062
3063             if isinstance(calculated_params[param_name], (dict, list, tuple)):
3064                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
3065                                                                width=256)
3066             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
3067                 calculated_params[param_name] = calculated_params[param_name][7:]
3068
3069         # add always ns_config_info if primitive name is config
3070         if primitive_desc["name"] == "config":
3071             if "ns_config_info" in instantiation_params:
3072                 calculated_params["ns_config_info"] = instantiation_params["ns_config_info"]
3073         return calculated_params
3074
3075     def _look_for_deployed_vca(self, deployed_vca, member_vnf_index, vdu_id, vdu_count_index, kdu_name=None):
3076         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
3077         for vca in deployed_vca:
3078             if not vca:
3079                 continue
3080             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
3081                 continue
3082             if vdu_count_index is not None and vdu_count_index != vca["vdu_count_index"]:
3083                 continue
3084             if kdu_name and kdu_name != vca["kdu_name"]:
3085                 continue
3086             break
3087         else:
3088             # vca_deployed not found
3089             raise LcmException("charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} is not "
3090                                "deployed".format(member_vnf_index, vdu_id, kdu_name, vdu_count_index))
3091
3092         # get ee_id
3093         ee_id = vca.get("ee_id")
3094         if not ee_id:
3095             raise LcmException("charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
3096                                "execution environment"
3097                                .format(member_vnf_index, vdu_id, kdu_name, vdu_count_index))
3098         return ee_id
3099
3100     async def _ns_execute_primitive(self, ee_id, primitive, primitive_params, retries=0,
3101                                     retries_interval=30, timeout=None) -> (str, str):
3102         try:
3103             if primitive == "config":
3104                 primitive_params = {"params": primitive_params}
3105
3106             while retries >= 0:
3107                 try:
3108                     output = await asyncio.wait_for(
3109                         self.n2vc.exec_primitive(
3110                             ee_id=ee_id,
3111                             primitive_name=primitive,
3112                             params_dict=primitive_params,
3113                             progress_timeout=self.timeout_progress_primitive,
3114                             total_timeout=self.timeout_primitive),
3115                         timeout=timeout or self.timeout_primitive)
3116                     # execution was OK
3117                     break
3118                 except asyncio.CancelledError:
3119                     raise
3120                 except Exception as e:  # asyncio.TimeoutError
3121                     if isinstance(e, asyncio.TimeoutError):
3122                         e = "Timeout"
3123                     retries -= 1
3124                     if retries >= 0:
3125                         self.logger.debug('Error executing action {} on {} -> {}'.format(primitive, ee_id, e))
3126                         # wait and retry
3127                         await asyncio.sleep(retries_interval, loop=self.loop)
3128                     else:
3129                         return 'FAILED', str(e)
3130
3131             return 'COMPLETED', output
3132
3133         except (LcmException, asyncio.CancelledError):
3134             raise
3135         except Exception as e:
3136             return 'FAIL', 'Error executing action {}: {}'.format(primitive, e)
3137
3138     async def action(self, nsr_id, nslcmop_id):
3139
3140         # Try to lock HA task here
3141         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3142         if not task_is_locked_by_me:
3143             return
3144
3145         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
3146         self.logger.debug(logging_text + "Enter")
3147         # get all needed from database
3148         db_nsr = None
3149         db_nslcmop = None
3150         db_nsr_update = {}
3151         db_nslcmop_update = {}
3152         nslcmop_operation_state = None
3153         error_description_nslcmop = None
3154         exc = None
3155         try:
3156             # wait for any previous tasks in process
3157             step = "Waiting for previous operations to terminate"
3158             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3159
3160             self._write_ns_status(
3161                 nsr_id=nsr_id,
3162                 ns_state=None,
3163                 current_operation="RUNNING ACTION",
3164                 current_operation_id=nslcmop_id
3165             )
3166
3167             step = "Getting information from database"
3168             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3169             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3170
3171             nsr_deployed = db_nsr["_admin"].get("deployed")
3172             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3173             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3174             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
3175             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3176             primitive = db_nslcmop["operationParams"]["primitive"]
3177             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
3178             timeout_ns_action = db_nslcmop["operationParams"].get("timeout_ns_action", self.timeout_primitive)
3179
3180             if vnf_index:
3181                 step = "Getting vnfr from database"
3182                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3183                 step = "Getting vnfd from database"
3184                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3185             else:
3186                 step = "Getting nsd from database"
3187                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
3188
3189             # for backward compatibility
3190             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3191                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3192                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3193                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3194
3195             # look for primitive
3196             config_primitive_desc = None
3197             if vdu_id:
3198                 for vdu in get_iterable(db_vnfd, "vdu"):
3199                     if vdu_id == vdu["id"]:
3200                         for config_primitive in deep_get(vdu, ("vdu-configuration", "config-primitive"), ()):
3201                             if config_primitive["name"] == primitive:
3202                                 config_primitive_desc = config_primitive
3203                                 break
3204                         break
3205             elif kdu_name:
3206                 for kdu in get_iterable(db_vnfd, "kdu"):
3207                     if kdu_name == kdu["name"]:
3208                         for config_primitive in deep_get(kdu, ("kdu-configuration", "config-primitive"), ()):
3209                             if config_primitive["name"] == primitive:
3210                                 config_primitive_desc = config_primitive
3211                                 break
3212                         break
3213             elif vnf_index:
3214                 for config_primitive in deep_get(db_vnfd, ("vnf-configuration", "config-primitive"), ()):
3215                     if config_primitive["name"] == primitive:
3216                         config_primitive_desc = config_primitive
3217                         break
3218             else:
3219                 for config_primitive in deep_get(db_nsd, ("ns-configuration", "config-primitive"), ()):
3220                     if config_primitive["name"] == primitive:
3221                         config_primitive_desc = config_primitive
3222                         break
3223
3224             if not config_primitive_desc and not (kdu_name and primitive in ("upgrade", "rollback", "status")):
3225                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
3226                                    format(primitive))
3227
3228             if vnf_index:
3229                 if vdu_id:
3230                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
3231                     desc_params = self._format_additional_params(vdur.get("additionalParams"))
3232                 elif kdu_name:
3233                     kdur = next((x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None)
3234                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
3235                 else:
3236                     desc_params = self._format_additional_params(db_vnfr.get("additionalParamsForVnf"))
3237             else:
3238                 desc_params = self._format_additional_params(db_nsr.get("additionalParamsForNs"))
3239
3240             if kdu_name:
3241                 kdu_action = True if not deep_get(kdu, ("kdu-configuration", "juju")) else False
3242
3243             # TODO check if ns is in a proper status
3244             if kdu_name and (primitive in ("upgrade", "rollback", "status") or kdu_action):
3245                 # kdur and desc_params already set from before
3246                 if primitive_params:
3247                     desc_params.update(primitive_params)
3248                 # TODO Check if we will need something at vnf level
3249                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
3250                     if kdu_name == kdu["kdu-name"] and kdu["member-vnf-index"] == vnf_index:
3251                         break
3252                 else:
3253                     raise LcmException("KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index))
3254
3255                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
3256                     msg = "unknown k8scluster-type '{}'".format(kdu.get("k8scluster-type"))
3257                     raise LcmException(msg)
3258
3259                 db_dict = {"collection": "nsrs",
3260                            "filter": {"_id": nsr_id},
3261                            "path": "_admin.deployed.K8s.{}".format(index)}
3262                 self.logger.debug(logging_text + "Exec k8s {} on {}.{}".format(primitive, vnf_index, kdu_name))
3263                 step = "Executing kdu {}".format(primitive)
3264                 if primitive == "upgrade":
3265                     if desc_params.get("kdu_model"):
3266                         kdu_model = desc_params.get("kdu_model")
3267                         del desc_params["kdu_model"]
3268                     else:
3269                         kdu_model = kdu.get("kdu-model")
3270                         parts = kdu_model.split(sep=":")
3271                         if len(parts) == 2:
3272                             kdu_model = parts[0]
3273
3274                     detailed_status = await asyncio.wait_for(
3275                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
3276                             cluster_uuid=kdu.get("k8scluster-uuid"),
3277                             kdu_instance=kdu.get("kdu-instance"),
3278                             atomic=True, kdu_model=kdu_model,
3279                             params=desc_params, db_dict=db_dict,
3280                             timeout=timeout_ns_action),
3281                         timeout=timeout_ns_action + 10)
3282                     self.logger.debug(logging_text + " Upgrade of kdu {} done".format(detailed_status))
3283                 elif primitive == "rollback":
3284                     detailed_status = await asyncio.wait_for(
3285                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
3286                             cluster_uuid=kdu.get("k8scluster-uuid"),
3287                             kdu_instance=kdu.get("kdu-instance"),
3288                             db_dict=db_dict),
3289                         timeout=timeout_ns_action)
3290                 elif primitive == "status":
3291                     detailed_status = await asyncio.wait_for(
3292                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
3293                             cluster_uuid=kdu.get("k8scluster-uuid"),
3294                             kdu_instance=kdu.get("kdu-instance")),
3295                         timeout=timeout_ns_action)
3296                 else:
3297                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(kdu["kdu-name"], nsr_id)
3298                     params = self._map_primitive_params(config_primitive_desc, primitive_params, desc_params)
3299
3300                     detailed_status = await asyncio.wait_for(
3301                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
3302                             cluster_uuid=kdu.get("k8scluster-uuid"),
3303                             kdu_instance=kdu_instance,
3304                             primitive_name=primitive,
3305                             params=params, db_dict=db_dict,
3306                             timeout=timeout_ns_action),
3307                         timeout=timeout_ns_action)
3308
3309                 if detailed_status:
3310                     nslcmop_operation_state = 'COMPLETED'
3311                 else:
3312                     detailed_status = ''
3313                     nslcmop_operation_state = 'FAILED'
3314             else:
3315                 nslcmop_operation_state, detailed_status = await self._ns_execute_primitive(
3316                     self._look_for_deployed_vca(nsr_deployed["VCA"],
3317                                                 member_vnf_index=vnf_index,
3318                                                 vdu_id=vdu_id,
3319                                                 vdu_count_index=vdu_count_index),
3320                     primitive=primitive,
3321                     primitive_params=self._map_primitive_params(config_primitive_desc, primitive_params, desc_params),
3322                     timeout=timeout_ns_action)
3323
3324             db_nslcmop_update["detailed-status"] = detailed_status
3325             error_description_nslcmop = detailed_status if nslcmop_operation_state == "FAILED" else ""
3326             self.logger.debug(logging_text + " task Done with result {} {}".format(nslcmop_operation_state,
3327                                                                                    detailed_status))
3328             return  # database update is called inside finally
3329
3330         except (DbException, LcmException, N2VCException, K8sException) as e:
3331             self.logger.error(logging_text + "Exit Exception {}".format(e))
3332             exc = e
3333         except asyncio.CancelledError:
3334             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
3335             exc = "Operation was cancelled"
3336         except asyncio.TimeoutError:
3337             self.logger.error(logging_text + "Timeout while '{}'".format(step))
3338             exc = "Timeout"
3339         except Exception as e:
3340             exc = traceback.format_exc()
3341             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
3342         finally:
3343             if exc:
3344                 db_nslcmop_update["detailed-status"] = detailed_status = error_description_nslcmop = \
3345                     "FAILED {}: {}".format(step, exc)
3346                 nslcmop_operation_state = "FAILED"
3347             if db_nsr:
3348                 self._write_ns_status(
3349                     nsr_id=nsr_id,
3350                     ns_state=db_nsr["nsState"],   # TODO check if degraded. For the moment use previous status
3351                     current_operation="IDLE",
3352                     current_operation_id=None,
3353                     # error_description=error_description_nsr,
3354                     # error_detail=error_detail,
3355                     other_update=db_nsr_update
3356                 )
3357
3358             self._write_op_status(
3359                 op_id=nslcmop_id,
3360                 stage="",
3361                 error_message=error_description_nslcmop,
3362                 operation_state=nslcmop_operation_state,
3363                 other_update=db_nslcmop_update,
3364             )
3365
3366             if nslcmop_operation_state:
3367                 try:
3368                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3369                                                                "operationState": nslcmop_operation_state},
3370                                             loop=self.loop)
3371                 except Exception as e:
3372                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3373             self.logger.debug(logging_text + "Exit")
3374             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
3375             return nslcmop_operation_state, detailed_status
3376
3377     async def scale(self, nsr_id, nslcmop_id):
3378
3379         # Try to lock HA task here
3380         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3381         if not task_is_locked_by_me:
3382             return
3383
3384         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
3385         self.logger.debug(logging_text + "Enter")
3386         # get all needed from database
3387         db_nsr = None
3388         db_nslcmop = None
3389         db_nslcmop_update = {}
3390         nslcmop_operation_state = None
3391         db_nsr_update = {}
3392         exc = None
3393         # in case of error, indicates what part of scale was failed to put nsr at error status
3394         scale_process = None
3395         old_operational_status = ""
3396         old_config_status = ""
3397         vnfr_scaled = False
3398         try:
3399             # wait for any previous tasks in process
3400             step = "Waiting for previous operations to terminate"
3401             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3402
3403             self._write_ns_status(
3404                 nsr_id=nsr_id,
3405                 ns_state=None,
3406                 current_operation="SCALING",
3407                 current_operation_id=nslcmop_id
3408             )
3409
3410             step = "Getting nslcmop from database"
3411             self.logger.debug(step + " after having waited for previous tasks to be completed")
3412             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3413             step = "Getting nsr from database"
3414             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3415
3416             old_operational_status = db_nsr["operational-status"]
3417             old_config_status = db_nsr["config-status"]
3418             step = "Parsing scaling parameters"
3419             # self.logger.debug(step)
3420             db_nsr_update["operational-status"] = "scaling"
3421             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3422             nsr_deployed = db_nsr["_admin"].get("deployed")
3423
3424             #######
3425             nsr_deployed = db_nsr["_admin"].get("deployed")
3426             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3427             # vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3428             # vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3429             # vdu_name = db_nslcmop["operationParams"].get("vdu_name")
3430             #######
3431
3432             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
3433             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
3434             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
3435             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
3436             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
3437
3438             # for backward compatibility
3439             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3440                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3441                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3442                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3443
3444             step = "Getting vnfr from database"
3445             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3446             step = "Getting vnfd from database"
3447             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3448
3449             step = "Getting scaling-group-descriptor"
3450             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
3451                 if scaling_descriptor["name"] == scaling_group:
3452                     break
3453             else:
3454                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
3455                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
3456
3457             # cooldown_time = 0
3458             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
3459             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
3460             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
3461             #         break
3462
3463             # TODO check if ns is in a proper status
3464             step = "Sending scale order to VIM"
3465             nb_scale_op = 0
3466             if not db_nsr["_admin"].get("scaling-group"):
3467                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
3468                 admin_scale_index = 0
3469             else:
3470                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
3471                     if admin_scale_info["name"] == scaling_group:
3472                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
3473                         break
3474                 else:  # not found, set index one plus last element and add new entry with the name
3475                     admin_scale_index += 1
3476                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
3477             RO_scaling_info = []
3478             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
3479             if scaling_type == "SCALE_OUT":
3480                 # count if max-instance-count is reached
3481                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
3482                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
3483                 if nb_scale_op >= max_instance_count:
3484                     raise LcmException("reached the limit of {} (max-instance-count) "
3485                                        "scaling-out operations for the "
3486                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3487
3488                 nb_scale_op += 1
3489                 vdu_scaling_info["scaling_direction"] = "OUT"
3490                 vdu_scaling_info["vdu-create"] = {}
3491                 for vdu_scale_info in scaling_descriptor["vdu"]:
3492                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3493                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
3494                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3495
3496             elif scaling_type == "SCALE_IN":
3497                 # count if min-instance-count is reached
3498                 min_instance_count = 0
3499                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
3500                     min_instance_count = int(scaling_descriptor["min-instance-count"])
3501                 if nb_scale_op <= min_instance_count:
3502                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
3503                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3504                 nb_scale_op -= 1
3505                 vdu_scaling_info["scaling_direction"] = "IN"
3506                 vdu_scaling_info["vdu-delete"] = {}
3507                 for vdu_scale_info in scaling_descriptor["vdu"]:
3508                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3509                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
3510                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3511
3512             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
3513             vdu_create = vdu_scaling_info.get("vdu-create")
3514             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
3515             if vdu_scaling_info["scaling_direction"] == "IN":
3516                 for vdur in reversed(db_vnfr["vdur"]):
3517                     if vdu_delete.get(vdur["vdu-id-ref"]):
3518                         vdu_delete[vdur["vdu-id-ref"]] -= 1
3519                         vdu_scaling_info["vdu"].append({
3520                             "name": vdur["name"],
3521                             "vdu_id": vdur["vdu-id-ref"],
3522                             "interface": []
3523                         })
3524                         for interface in vdur["interfaces"]:
3525                             vdu_scaling_info["vdu"][-1]["interface"].append({
3526                                 "name": interface["name"],
3527                                 "ip_address": interface["ip-address"],
3528                                 "mac_address": interface.get("mac-address"),
3529                             })
3530                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
3531
3532             # PRE-SCALE BEGIN
3533             step = "Executing pre-scale vnf-config-primitive"
3534             if scaling_descriptor.get("scaling-config-action"):
3535                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
3536                     if (scaling_config_action.get("trigger") == "pre-scale-in" and scaling_type == "SCALE_IN") \
3537                        or (scaling_config_action.get("trigger") == "pre-scale-out" and scaling_type == "SCALE_OUT"):
3538                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
3539                         step = db_nslcmop_update["detailed-status"] = \
3540                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
3541
3542                         # look for primitive
3543                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
3544                             if config_primitive["name"] == vnf_config_primitive:
3545                                 break
3546                         else:
3547                             raise LcmException(
3548                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
3549                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
3550                                 "primitive".format(scaling_group, config_primitive))
3551
3552                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
3553                         if db_vnfr.get("additionalParamsForVnf"):
3554                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
3555
3556                         scale_process = "VCA"
3557                         db_nsr_update["config-status"] = "configuring pre-scaling"
3558                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
3559
3560                         # Pre-scale retry check: Check if this sub-operation has been executed before
3561                         op_index = self._check_or_add_scale_suboperation(
3562                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
3563                         if op_index == self.SUBOPERATION_STATUS_SKIP:
3564                             # Skip sub-operation
3565                             result = 'COMPLETED'
3566                             result_detail = 'Done'
3567                             self.logger.debug(logging_text +
3568                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
3569                                                   vnf_config_primitive, result, result_detail))
3570                         else:
3571                             if op_index == self.SUBOPERATION_STATUS_NEW:
3572                                 # New sub-operation: Get index of this sub-operation
3573                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3574                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
3575                                                   format(vnf_config_primitive))
3576                             else:
3577                                 # retry:  Get registered params for this existing sub-operation
3578                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3579                                 vnf_index = op.get('member_vnf_index')
3580                                 vnf_config_primitive = op.get('primitive')
3581                                 primitive_params = op.get('primitive_params')
3582                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
3583                                                   format(vnf_config_primitive))
3584                             # Execute the primitive, either with new (first-time) or registered (retry) args
3585                             result, result_detail = await self._ns_execute_primitive(
3586                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
3587                                                             member_vnf_index=vnf_index,
3588                                                             vdu_id=None,
3589                                                             vdu_count_index=None),
3590                                 vnf_config_primitive, primitive_params)
3591                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
3592                                 vnf_config_primitive, result, result_detail))
3593                             # Update operationState = COMPLETED | FAILED
3594                             self._update_suboperation_status(
3595                                 db_nslcmop, op_index, result, result_detail)
3596
3597                         if result == "FAILED":
3598                             raise LcmException(result_detail)
3599                         db_nsr_update["config-status"] = old_config_status
3600                         scale_process = None
3601             # PRE-SCALE END
3602
3603             # SCALE RO - BEGIN
3604             # Should this block be skipped if 'RO_nsr_id' == None ?
3605             # if (RO_nsr_id and RO_scaling_info):
3606             if RO_scaling_info:
3607                 scale_process = "RO"
3608                 # Scale RO retry check: Check if this sub-operation has been executed before
3609                 op_index = self._check_or_add_scale_suboperation(
3610                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
3611                 if op_index == self.SUBOPERATION_STATUS_SKIP:
3612                     # Skip sub-operation
3613                     result = 'COMPLETED'
3614                     result_detail = 'Done'
3615                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
3616                         result, result_detail))
3617                 else:
3618                     if op_index == self.SUBOPERATION_STATUS_NEW:
3619                         # New sub-operation: Get index of this sub-operation
3620                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3621                         self.logger.debug(logging_text + "New sub-operation RO")
3622                     else:
3623                         # retry:  Get registered params for this existing sub-operation
3624                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3625                         RO_nsr_id = op.get('RO_nsr_id')
3626                         RO_scaling_info = op.get('RO_scaling_info')
3627                         self.logger.debug(logging_text + "Sub-operation RO retry for primitive {}".format(
3628                             vnf_config_primitive))
3629
3630                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
3631                     db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
3632                     db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
3633                     # wait until ready
3634                     RO_nslcmop_id = RO_desc["instance_action_id"]
3635                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
3636
3637                     RO_task_done = False
3638                     step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
3639                     detailed_status_old = None
3640                     self.logger.debug(logging_text + step)
3641
3642                     deployment_timeout = 1 * 3600   # One hour
3643                     while deployment_timeout > 0:
3644                         if not RO_task_done:
3645                             desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
3646                                                       extra_item_id=RO_nslcmop_id)
3647
3648                             # deploymentStatus
3649                             self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
3650
3651                             ns_status, ns_status_info = self.RO.check_action_status(desc)
3652                             if ns_status == "ERROR":
3653                                 raise ROclient.ROClientException(ns_status_info)
3654                             elif ns_status == "BUILD":
3655                                 detailed_status = step + "; {}".format(ns_status_info)
3656                             elif ns_status == "ACTIVE":
3657                                 RO_task_done = True
3658                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
3659                                 self.logger.debug(logging_text + step)
3660                             else:
3661                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
3662                         else:
3663
3664                             if ns_status == "ERROR":
3665                                 raise ROclient.ROClientException(ns_status_info)
3666                             elif ns_status == "BUILD":
3667                                 detailed_status = step + "; {}".format(ns_status_info)
3668                             elif ns_status == "ACTIVE":
3669                                 step = detailed_status = \
3670                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
3671                                 if not vnfr_scaled:
3672                                     self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
3673                                     vnfr_scaled = True
3674                                 try:
3675                                     desc = await self.RO.show("ns", RO_nsr_id)
3676
3677                                     # deploymentStatus
3678                                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
3679
3680                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
3681                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
3682                                     break
3683                                 except LcmExceptionNoMgmtIP:
3684                                     pass
3685                             else:
3686                                 assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
3687                         if detailed_status != detailed_status_old:
3688                             self._update_suboperation_status(
3689                                 db_nslcmop, op_index, 'COMPLETED', detailed_status)
3690                             detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
3691                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
3692
3693                         await asyncio.sleep(5, loop=self.loop)
3694                         deployment_timeout -= 5
3695                     if deployment_timeout <= 0:
3696                         self._update_suboperation_status(
3697                             db_nslcmop, nslcmop_id, op_index, 'FAILED', "Timeout when waiting for ns to get ready")
3698                         raise ROclient.ROClientException("Timeout waiting ns to be ready")
3699
3700                     # update VDU_SCALING_INFO with the obtained ip_addresses
3701                     if vdu_scaling_info["scaling_direction"] == "OUT":
3702                         for vdur in reversed(db_vnfr["vdur"]):
3703                             if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
3704                                 vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
3705                                 vdu_scaling_info["vdu"].append({
3706                                     "name": vdur["name"],
3707                                     "vdu_id": vdur["vdu-id-ref"],
3708                                     "interface": []
3709                                 })
3710                                 for interface in vdur["interfaces"]:
3711                                     vdu_scaling_info["vdu"][-1]["interface"].append({
3712                                         "name": interface["name"],
3713                                         "ip_address": interface["ip-address"],
3714                                         "mac_address": interface.get("mac-address"),
3715                                     })
3716                         del vdu_scaling_info["vdu-create"]
3717
3718                     self._update_suboperation_status(db_nslcmop, op_index, 'COMPLETED', 'Done')
3719             # SCALE RO - END
3720
3721             scale_process = None
3722             if db_nsr_update:
3723                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3724
3725             # POST-SCALE BEGIN
3726             # execute primitive service POST-SCALING
3727             step = "Executing post-scale vnf-config-primitive"
3728             if scaling_descriptor.get("scaling-config-action"):
3729                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
3730                     if (scaling_config_action.get("trigger") == "post-scale-in" and scaling_type == "SCALE_IN") \
3731                        or (scaling_config_action.get("trigger") == "post-scale-out" and scaling_type == "SCALE_OUT"):
3732                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
3733                         step = db_nslcmop_update["detailed-status"] = \
3734                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
3735
3736                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
3737                         if db_vnfr.get("additionalParamsForVnf"):
3738                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
3739
3740                         # look for primitive
3741                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
3742                             if config_primitive["name"] == vnf_config_primitive:
3743                                 break
3744                         else:
3745                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
3746                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
3747                                                "match any vnf-configuration:config-primitive".format(scaling_group,
3748                                                                                                      config_primitive))
3749                         scale_process = "VCA"
3750                         db_nsr_update["config-status"] = "configuring post-scaling"
3751                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
3752
3753                         # Post-scale retry check: Check if this sub-operation has been executed before
3754                         op_index = self._check_or_add_scale_suboperation(
3755                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
3756                         if op_index == self.SUBOPERATION_STATUS_SKIP:
3757                             # Skip sub-operation
3758                             result = 'COMPLETED'
3759                             result_detail = 'Done'
3760                             self.logger.debug(logging_text +
3761                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".
3762                                               format(vnf_config_primitive, result, result_detail))
3763                         else:
3764                             if op_index == self.SUBOPERATION_STATUS_NEW:
3765                                 # New sub-operation: Get index of this sub-operation
3766                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3767                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
3768                                                   format(vnf_config_primitive))
3769                             else:
3770                                 # retry:  Get registered params for this existing sub-operation
3771                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3772                                 vnf_index = op.get('member_vnf_index')
3773                                 vnf_config_primitive = op.get('primitive')
3774                                 primitive_params = op.get('primitive_params')
3775                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
3776                                                   format(vnf_config_primitive))
3777                             # Execute the primitive, either with new (first-time) or registered (retry) args
3778                             result, result_detail = await self._ns_execute_primitive(
3779                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
3780                                                             member_vnf_index=vnf_index,
3781                                                             vdu_id=None,
3782                                                             vdu_count_index=None),
3783                                 vnf_config_primitive, primitive_params)
3784                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
3785                                 vnf_config_primitive, result, result_detail))
3786                             # Update operationState = COMPLETED | FAILED
3787                             self._update_suboperation_status(
3788                                 db_nslcmop, op_index, result, result_detail)
3789
3790                         if result == "FAILED":
3791                             raise LcmException(result_detail)
3792                         db_nsr_update["config-status"] = old_config_status
3793                         scale_process = None
3794             # POST-SCALE END
3795
3796             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
3797             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
3798                 else old_operational_status
3799             db_nsr_update["config-status"] = old_config_status
3800             return
3801         except (ROclient.ROClientException, DbException, LcmException) as e:
3802             self.logger.error(logging_text + "Exit Exception {}".format(e))
3803             exc = e
3804         except asyncio.CancelledError:
3805             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
3806             exc = "Operation was cancelled"
3807         except Exception as e:
3808             exc = traceback.format_exc()
3809             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
3810         finally:
3811             self._write_ns_status(
3812                 nsr_id=nsr_id,
3813                 ns_state=None,
3814                 current_operation="IDLE",
3815                 current_operation_id=None
3816             )
3817             if exc:
3818                 db_nslcmop_update["detailed-status"] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
3819                 nslcmop_operation_state = "FAILED"
3820                 if db_nsr:
3821                     db_nsr_update["operational-status"] = old_operational_status
3822                     db_nsr_update["config-status"] = old_config_status
3823                     db_nsr_update["detailed-status"] = ""
3824                     if scale_process:
3825                         if "VCA" in scale_process:
3826                             db_nsr_update["config-status"] = "failed"
3827                         if "RO" in scale_process:
3828                             db_nsr_update["operational-status"] = "failed"
3829                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
3830                                                                                                      exc)
3831             else:
3832                 error_description_nslcmop = None
3833                 nslcmop_operation_state = "COMPLETED"
3834                 db_nslcmop_update["detailed-status"] = "Done"
3835
3836             self._write_op_status(
3837                 op_id=nslcmop_id,
3838                 stage="",
3839                 error_message=error_description_nslcmop,
3840                 operation_state=nslcmop_operation_state,
3841                 other_update=db_nslcmop_update,
3842             )
3843             if db_nsr:
3844                 self._write_ns_status(
3845                     nsr_id=nsr_id,
3846                     ns_state=None,
3847                     current_operation="IDLE",
3848                     current_operation_id=None,
3849                     other_update=db_nsr_update
3850                 )
3851
3852             if nslcmop_operation_state:
3853                 try:
3854                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3855                                                              "operationState": nslcmop_operation_state},
3856                                             loop=self.loop)
3857                     # if cooldown_time:
3858                     #     await asyncio.sleep(cooldown_time, loop=self.loop)
3859                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
3860                 except Exception as e:
3861                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3862             self.logger.debug(logging_text + "Exit")
3863             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")