osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import traceback
  24 import json
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.ng_ro import NgRoClient, NgRoException
  29 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase, deep_get, get_iterable, populate_dict
  30 from n2vc.k8s_helm_conn import K8sHelmConnector
  31 from n2vc.k8s_juju_conn import K8sJujuConnector
  32
  33 from osm_common.dbbase import DbException
  34 from osm_common.fsbase import FsException
  35
  36 from n2vc.n2vc_juju_conn import N2VCJujuConnector
  37 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
  38
  39 from copy import copy, deepcopy
  40 from http import HTTPStatus
  41 from time import time
  42 from uuid import uuid4
  43 from functools import partial
  44
  45 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
  46
  47
  48 class NsLcm(LcmBase):
  49     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  50     timeout_ns_deploy = 2 * 3600   # default global timeout for deployment a ns
  51     timeout_ns_terminate = 1800   # default global timeout for un deployment a ns
  52     timeout_charm_delete = 10 * 60
  53     timeout_primitive = 30 * 60  # timeout for primitive execution
  54     timeout_progress_primitive = 10 * 60  # timeout for some progress in a primitive execution
  55
  56     SUBOPERATION_STATUS_NOT_FOUND = -1
  57     SUBOPERATION_STATUS_NEW = -2
  58     SUBOPERATION_STATUS_SKIP = -3
  59     task_name_deploy_vca = "Deploying VCA"
  60
  61     def __init__(self, db, msg, fs, lcm_tasks, config, loop):
  62         """
  63         Init, Connect to database, filesystem storage, and messaging
  64         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  65         :return: None
  66         """
  67         super().__init__(
  68             db=db,
  69             msg=msg,
  70             fs=fs,
  71             logger=logging.getLogger('lcm.ns')
  72         )
  73
  74         self.loop = loop
  75         self.lcm_tasks = lcm_tasks
  76         self.timeout = config["timeout"]
  77         self.ro_config = config["ro_config"]
  78         self.ng_ro = config["ro_config"].get("ng")
  79         self.vca_config = config["VCA"].copy()
  80
  81         # create N2VC connector
  82         self.n2vc = N2VCJujuConnector(
  83             db=self.db,
  84             fs=self.fs,
  85             log=self.logger,
  86             loop=self.loop,
  87             url='{}:{}'.format(self.vca_config['host'], self.vca_config['port']),
  88             username=self.vca_config.get('user', None),
  89             vca_config=self.vca_config,
  90             on_update_db=self._on_update_n2vc_db
  91         )
  92
  93         self.k8sclusterhelm = K8sHelmConnector(
  94             kubectl_command=self.vca_config.get("kubectlpath"),
  95             helm_command=self.vca_config.get("helmpath"),
  96             fs=self.fs,
  97             log=self.logger,
  98             db=self.db,
  99             on_update_db=None,
 100         )
 101
 102         self.k8sclusterjuju = K8sJujuConnector(
 103             kubectl_command=self.vca_config.get("kubectlpath"),
 104             juju_command=self.vca_config.get("jujupath"),
 105             fs=self.fs,
 106             log=self.logger,
 107             db=self.db,
 108             on_update_db=None,
 109         )
 110
 111         self.k8scluster_map = {
 112             "helm-chart": self.k8sclusterhelm,
 113             "chart": self.k8sclusterhelm,
 114             "juju-bundle": self.k8sclusterjuju,
 115             "juju": self.k8sclusterjuju,
 116         }
 117         # create RO client
 118         if self.ng_ro:
 119             self.RO = NgRoClient(self.loop, **self.ro_config)
 120         else:
 121             self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 122
 123     def _on_update_ro_db(self, nsrs_id, ro_descriptor):
 124
 125         # self.logger.debug('_on_update_ro_db(nsrs_id={}'.format(nsrs_id))
 126
 127         try:
 128             # TODO filter RO descriptor fields...
 129
 130             # write to database
 131             db_dict = dict()
 132             # db_dict['deploymentStatus'] = yaml.dump(ro_descriptor, default_flow_style=False, indent=2)
 133             db_dict['deploymentStatus'] = ro_descriptor
 134             self.update_db_2("nsrs", nsrs_id, db_dict)
 135
 136         except Exception as e:
 137             self.logger.warn('Cannot write database RO deployment for ns={} -> {}'.format(nsrs_id, e))
 138
 139     async def _on_update_n2vc_db(self, table, filter, path, updated_data):
 140
 141         # remove last dot from path (if exists)
 142         if path.endswith('.'):
 143             path = path[:-1]
 144
 145         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 146         #                   .format(table, filter, path, updated_data))
 147
 148         try:
 149
 150             nsr_id = filter.get('_id')
 151
 152             # read ns record from database
 153             nsr = self.db.get_one(table='nsrs', q_filter=filter)
 154             current_ns_status = nsr.get('nsState')
 155
 156             # get vca status for NS
 157             status_dict = await self.n2vc.get_status(namespace='.' + nsr_id, yaml_format=False)
 158
 159             # vcaStatus
 160             db_dict = dict()
 161             db_dict['vcaStatus'] = status_dict
 162
 163             # update configurationStatus for this VCA
 164             try:
 165                 vca_index = int(path[path.rfind(".")+1:])
 166
 167                 vca_list = deep_get(target_dict=nsr, key_list=('_admin', 'deployed', 'VCA'))
 168                 vca_status = vca_list[vca_index].get('status')
 169
 170                 configuration_status_list = nsr.get('configurationStatus')
 171                 config_status = configuration_status_list[vca_index].get('status')
 172
 173                 if config_status == 'BROKEN' and vca_status != 'failed':
 174                     db_dict['configurationStatus'][vca_index] = 'READY'
 175                 elif config_status != 'BROKEN' and vca_status == 'failed':
 176                     db_dict['configurationStatus'][vca_index] = 'BROKEN'
 177             except Exception as e:
 178                 # not update configurationStatus
 179                 self.logger.debug('Error updating vca_index (ignore): {}'.format(e))
 180
 181             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 182             # if nsState = 'DEGRADED' check if all is OK
 183             is_degraded = False
 184             if current_ns_status in ('READY', 'DEGRADED'):
 185                 error_description = ''
 186                 # check machines
 187                 if status_dict.get('machines'):
 188                     for machine_id in status_dict.get('machines'):
 189                         machine = status_dict.get('machines').get(machine_id)
 190                         # check machine agent-status
 191                         if machine.get('agent-status'):
 192                             s = machine.get('agent-status').get('status')
 193                             if s != 'started':
 194                                 is_degraded = True
 195                                 error_description += 'machine {} agent-status={} ; '.format(machine_id, s)
 196                         # check machine instance status
 197                         if machine.get('instance-status'):
 198                             s = machine.get('instance-status').get('status')
 199                             if s != 'running':
 200                                 is_degraded = True
 201                                 error_description += 'machine {} instance-status={} ; '.format(machine_id, s)
 202                 # check applications
 203                 if status_dict.get('applications'):
 204                     for app_id in status_dict.get('applications'):
 205                         app = status_dict.get('applications').get(app_id)
 206                         # check application status
 207                         if app.get('status'):
 208                             s = app.get('status').get('status')
 209                             if s != 'active':
 210                                 is_degraded = True
 211                                 error_description += 'application {} status={} ; '.format(app_id, s)
 212
 213                 if error_description:
 214                     db_dict['errorDescription'] = error_description
 215                 if current_ns_status == 'READY' and is_degraded:
 216                     db_dict['nsState'] = 'DEGRADED'
 217                 if current_ns_status == 'DEGRADED' and not is_degraded:
 218                     db_dict['nsState'] = 'READY'
 219
 220             # write to database
 221             self.update_db_2("nsrs", nsr_id, db_dict)
 222
 223         except (asyncio.CancelledError, asyncio.TimeoutError):
 224             raise
 225         except Exception as e:
 226             self.logger.warn('Error updating NS state for ns={}: {}'.format(nsr_id, e))
 227
 228     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 229         """
 230         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 231         :param vnfd: input vnfd
 232         :param new_id: overrides vnf id if provided
 233         :param additionalParams: Instantiation params for VNFs provided
 234         :param nsrId: Id of the NSR
 235         :return: copy of vnfd
 236         """
 237         try:
 238             vnfd_RO = deepcopy(vnfd)
 239             # remove unused by RO configuration, monitoring, scaling and internal keys
 240             vnfd_RO.pop("_id", None)
 241             vnfd_RO.pop("_admin", None)
 242             vnfd_RO.pop("vnf-configuration", None)
 243             vnfd_RO.pop("monitoring-param", None)
 244             vnfd_RO.pop("scaling-group-descriptor", None)
 245             vnfd_RO.pop("kdu", None)
 246             vnfd_RO.pop("k8s-cluster", None)
 247             if new_id:
 248                 vnfd_RO["id"] = new_id
 249
 250             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 251             for vdu in get_iterable(vnfd_RO, "vdu"):
 252                 cloud_init_file = None
 253                 if vdu.get("cloud-init-file"):
 254                     base_folder = vnfd["_admin"]["storage"]
 255                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 256                                                                    vdu["cloud-init-file"])
 257                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 258                         cloud_init_content = ci_file.read()
 259                     vdu.pop("cloud-init-file", None)
 260                 elif vdu.get("cloud-init"):
 261                     cloud_init_content = vdu["cloud-init"]
 262                 else:
 263                     continue
 264
 265                 env = Environment()
 266                 ast = env.parse(cloud_init_content)
 267                 mandatory_vars = meta.find_undeclared_variables(ast)
 268                 if mandatory_vars:
 269                     for var in mandatory_vars:
 270                         if not additionalParams or var not in additionalParams.keys():
 271                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 272                                                "file, must be provided in the instantiation parameters inside the "
 273                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 274                 template = Template(cloud_init_content)
 275                 cloud_init_content = template.render(additionalParams or {})
 276                 vdu["cloud-init"] = cloud_init_content
 277
 278             return vnfd_RO
 279         except FsException as e:
 280             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 281                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 282         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 283             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 284                                format(vnfd["id"], vdu["id"], e))
 285
 286     def _ns_params_2_RO(self, ns_params, nsd, vnfd_dict, db_vnfrs, n2vc_key_list):
 287         """
 288         Creates a RO ns descriptor from OSM ns_instantiate params
 289         :param ns_params: OSM instantiate params
 290         :param vnfd_dict: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
 291         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index. {member-vnf-index: {vnfr_object}, ...}
 292         :return: The RO ns descriptor
 293         """
 294         vim_2_RO = {}
 295         wim_2_RO = {}
 296         # TODO feature 1417: Check that no instantiation is set over PDU
 297         # check if PDU forces a concrete vim-network-id and add it
 298         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 299
 300         def vim_account_2_RO(vim_account):
 301             if vim_account in vim_2_RO:
 302                 return vim_2_RO[vim_account]
 303
 304             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 305             if db_vim["_admin"]["operationalState"] != "ENABLED":
 306                 raise LcmException("VIM={} is not available. operationalState={}".format(
 307                     vim_account, db_vim["_admin"]["operationalState"]))
 308             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 309             vim_2_RO[vim_account] = RO_vim_id
 310             return RO_vim_id
 311
 312         def wim_account_2_RO(wim_account):
 313             if isinstance(wim_account, str):
 314                 if wim_account in wim_2_RO:
 315                     return wim_2_RO[wim_account]
 316
 317                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 318                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 319                     raise LcmException("WIM={} is not available. operationalState={}".format(
 320                         wim_account, db_wim["_admin"]["operationalState"]))
 321                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 322                 wim_2_RO[wim_account] = RO_wim_id
 323                 return RO_wim_id
 324             else:
 325                 return wim_account
 326
 327         def ip_profile_2_RO(ip_profile):
 328             RO_ip_profile = deepcopy((ip_profile))
 329             if "dns-server" in RO_ip_profile:
 330                 if isinstance(RO_ip_profile["dns-server"], list):
 331                     RO_ip_profile["dns-address"] = []
 332                     for ds in RO_ip_profile.pop("dns-server"):
 333                         RO_ip_profile["dns-address"].append(ds['address'])
 334                 else:
 335                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 336             if RO_ip_profile.get("ip-version") == "ipv4":
 337                 RO_ip_profile["ip-version"] = "IPv4"
 338             if RO_ip_profile.get("ip-version") == "ipv6":
 339                 RO_ip_profile["ip-version"] = "IPv6"
 340             if "dhcp-params" in RO_ip_profile:
 341                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 342             return RO_ip_profile
 343
 344         if not ns_params:
 345             return None
 346         RO_ns_params = {
 347             # "name": ns_params["nsName"],
 348             # "description": ns_params.get("nsDescription"),
 349             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 350             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 351             # "scenario": ns_params["nsdId"],
 352         }
 353         # set vim_account of each vnf if different from general vim_account.
 354         # Get this information from <vnfr> database content, key vim-account-id
 355         # Vim account can be set by placement_engine and it may be different from
 356         # the instantiate parameters (vnfs.member-vnf-index.datacenter).
 357         for vnf_index, vnfr in db_vnfrs.items():
 358             if vnfr.get("vim-account-id") and vnfr["vim-account-id"] != ns_params["vimAccountId"]:
 359                 populate_dict(RO_ns_params, ("vnfs", vnf_index, "datacenter"), vim_account_2_RO(vnfr["vim-account-id"]))
 360
 361         n2vc_key_list = n2vc_key_list or []
 362         for vnfd_ref, vnfd in vnfd_dict.items():
 363             vdu_needed_access = []
 364             mgmt_cp = None
 365             if vnfd.get("vnf-configuration"):
 366                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 367                 if ssh_required and vnfd.get("mgmt-interface"):
 368                     if vnfd["mgmt-interface"].get("vdu-id"):
 369                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 370                     elif vnfd["mgmt-interface"].get("cp"):
 371                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 372
 373             for vdu in vnfd.get("vdu", ()):
 374                 if vdu.get("vdu-configuration"):
 375                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 376                     if ssh_required:
 377                         vdu_needed_access.append(vdu["id"])
 378                 elif mgmt_cp:
 379                     for vdu_interface in vdu.get("interface"):
 380                         if vdu_interface.get("external-connection-point-ref") and \
 381                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 382                             vdu_needed_access.append(vdu["id"])
 383                             mgmt_cp = None
 384                             break
 385
 386             if vdu_needed_access:
 387                 for vnf_member in nsd.get("constituent-vnfd"):
 388                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 389                         continue
 390                     for vdu in vdu_needed_access:
 391                         populate_dict(RO_ns_params,
 392                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 393                                       n2vc_key_list)
 394
 395         if ns_params.get("vduImage"):
 396             RO_ns_params["vduImage"] = ns_params["vduImage"]
 397
 398         if ns_params.get("ssh_keys"):
 399             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 400         for vnf_params in get_iterable(ns_params, "vnf"):
 401             for constituent_vnfd in nsd["constituent-vnfd"]:
 402                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 403                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 404                     break
 405             else:
 406                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 407                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 408
 409             for vdu_params in get_iterable(vnf_params, "vdu"):
 410                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 411                 if vdu_params.get("volume"):
 412                     for volume_params in vdu_params["volume"]:
 413                         if volume_params.get("vim-volume-id"):
 414                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 415                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 416                                           volume_params["vim-volume-id"])
 417                 if vdu_params.get("interface"):
 418                     for interface_params in vdu_params["interface"]:
 419                         if interface_params.get("ip-address"):
 420                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 421                                                          vdu_params["id"], "interfaces", interface_params["name"],
 422                                                          "ip_address"),
 423                                           interface_params["ip-address"])
 424                         if interface_params.get("mac-address"):
 425                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 426                                                          vdu_params["id"], "interfaces", interface_params["name"],
 427                                                          "mac_address"),
 428                                           interface_params["mac-address"])
 429                         if interface_params.get("floating-ip-required"):
 430                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 431                                                          vdu_params["id"], "interfaces", interface_params["name"],
 432                                                          "floating-ip"),
 433                                           interface_params["floating-ip-required"])
 434
 435             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 436                 if internal_vld_params.get("vim-network-name"):
 437                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 438                                                  internal_vld_params["name"], "vim-network-name"),
 439                                   internal_vld_params["vim-network-name"])
 440                 if internal_vld_params.get("vim-network-id"):
 441                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 442                                                  internal_vld_params["name"], "vim-network-id"),
 443                                   internal_vld_params["vim-network-id"])
 444                 if internal_vld_params.get("ip-profile"):
 445                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 446                                                  internal_vld_params["name"], "ip-profile"),
 447                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 448                 if internal_vld_params.get("provider-network"):
 449
 450                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 451                                                  internal_vld_params["name"], "provider-network"),
 452                                   internal_vld_params["provider-network"].copy())
 453
 454                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 455                     # look for interface
 456                     iface_found = False
 457                     for vdu_descriptor in vnf_descriptor["vdu"]:
 458                         for vdu_interface in vdu_descriptor["interface"]:
 459                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 460                                 if icp_params.get("ip-address"):
 461                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 462                                                                  vdu_descriptor["id"], "interfaces",
 463                                                                  vdu_interface["name"], "ip_address"),
 464                                                   icp_params["ip-address"])
 465
 466                                 if icp_params.get("mac-address"):
 467                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 468                                                                  vdu_descriptor["id"], "interfaces",
 469                                                                  vdu_interface["name"], "mac_address"),
 470                                                   icp_params["mac-address"])
 471                                 iface_found = True
 472                                 break
 473                         if iface_found:
 474                             break
 475                     else:
 476                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 477                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 478                                            "connection-point".format(vnf_params["member-vnf-index"],
 479                                                                      icp_params["id-ref"]))
 480
 481         for vld_params in get_iterable(ns_params, "vld"):
 482             if "ip-profile" in vld_params:
 483                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 484                               ip_profile_2_RO(vld_params["ip-profile"]))
 485
 486             if vld_params.get("provider-network"):
 487
 488                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "provider-network"),
 489                               vld_params["provider-network"].copy())
 490
 491             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 492                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 493                               wim_account_2_RO(vld_params["wimAccountId"])),
 494             if vld_params.get("vim-network-name"):
 495                 RO_vld_sites = []
 496                 if isinstance(vld_params["vim-network-name"], dict):
 497                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 498                         RO_vld_sites.append({
 499                             "netmap-use": vim_net,
 500                             "datacenter": vim_account_2_RO(vim_account)
 501                         })
 502                 else:  # isinstance str
 503                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 504                 if RO_vld_sites:
 505                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 506
 507             if vld_params.get("vim-network-id"):
 508                 RO_vld_sites = []
 509                 if isinstance(vld_params["vim-network-id"], dict):
 510                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 511                         RO_vld_sites.append({
 512                             "netmap-use": vim_net,
 513                             "datacenter": vim_account_2_RO(vim_account)
 514                         })
 515                 else:  # isinstance str
 516                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 517                 if RO_vld_sites:
 518                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 519             if vld_params.get("ns-net"):
 520                 if isinstance(vld_params["ns-net"], dict):
 521                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 522                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 523                         populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 524             if "vnfd-connection-point-ref" in vld_params:
 525                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 526                     # look for interface
 527                     for constituent_vnfd in nsd["constituent-vnfd"]:
 528                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 529                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 530                             break
 531                     else:
 532                         raise LcmException(
 533                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 534                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 535                     match_cp = False
 536                     for vdu_descriptor in vnf_descriptor["vdu"]:
 537                         for interface_descriptor in vdu_descriptor["interface"]:
 538                             if interface_descriptor.get("external-connection-point-ref") == \
 539                                     cp_params["vnfd-connection-point-ref"]:
 540                                 match_cp = True
 541                                 break
 542                         if match_cp:
 543                             break
 544                     else:
 545                         raise LcmException(
 546                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 547                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 548                                 cp_params["member-vnf-index-ref"],
 549                                 cp_params["vnfd-connection-point-ref"],
 550                                 vnf_descriptor["id"]))
 551                     if cp_params.get("ip-address"):
 552                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 553                                                      vdu_descriptor["id"], "interfaces",
 554                                                      interface_descriptor["name"], "ip_address"),
 555                                       cp_params["ip-address"])
 556                     if cp_params.get("mac-address"):
 557                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 558                                                      vdu_descriptor["id"], "interfaces",
 559                                                      interface_descriptor["name"], "mac_address"),
 560                                       cp_params["mac-address"])
 561         return RO_ns_params
 562
 563     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 564         # make a copy to do not change
 565         vdu_create = copy(vdu_create)
 566         vdu_delete = copy(vdu_delete)
 567
 568         vdurs = db_vnfr.get("vdur")
 569         if vdurs is None:
 570             vdurs = []
 571         vdu_index = len(vdurs)
 572         while vdu_index:
 573             vdu_index -= 1
 574             vdur = vdurs[vdu_index]
 575             if vdur.get("pdu-type"):
 576                 continue
 577             vdu_id_ref = vdur["vdu-id-ref"]
 578             if vdu_create and vdu_create.get(vdu_id_ref):
 579                 for index in range(0, vdu_create[vdu_id_ref]):
 580                     vdur = deepcopy(vdur)
 581                     vdur["_id"] = str(uuid4())
 582                     vdur["count-index"] += 1
 583                     vdurs.insert(vdu_index+1+index, vdur)
 584                 del vdu_create[vdu_id_ref]
 585             if vdu_delete and vdu_delete.get(vdu_id_ref):
 586                 del vdurs[vdu_index]
 587                 vdu_delete[vdu_id_ref] -= 1
 588                 if not vdu_delete[vdu_id_ref]:
 589                     del vdu_delete[vdu_id_ref]
 590         # check all operations are done
 591         if vdu_create or vdu_delete:
 592             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 593                 vdu_create))
 594         if vdu_delete:
 595             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 596                 vdu_delete))
 597
 598         vnfr_update = {"vdur": vdurs}
 599         db_vnfr["vdur"] = vdurs
 600         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 601
 602     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 603         """
 604         Updates database nsr with the RO info for the created vld
 605         :param ns_update_nsr: dictionary to be filled with the updated info
 606         :param db_nsr: content of db_nsr. This is also modified
 607         :param nsr_desc_RO: nsr descriptor from RO
 608         :return: Nothing, LcmException is raised on errors
 609         """
 610
 611         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 612             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 613                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 614                     continue
 615                 vld["vim-id"] = net_RO.get("vim_net_id")
 616                 vld["name"] = net_RO.get("vim_name")
 617                 vld["status"] = net_RO.get("status")
 618                 vld["status-detailed"] = net_RO.get("error_msg")
 619                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 620                 break
 621             else:
 622                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 623
 624     def set_vnfr_at_error(self, db_vnfrs, error_text):
 625         try:
 626             for db_vnfr in db_vnfrs.values():
 627                 vnfr_update = {"status": "ERROR"}
 628                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 629                     if "status" not in vdur:
 630                         vdur["status"] = "ERROR"
 631                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 632                         if error_text:
 633                             vdur["status-detailed"] = str(error_text)
 634                             vnfr_update["vdur.{}.status-detailed".format(vdu_index)] = "ERROR"
 635                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 636         except DbException as e:
 637             self.logger.error("Cannot update vnf. {}".format(e))
 638
 639     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 640         """
 641         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 642         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 643         :param nsr_desc_RO: nsr descriptor from RO
 644         :return: Nothing, LcmException is raised on errors
 645         """
 646         for vnf_index, db_vnfr in db_vnfrs.items():
 647             for vnf_RO in nsr_desc_RO["vnfs"]:
 648                 if vnf_RO["member_vnf_index"] != vnf_index:
 649                     continue
 650                 vnfr_update = {}
 651                 if vnf_RO.get("ip_address"):
 652                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 653                 elif not db_vnfr.get("ip-address"):
 654                     if db_vnfr.get("vdur"):   # if not VDUs, there is not ip_address
 655                         raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 656
 657                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 658                     vdur_RO_count_index = 0
 659                     if vdur.get("pdu-type"):
 660                         continue
 661                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 662                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 663                             continue
 664                         if vdur["count-index"] != vdur_RO_count_index:
 665                             vdur_RO_count_index += 1
 666                             continue
 667                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 668                         if vdur_RO.get("ip_address"):
 669                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 670                         else:
 671                             vdur["ip-address"] = None
 672                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 673                         vdur["name"] = vdur_RO.get("vim_name")
 674                         vdur["status"] = vdur_RO.get("status")
 675                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 676                         for ifacer in get_iterable(vdur, "interfaces"):
 677                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 678                                 if ifacer["name"] == interface_RO.get("internal_name"):
 679                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 680                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 681                                     break
 682                             else:
 683                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 684                                                    "from VIM info"
 685                                                    .format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 686                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 687                         break
 688                     else:
 689                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 690                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 691
 692                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 693                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 694                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 695                             continue
 696                         vld["vim-id"] = net_RO.get("vim_net_id")
 697                         vld["name"] = net_RO.get("vim_name")
 698                         vld["status"] = net_RO.get("status")
 699                         vld["status-detailed"] = net_RO.get("error_msg")
 700                         vnfr_update["vld.{}".format(vld_index)] = vld
 701                         break
 702                     else:
 703                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 704                             vnf_index, vld["id"]))
 705
 706                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 707                 break
 708
 709             else:
 710                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 711
 712     def _get_ns_config_info(self, nsr_id):
 713         """
 714         Generates a mapping between vnf,vdu elements and the N2VC id
 715         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 716         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 717             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 718             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 719         """
 720         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 721         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 722         mapping = {}
 723         ns_config_info = {"osm-config-mapping": mapping}
 724         for vca in vca_deployed_list:
 725             if not vca["member-vnf-index"]:
 726                 continue
 727             if not vca["vdu_id"]:
 728                 mapping[vca["member-vnf-index"]] = vca["application"]
 729             else:
 730                 mapping["{}.{}.{}".format(vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"])] =\
 731                     vca["application"]
 732         return ns_config_info
 733
 734     @staticmethod
 735     def _get_initial_config_primitive_list(desc_primitive_list, vca_deployed):
 736         """
 737         Generates a list of initial-config-primitive based on the list provided by the descriptor. It includes internal
 738         primitives as verify-ssh-credentials, or config when needed
 739         :param desc_primitive_list: information of the descriptor
 740         :param vca_deployed: information of the deployed, needed for known if it is related to an NS, VNF, VDU and if
 741             this element contains a ssh public key
 742         :return: The modified list. Can ba an empty list, but always a list
 743         """
 744         if desc_primitive_list:
 745             primitive_list = desc_primitive_list.copy()
 746         else:
 747             primitive_list = []
 748         # look for primitive config, and get the position. None if not present
 749         config_position = None
 750         for index, primitive in enumerate(primitive_list):
 751             if primitive["name"] == "config":
 752                 config_position = index
 753                 break
 754
 755         # for NS, add always a config primitive if not present (bug 874)
 756         if not vca_deployed["member-vnf-index"] and config_position is None:
 757             primitive_list.insert(0, {"name": "config", "parameter": []})
 758             config_position = 0
 759         # for VNF/VDU add verify-ssh-credentials after config
 760         if vca_deployed["member-vnf-index"] and config_position is not None and vca_deployed.get("ssh-public-key"):
 761             primitive_list.insert(config_position + 1, {"name": "verify-ssh-credentials", "parameter": []})
 762         return primitive_list
 763
 764     async def _instantiate_ng_ro(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
 765                                  n2vc_key_list, stage, start_deploy, timeout_ns_deploy):
 766         nslcmop_id = db_nslcmop["_id"]
 767         target = {
 768             "name": db_nsr["name"],
 769             "ns": {"vld": []},
 770             "vnf": [],
 771             "image": deepcopy(db_nsr["image"]),
 772             "flavor": deepcopy(db_nsr["flavor"]),
 773             "action_id": nslcmop_id,
 774         }
 775         for image in target["image"]:
 776             image["vim_info"] = []
 777         for flavor in target["flavor"]:
 778             flavor["vim_info"] = []
 779
 780         ns_params = db_nslcmop.get("operationParams")
 781         ssh_keys = []
 782         if ns_params.get("ssh_keys"):
 783             ssh_keys += ns_params.get("ssh_keys")
 784         if n2vc_key_list:
 785             ssh_keys += n2vc_key_list
 786
 787         cp2target = {}
 788         for vld_index, vld in enumerate(nsd.get("vld")):
 789             target_vld = {"id": vld["id"],
 790                           "name": vld["name"],
 791                           "mgmt-network": vld.get("mgmt-network", False),
 792                           "type": vld.get("type"),
 793                           "vim_info": [{"vim-network-name": vld.get("vim-network-name"),
 794                                         "vim_account_id": ns_params["vimAccountId"]}],
 795                           }
 796             for cp in vld["vnfd-connection-point-ref"]:
 797                 cp2target["member_vnf:{}.{}".format(cp["member-vnf-index-ref"], cp["vnfd-connection-point-ref"])] = \
 798                     "nsrs:{}:vld.{}".format(nsr_id, vld_index)
 799             target["ns"]["vld"].append(target_vld)
 800         for vnfr in db_vnfrs.values():
 801             vnfd = db_vnfds_ref[vnfr["vnfd-ref"]]
 802             target_vnf = deepcopy(vnfr)
 803             for vld in target_vnf.get("vld", ()):
 804                 # check if connected to a ns.vld
 805                 vnf_cp = next((cp for cp in vnfd.get("connection-point", ()) if
 806                                cp.get("internal-vld-ref") == vld["id"]), None)
 807                 if vnf_cp:
 808                     ns_cp = "member_vnf:{}.{}".format(vnfr["member-vnf-index-ref"], vnf_cp["id"])
 809                     if cp2target.get(ns_cp):
 810                         vld["target"] = cp2target[ns_cp]
 811                 vld["vim_info"] = [{"vim-network-name": vld.get("vim-network-name"),
 812                                     "vim_account_id": vnfr["vim-account-id"]}]
 813
 814             for vdur in target_vnf.get("vdur", ()):
 815                 vdur["vim_info"] = [{"vim_account_id": vnfr["vim-account-id"]}]
 816                 vdud_index, vdud = next(k for k in enumerate(vnfd["vdu"]) if k[1]["id"] == vdur["vdu-id-ref"])
 817                 # vdur["additionalParams"] = vnfr.get("additionalParamsForVnf")  # TODO additional params for VDU
 818
 819                 if ssh_keys:
 820                     if deep_get(vdud, ("vdu-configuration", "config-access", "ssh-access", "required")):
 821                         vdur["ssh-keys"] = ssh_keys
 822                         vdur["ssh-access-required"] = True
 823                     elif deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required")) and \
 824                             any(iface.get("mgmt-vnf") for iface in vdur["interfaces"]):
 825                         vdur["ssh-keys"] = ssh_keys
 826                         vdur["ssh-access-required"] = True
 827
 828                 # cloud-init
 829                 if vdud.get("cloud-init-file"):
 830                     vdur["cloud-init"] = "{}:file:{}".format(vnfd["_id"], vdud.get("cloud-init-file"))
 831                 elif vdud.get("cloud-init"):
 832                     vdur["cloud-init"] = "{}:vdu:{}".format(vnfd["_id"], vdud_index)
 833
 834                 # flavor
 835                 ns_flavor = target["flavor"][int(vdur["ns-flavor-id"])]
 836                 if not next((vi for vi in ns_flavor["vim_info"] if
 837                              vi and vi.get("vim_account_id") == vnfr["vim-account-id"]), None):
 838                     ns_flavor["vim_info"].append({"vim_account_id": vnfr["vim-account-id"]})
 839                 # image
 840                 ns_image = target["image"][int(vdur["ns-image-id"])]
 841                 if not next((vi for vi in ns_image["vim_info"] if
 842                              vi and vi.get("vim_account_id") == vnfr["vim-account-id"]), None):
 843                     ns_image["vim_info"].append({"vim_account_id": vnfr["vim-account-id"]})
 844
 845                 vdur["vim_info"] = [{"vim_account_id": vnfr["vim-account-id"]}]
 846             target["vnf"].append(target_vnf)
 847
 848         desc = await self.RO.deploy(nsr_id, target)
 849         action_id = desc["action_id"]
 850         await self._wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_deploy, timeout_ns_deploy, stage)
 851
 852         # Updating NSR
 853         db_nsr_update = {
 854             "_admin.deployed.RO.operational-status": "running",
 855             "detailed-status": " ".join(stage)
 856         }
 857         # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
 858         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 859         self._write_op_status(nslcmop_id, stage)
 860         self.logger.debug(logging_text + "ns deployed at RO. RO_id={}".format(action_id))
 861         return
 862
 863     async def _wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_time, timeout, stage):
 864         detailed_status_old = None
 865         db_nsr_update = {}
 866         while time() <= start_time + timeout:
 867             desc_status = await self.RO.status(nsr_id, action_id)
 868             if desc_status["status"] == "FAILED":
 869                 raise NgRoException(desc_status["details"])
 870             elif desc_status["status"] == "BUILD":
 871                 stage[2] = "VIM: ({})".format(desc_status["details"])
 872             elif desc_status["status"] == "DONE":
 873                 stage[2] = "Deployed at VIM"
 874                 break
 875             else:
 876                 assert False, "ROclient.check_ns_status returns unknown {}".format(desc_status["status"])
 877             if stage[2] != detailed_status_old:
 878                 detailed_status_old = stage[2]
 879                 db_nsr_update["detailed-status"] = " ".join(stage)
 880                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 881                 self._write_op_status(nslcmop_id, stage)
 882             await asyncio.sleep(5, loop=self.loop)
 883         else:  # timeout_ns_deploy
 884             raise NgRoException("Timeout waiting ns to deploy")
 885
 886     async def _terminate_ng_ro(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
 887         db_nsr_update = {}
 888         failed_detail = []
 889         action_id = None
 890         start_deploy = time()
 891         try:
 892             target = {
 893                 "ns": {"vld": []},
 894                 "vnf": [],
 895                 "image": [],
 896                 "flavor": [],
 897             }
 898             desc = await self.RO.deploy(nsr_id, target)
 899             action_id = desc["action_id"]
 900             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = action_id
 901             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETING"
 902             self.logger.debug(logging_text + "ns terminate action at RO. action_id={}".format(action_id))
 903
 904             # wait until done
 905             delete_timeout = 20 * 60  # 20 minutes
 906             await self._wait_ng_ro(self, nsr_id, action_id, nslcmop_id, start_deploy, delete_timeout, stage)
 907
 908             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
 909             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 910             # delete all nsr
 911             await self.RO.delete(nsr_id)
 912         except Exception as e:
 913             if isinstance(e, NgRoException) and e.http_code == 404:  # not found
 914                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 915                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 916                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
 917                 self.logger.debug(logging_text + "RO_action_id={} already deleted".format(action_id))
 918             elif isinstance(e, NgRoException) and e.http_code == 409:  # conflict
 919                 failed_detail.append("delete conflict: {}".format(e))
 920                 self.logger.debug(logging_text + "RO_action_id={} delete conflict: {}".format(action_id, e))
 921             else:
 922                 failed_detail.append("delete error: {}".format(e))
 923                 self.logger.error(logging_text + "RO_action_id={} delete error: {}".format(action_id, e))
 924
 925         if failed_detail:
 926             stage[2] = "Error deleting from VIM"
 927         else:
 928             stage[2] = "Deleted from VIM"
 929         db_nsr_update["detailed-status"] = " ".join(stage)
 930         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 931         self._write_op_status(nslcmop_id, stage)
 932
 933         if failed_detail:
 934             raise LcmException("; ".join(failed_detail))
 935         return
 936
 937     async def instantiate_RO(self, logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs, db_vnfds_ref,
 938                              n2vc_key_list, stage):
 939         """
 940         Instantiate at RO
 941         :param logging_text: preffix text to use at logging
 942         :param nsr_id: nsr identity
 943         :param nsd: database content of ns descriptor
 944         :param db_nsr: database content of ns record
 945         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
 946         :param db_vnfrs:
 947         :param db_vnfds_ref: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
 948         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
 949         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
 950         :return: None or exception
 951         """
 952         try:
 953             db_nsr_update = {}
 954             RO_descriptor_number = 0   # number of descriptors created at RO
 955             vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 956             nslcmop_id = db_nslcmop["_id"]
 957             start_deploy = time()
 958             ns_params = db_nslcmop.get("operationParams")
 959             if ns_params and ns_params.get("timeout_ns_deploy"):
 960                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
 961             else:
 962                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
 963
 964             # Check for and optionally request placement optimization. Database will be updated if placement activated
 965             stage[2] = "Waiting for Placement."
 966             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
 967                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
 968                 for vnfr in db_vnfrs.values():
 969                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
 970                         break
 971                 else:
 972                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
 973
 974             if self.ng_ro:
 975                 return await self._instantiate_ng_ro(logging_text, nsr_id, nsd, db_nsr, db_nslcmop, db_vnfrs,
 976                                                      db_vnfds_ref, n2vc_key_list, stage, start_deploy,
 977                                                      timeout_ns_deploy)
 978             # deploy RO
 979             # get vnfds, instantiate at RO
 980             for c_vnf in nsd.get("constituent-vnfd", ()):
 981                 member_vnf_index = c_vnf["member-vnf-index"]
 982                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 983                 vnfd_ref = vnfd["id"]
 984
 985                 stage[2] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(vnfd_ref, member_vnf_index)
 986                 db_nsr_update["detailed-status"] = " ".join(stage)
 987                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 988                 self._write_op_status(nslcmop_id, stage)
 989
 990                 # self.logger.debug(logging_text + stage[2])
 991                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 992                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 993                 RO_descriptor_number += 1
 994
 995                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 996                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 997                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 998                         break
 999                 else:
1000                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1001                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1002
1003                 # look if present
1004                 RO_update = {"member-vnf-index": member_vnf_index}
1005                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1006                 if vnfd_list:
1007                     RO_update["id"] = vnfd_list[0]["uuid"]
1008                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1009                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1010                 else:
1011                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1012                                            get("additionalParamsForVnf"), nsr_id)
1013                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1014                     RO_update["id"] = desc["uuid"]
1015                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1016                         vnfd_ref, member_vnf_index, desc["uuid"]))
1017                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1018                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1019
1020             # create nsd at RO
1021             nsd_ref = nsd["id"]
1022
1023             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
1024             db_nsr_update["detailed-status"] = " ".join(stage)
1025             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1026             self._write_op_status(nslcmop_id, stage)
1027
1028             # self.logger.debug(logging_text + stage[2])
1029             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1030             RO_descriptor_number += 1
1031             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1032             if nsd_list:
1033                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1034                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1035                     nsd_ref, RO_nsd_uuid))
1036             else:
1037                 nsd_RO = deepcopy(nsd)
1038                 nsd_RO["id"] = RO_osm_nsd_id
1039                 nsd_RO.pop("_id", None)
1040                 nsd_RO.pop("_admin", None)
1041                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1042                     member_vnf_index = c_vnf["member-vnf-index"]
1043                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1044                 for c_vld in nsd_RO.get("vld", ()):
1045                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1046                         member_vnf_index = cp["member-vnf-index-ref"]
1047                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1048
1049                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1050                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1051                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1052                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1053             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1054
1055             # Crate ns at RO
1056             stage[2] = "Creating nsd={} at RO".format(nsd_ref)
1057             db_nsr_update["detailed-status"] = " ".join(stage)
1058             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1059             self._write_op_status(nslcmop_id, stage)
1060
1061             # if present use it unless in error status
1062             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1063             if RO_nsr_id:
1064                 try:
1065                     stage[2] = "Looking for existing ns at RO"
1066                     db_nsr_update["detailed-status"] = " ".join(stage)
1067                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1068                     self._write_op_status(nslcmop_id, stage)
1069                     # self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
1070                     desc = await self.RO.show("ns", RO_nsr_id)
1071
1072                 except ROclient.ROClientException as e:
1073                     if e.http_code != HTTPStatus.NOT_FOUND:
1074                         raise
1075                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1076                 if RO_nsr_id:
1077                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1078                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1079                     if ns_status == "ERROR":
1080                         stage[2] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1081                         self.logger.debug(logging_text + stage[2])
1082                         await self.RO.delete("ns", RO_nsr_id)
1083                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1084             if not RO_nsr_id:
1085                 stage[2] = "Checking dependencies"
1086                 db_nsr_update["detailed-status"] = " ".join(stage)
1087                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1088                 self._write_op_status(nslcmop_id, stage)
1089                 # self.logger.debug(logging_text + stage[2])
1090
1091                 # check if VIM is creating and wait  look if previous tasks in process
1092                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1093                 if task_dependency:
1094                     stage[2] = "Waiting for related tasks '{}' to be completed".format(task_name)
1095                     self.logger.debug(logging_text + stage[2])
1096                     await asyncio.wait(task_dependency, timeout=3600)
1097                 if ns_params.get("vnf"):
1098                     for vnf in ns_params["vnf"]:
1099                         if "vimAccountId" in vnf:
1100                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1101                                                                                         vnf["vimAccountId"])
1102                         if task_dependency:
1103                             stage[2] = "Waiting for related tasks '{}' to be completed.".format(task_name)
1104                             self.logger.debug(logging_text + stage[2])
1105                             await asyncio.wait(task_dependency, timeout=3600)
1106
1107                 stage[2] = "Checking instantiation parameters."
1108                 RO_ns_params = self._ns_params_2_RO(ns_params, nsd, db_vnfds_ref, db_vnfrs, n2vc_key_list)
1109                 stage[2] = "Deploying ns at VIM."
1110                 db_nsr_update["detailed-status"] = " ".join(stage)
1111                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1112                 self._write_op_status(nslcmop_id, stage)
1113
1114                 desc = await self.RO.create("ns", descriptor=RO_ns_params, name=db_nsr["name"], scenario=RO_nsd_uuid)
1115                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1116                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1117                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1118                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1119
1120             # wait until NS is ready
1121             stage[2] = "Waiting VIM to deploy ns."
1122             db_nsr_update["detailed-status"] = " ".join(stage)
1123             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1124             self._write_op_status(nslcmop_id, stage)
1125             detailed_status_old = None
1126             self.logger.debug(logging_text + stage[2] + " RO_ns_id={}".format(RO_nsr_id))
1127
1128             old_desc = None
1129             while time() <= start_deploy + timeout_ns_deploy:
1130                 desc = await self.RO.show("ns", RO_nsr_id)
1131
1132                 # deploymentStatus
1133                 if desc != old_desc:
1134                     # desc has changed => update db
1135                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
1136                     old_desc = desc
1137
1138                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1139                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1140                     if ns_status == "ERROR":
1141                         raise ROclient.ROClientException(ns_status_info)
1142                     elif ns_status == "BUILD":
1143                         stage[2] = "VIM: ({})".format(ns_status_info)
1144                     elif ns_status == "ACTIVE":
1145                         stage[2] = "Waiting for management IP address reported by the VIM. Updating VNFRs."
1146                         try:
1147                             self.ns_update_vnfr(db_vnfrs, desc)
1148                             break
1149                         except LcmExceptionNoMgmtIP:
1150                             pass
1151                     else:
1152                         assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1153                     if stage[2] != detailed_status_old:
1154                         detailed_status_old = stage[2]
1155                         db_nsr_update["detailed-status"] = " ".join(stage)
1156                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1157                         self._write_op_status(nslcmop_id, stage)
1158                     await asyncio.sleep(5, loop=self.loop)
1159             else:  # timeout_ns_deploy
1160                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1161
1162             # Updating NSR
1163             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1164
1165             db_nsr_update["_admin.deployed.RO.operational-status"] = "running"
1166             # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
1167             stage[2] = "Deployed at VIM"
1168             db_nsr_update["detailed-status"] = " ".join(stage)
1169             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1170             self._write_op_status(nslcmop_id, stage)
1171             # await self._on_update_n2vc_db("nsrs", {"_id": nsr_id}, "_admin.deployed", db_nsr_update)
1172             # self.logger.debug(logging_text + "Deployed at VIM")
1173         except (ROclient.ROClientException, LcmException, DbException, NgRoException) as e:
1174             stage[2] = "ERROR deploying at VIM"
1175             self.set_vnfr_at_error(db_vnfrs, str(e))
1176             raise
1177
1178     async def wait_vm_up_insert_key_ro(self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None):
1179         """
1180         Wait for ip addres at RO, and optionally, insert public key in virtual machine
1181         :param logging_text: prefix use for logging
1182         :param nsr_id:
1183         :param vnfr_id:
1184         :param vdu_id:
1185         :param vdu_index:
1186         :param pub_key: public ssh key to inject, None to skip
1187         :param user: user to apply the public ssh key
1188         :return: IP address
1189         """
1190
1191         # self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1192         ro_nsr_id = None
1193         ip_address = None
1194         nb_tries = 0
1195         target_vdu_id = None
1196         ro_retries = 0
1197
1198         while True:
1199
1200             ro_retries += 1
1201             if ro_retries >= 360:  # 1 hour
1202                 raise LcmException("Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id))
1203
1204             await asyncio.sleep(10, loop=self.loop)
1205
1206             # get ip address
1207             if not target_vdu_id:
1208                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1209
1210                 if not vdu_id:  # for the VNF case
1211                     if db_vnfr.get("status") == "ERROR":
1212                         raise LcmException("Cannot inject ssh-key because target VNF is in error state")
1213                     ip_address = db_vnfr.get("ip-address")
1214                     if not ip_address:
1215                         continue
1216                     vdur = next((x for x in get_iterable(db_vnfr, "vdur") if x.get("ip-address") == ip_address), None)
1217                 else:  # VDU case
1218                     vdur = next((x for x in get_iterable(db_vnfr, "vdur")
1219                                  if x.get("vdu-id-ref") == vdu_id and x.get("count-index") == vdu_index), None)
1220
1221                 if not vdur and len(db_vnfr.get("vdur", ())) == 1:  # If only one, this should be the target vdu
1222                     vdur = db_vnfr["vdur"][0]
1223                 if not vdur:
1224                     raise LcmException("Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(vnfr_id, vdu_id,
1225                                                                                               vdu_index))
1226
1227                 if vdur.get("pdu-type") or vdur.get("status") == "ACTIVE":
1228                     ip_address = vdur.get("ip-address")
1229                     if not ip_address:
1230                         continue
1231                     target_vdu_id = vdur["vdu-id-ref"]
1232                 elif vdur.get("status") == "ERROR":
1233                     raise LcmException("Cannot inject ssh-key because target VM is in error state")
1234
1235             if not target_vdu_id:
1236                 continue
1237
1238             # inject public key into machine
1239             if pub_key and user:
1240                 # wait until NS is deployed at RO
1241                 if not ro_nsr_id:
1242                     db_nsrs = self.db.get_one("nsrs", {"_id": nsr_id})
1243                     ro_nsr_id = deep_get(db_nsrs, ("_admin", "deployed", "RO", "nsr_id"))
1244                 if not ro_nsr_id:
1245                     continue
1246
1247                 # self.logger.debug(logging_text + "Inserting RO key")
1248                 if vdur.get("pdu-type"):
1249                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1250                     return ip_address
1251                 try:
1252                     ro_vm_id = "{}-{}".format(db_vnfr["member-vnf-index-ref"], target_vdu_id)  # TODO add vdu_index
1253                     if self.ng_ro:
1254                         target = {"action": "inject_ssh_key", "key": pub_key, "user": user,
1255                                   "vnf": [{"_id": vnfr_id, "vdur": [{"id": vdu_id}]}],
1256                                   }
1257                         await self.RO.deploy(nsr_id, target)
1258                     else:
1259                         result_dict = await self.RO.create_action(
1260                             item="ns",
1261                             item_id_name=ro_nsr_id,
1262                             descriptor={"add_public_key": pub_key, "vms": [ro_vm_id], "user": user}
1263                         )
1264                         # result_dict contains the format {VM-id: {vim_result: 200, description: text}}
1265                         if not result_dict or not isinstance(result_dict, dict):
1266                             raise LcmException("Unknown response from RO when injecting key")
1267                         for result in result_dict.values():
1268                             if result.get("vim_result") == 200:
1269                                 break
1270                             else:
1271                                 raise ROclient.ROClientException("error injecting key: {}".format(
1272                                     result.get("description")))
1273                         break
1274                 except NgRoException as e:
1275                     raise LcmException("Reaching max tries injecting key. Error: {}".format(e))
1276                 except ROclient.ROClientException as e:
1277                     if not nb_tries:
1278                         self.logger.debug(logging_text + "error injecting key: {}. Retrying until {} seconds".
1279                                           format(e, 20*10))
1280                     nb_tries += 1
1281                     if nb_tries >= 20:
1282                         raise LcmException("Reaching max tries injecting key. Error: {}".format(e))
1283             else:
1284                 break
1285
1286         return ip_address
1287
1288     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1289         """
1290         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1291         """
1292         my_vca = vca_deployed_list[vca_index]
1293         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1294             # vdu or kdu: no dependencies
1295             return
1296         timeout = 300
1297         while timeout >= 0:
1298             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1299             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1300             configuration_status_list = db_nsr["configurationStatus"]
1301             for index, vca_deployed in enumerate(configuration_status_list):
1302                 if index == vca_index:
1303                     # myself
1304                     continue
1305                 if not my_vca.get("member-vnf-index") or \
1306                         (vca_deployed.get("member-vnf-index") == my_vca.get("member-vnf-index")):
1307                     internal_status = configuration_status_list[index].get("status")
1308                     if internal_status == 'READY':
1309                         continue
1310                     elif internal_status == 'BROKEN':
1311                         raise LcmException("Configuration aborted because dependent charm/s has failed")
1312                     else:
1313                         break
1314             else:
1315                 # no dependencies, return
1316                 return
1317             await asyncio.sleep(10)
1318             timeout -= 1
1319
1320         raise LcmException("Configuration aborted because dependent charm/s timeout")
1321
1322     async def instantiate_N2VC(self, logging_text, vca_index, nsi_id, db_nsr, db_vnfr, vdu_id, kdu_name, vdu_index,
1323                                config_descriptor, deploy_params, base_folder, nslcmop_id, stage):
1324         nsr_id = db_nsr["_id"]
1325         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1326         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1327         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1328         db_dict = {
1329             'collection': 'nsrs',
1330             'filter': {'_id': nsr_id},
1331             'path': db_update_entry
1332         }
1333         step = ""
1334         try:
1335
1336             element_type = 'NS'
1337             element_under_configuration = nsr_id
1338
1339             vnfr_id = None
1340             if db_vnfr:
1341                 vnfr_id = db_vnfr["_id"]
1342
1343             namespace = "{nsi}.{ns}".format(
1344                 nsi=nsi_id if nsi_id else "",
1345                 ns=nsr_id)
1346
1347             if vnfr_id:
1348                 element_type = 'VNF'
1349                 element_under_configuration = vnfr_id
1350                 namespace += ".{}".format(vnfr_id)
1351                 if vdu_id:
1352                     namespace += ".{}-{}".format(vdu_id, vdu_index or 0)
1353                     element_type = 'VDU'
1354                     element_under_configuration = "{}-{}".format(vdu_id, vdu_index or 0)
1355                 elif kdu_name:
1356                     namespace += ".{}".format(kdu_name)
1357                     element_type = 'KDU'
1358                     element_under_configuration = kdu_name
1359
1360             # Get artifact path
1361             artifact_path = "{}/{}/charms/{}".format(
1362                 base_folder["folder"],
1363                 base_folder["pkg-dir"],
1364                 config_descriptor["juju"]["charm"]
1365             )
1366
1367             is_proxy_charm = deep_get(config_descriptor, ('juju', 'charm')) is not None
1368             if deep_get(config_descriptor, ('juju', 'proxy')) is False:
1369                 is_proxy_charm = False
1370
1371             # n2vc_redesign STEP 3.1
1372
1373             # find old ee_id if exists
1374             ee_id = vca_deployed.get("ee_id")
1375
1376             # create or register execution environment in VCA
1377             if is_proxy_charm:
1378
1379                 self._write_configuration_status(
1380                     nsr_id=nsr_id,
1381                     vca_index=vca_index,
1382                     status='CREATING',
1383                     element_under_configuration=element_under_configuration,
1384                     element_type=element_type
1385                 )
1386
1387                 step = "create execution environment"
1388                 self.logger.debug(logging_text + step)
1389                 ee_id, credentials = await self.n2vc.create_execution_environment(namespace=namespace,
1390                                                                                   reuse_ee_id=ee_id,
1391                                                                                   db_dict=db_dict)
1392
1393             else:
1394                 step = "Waiting to VM being up and getting IP address"
1395                 self.logger.debug(logging_text + step)
1396                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
1397                                                                  user=None, pub_key=None)
1398                 credentials = {"hostname": rw_mgmt_ip}
1399                 # get username
1400                 username = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1401                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1402                 #  merged. Meanwhile let's get username from initial-config-primitive
1403                 if not username and config_descriptor.get("initial-config-primitive"):
1404                     for config_primitive in config_descriptor["initial-config-primitive"]:
1405                         for param in config_primitive.get("parameter", ()):
1406                             if param["name"] == "ssh-username":
1407                                 username = param["value"]
1408                                 break
1409                 if not username:
1410                     raise LcmException("Cannot determine the username neither with 'initial-config-promitive' nor with "
1411                                        "'config-access.ssh-access.default-user'")
1412                 credentials["username"] = username
1413                 # n2vc_redesign STEP 3.2
1414
1415                 self._write_configuration_status(
1416                     nsr_id=nsr_id,
1417                     vca_index=vca_index,
1418                     status='REGISTERING',
1419                     element_under_configuration=element_under_configuration,
1420                     element_type=element_type
1421                 )
1422
1423                 step = "register execution environment {}".format(credentials)
1424                 self.logger.debug(logging_text + step)
1425                 ee_id = await self.n2vc.register_execution_environment(credentials=credentials, namespace=namespace,
1426                                                                        db_dict=db_dict)
1427
1428             # for compatibility with MON/POL modules, the need model and application name at database
1429             # TODO ask to N2VC instead of assuming the format "model_name.application_name"
1430             ee_id_parts = ee_id.split('.')
1431             model_name = ee_id_parts[0]
1432             application_name = ee_id_parts[1]
1433             db_nsr_update = {db_update_entry + "model": model_name,
1434                              db_update_entry + "application": application_name,
1435                              db_update_entry + "ee_id": ee_id}
1436
1437             # n2vc_redesign STEP 3.3
1438
1439             step = "Install configuration Software"
1440
1441             self._write_configuration_status(
1442                 nsr_id=nsr_id,
1443                 vca_index=vca_index,
1444                 status='INSTALLING SW',
1445                 element_under_configuration=element_under_configuration,
1446                 element_type=element_type,
1447                 other_update=db_nsr_update
1448             )
1449
1450             # TODO check if already done
1451             self.logger.debug(logging_text + step)
1452             config = None
1453             if not is_proxy_charm:
1454                 initial_config_primitive_list = config_descriptor.get('initial-config-primitive')
1455                 if initial_config_primitive_list:
1456                     for primitive in initial_config_primitive_list:
1457                         if primitive["name"] == "config":
1458                             config = self._map_primitive_params(
1459                                 primitive,
1460                                 {},
1461                                 deploy_params
1462                             )
1463                             break
1464             num_units = 1
1465             if is_proxy_charm:
1466                 if element_type == "NS":
1467                     num_units = db_nsr.get("config-units") or 1
1468                 elif element_type == "VNF":
1469                     num_units = db_vnfr.get("config-units") or 1
1470                 elif element_type == "VDU":
1471                     for v in db_vnfr["vdur"]:
1472                         if vdu_id == v["vdu-id-ref"]:
1473                             num_units = v.get("config-units") or 1
1474                             break
1475
1476             await self.n2vc.install_configuration_sw(
1477                 ee_id=ee_id,
1478                 artifact_path=artifact_path,
1479                 db_dict=db_dict,
1480                 config=config,
1481                 num_units=num_units
1482             )
1483
1484             # write in db flag of configuration_sw already installed
1485             self.update_db_2("nsrs", nsr_id, {db_update_entry + "config_sw_installed": True})
1486
1487             # add relations for this VCA (wait for other peers related with this VCA)
1488             await self._add_vca_relations(logging_text=logging_text, nsr_id=nsr_id, vca_index=vca_index)
1489
1490             # if SSH access is required, then get execution environment SSH public
1491             if is_proxy_charm:  # if native charm we have waited already to VM be UP
1492                 pub_key = None
1493                 user = None
1494                 if deep_get(config_descriptor, ("config-access", "ssh-access", "required")):
1495                     # Needed to inject a ssh key
1496                     user = deep_get(config_descriptor, ("config-access", "ssh-access", "default-user"))
1497                     step = "Install configuration Software, getting public ssh key"
1498                     pub_key = await self.n2vc.get_ee_ssh_public__key(ee_id=ee_id, db_dict=db_dict)
1499
1500                     step = "Insert public key into VM user={} ssh_key={}".format(user, pub_key)
1501                 else:
1502                     step = "Waiting to VM being up and getting IP address"
1503                 self.logger.debug(logging_text + step)
1504
1505                 # n2vc_redesign STEP 5.1
1506                 # wait for RO (ip-address) Insert pub_key into VM
1507                 if vnfr_id:
1508                     rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(logging_text, nsr_id, vnfr_id, vdu_id, vdu_index,
1509                                                                      user=user, pub_key=pub_key)
1510                 else:
1511                     rw_mgmt_ip = None   # This is for a NS configuration
1512
1513                 self.logger.debug(logging_text + ' VM_ip_address={}'.format(rw_mgmt_ip))
1514
1515             # store rw_mgmt_ip in deploy params for later replacement
1516             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
1517
1518             # n2vc_redesign STEP 6  Execute initial config primitive
1519             step = 'execute initial config primitive'
1520             initial_config_primitive_list = config_descriptor.get('initial-config-primitive')
1521
1522             # sort initial config primitives by 'seq'
1523             if initial_config_primitive_list:
1524                 try:
1525                     initial_config_primitive_list.sort(key=lambda val: int(val['seq']))
1526                 except Exception as e:
1527                     self.logger.error(logging_text + step + ": " + str(e))
1528             else:
1529                 self.logger.debug(logging_text + step + ": No initial-config-primitive")
1530
1531             # add config if not present for NS charm
1532             initial_config_primitive_list = self._get_initial_config_primitive_list(initial_config_primitive_list,
1533                                                                                     vca_deployed)
1534
1535             # wait for dependent primitives execution (NS -> VNF -> VDU)
1536             if initial_config_primitive_list:
1537                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
1538
1539             # stage, in function of element type: vdu, kdu, vnf or ns
1540             my_vca = vca_deployed_list[vca_index]
1541             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1542                 # VDU or KDU
1543                 stage[0] = 'Stage 3/5: running Day-1 primitives for VDU.'
1544             elif my_vca.get("member-vnf-index"):
1545                 # VNF
1546                 stage[0] = 'Stage 4/5: running Day-1 primitives for VNF.'
1547             else:
1548                 # NS
1549                 stage[0] = 'Stage 5/5: running Day-1 primitives for NS.'
1550
1551             self._write_configuration_status(
1552                 nsr_id=nsr_id,
1553                 vca_index=vca_index,
1554                 status='EXECUTING PRIMITIVE'
1555             )
1556
1557             self._write_op_status(
1558                 op_id=nslcmop_id,
1559                 stage=stage
1560             )
1561
1562             check_if_terminated_needed = True
1563             for initial_config_primitive in initial_config_primitive_list:
1564                 # adding information on the vca_deployed if it is a NS execution environment
1565                 if not vca_deployed["member-vnf-index"]:
1566                     deploy_params["ns_config_info"] = json.dumps(self._get_ns_config_info(nsr_id))
1567                 # TODO check if already done
1568                 primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, deploy_params)
1569
1570                 step = "execute primitive '{}' params '{}'".format(initial_config_primitive["name"], primitive_params_)
1571                 self.logger.debug(logging_text + step)
1572                 await self.n2vc.exec_primitive(
1573                     ee_id=ee_id,
1574                     primitive_name=initial_config_primitive["name"],
1575                     params_dict=primitive_params_,
1576                     db_dict=db_dict
1577                 )
1578                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
1579                 if check_if_terminated_needed:
1580                     if config_descriptor.get('terminate-config-primitive'):
1581                         self.update_db_2("nsrs", nsr_id, {db_update_entry + "needed_terminate": True})
1582                     check_if_terminated_needed = False
1583
1584                 # TODO register in database that primitive is done
1585
1586             step = "instantiated at VCA"
1587             self.logger.debug(logging_text + step)
1588
1589             self._write_configuration_status(
1590                 nsr_id=nsr_id,
1591                 vca_index=vca_index,
1592                 status='READY'
1593             )
1594
1595         except Exception as e:  # TODO not use Exception but N2VC exception
1596             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
1597             if not isinstance(e, (DbException, N2VCException, LcmException, asyncio.CancelledError)):
1598                 self.logger.error("Exception while {} : {}".format(step, e), exc_info=True)
1599             self._write_configuration_status(
1600                 nsr_id=nsr_id,
1601                 vca_index=vca_index,
1602                 status='BROKEN'
1603             )
1604             raise LcmException("{} {}".format(step, e)) from e
1605
1606     def _write_ns_status(self, nsr_id: str, ns_state: str, current_operation: str, current_operation_id: str,
1607                          error_description: str = None, error_detail: str = None, other_update: dict = None):
1608         """
1609         Update db_nsr fields.
1610         :param nsr_id:
1611         :param ns_state:
1612         :param current_operation:
1613         :param current_operation_id:
1614         :param error_description:
1615         :param error_detail:
1616         :param other_update: Other required changes at database if provided, will be cleared
1617         :return:
1618         """
1619         try:
1620             db_dict = other_update or {}
1621             db_dict["_admin.nslcmop"] = current_operation_id    # for backward compatibility
1622             db_dict["_admin.current-operation"] = current_operation_id
1623             db_dict["_admin.operation-type"] = current_operation if current_operation != "IDLE" else None
1624             db_dict["currentOperation"] = current_operation
1625             db_dict["currentOperationID"] = current_operation_id
1626             db_dict["errorDescription"] = error_description
1627             db_dict["errorDetail"] = error_detail
1628
1629             if ns_state:
1630                 db_dict["nsState"] = ns_state
1631             self.update_db_2("nsrs", nsr_id, db_dict)
1632         except DbException as e:
1633             self.logger.warn('Error writing NS status, ns={}: {}'.format(nsr_id, e))
1634
1635     def _write_op_status(self, op_id: str, stage: list = None, error_message: str = None, queuePosition: int = 0,
1636                          operation_state: str = None, other_update: dict = None):
1637         try:
1638             db_dict = other_update or {}
1639             db_dict['queuePosition'] = queuePosition
1640             if isinstance(stage, list):
1641                 db_dict['stage'] = stage[0]
1642                 db_dict['detailed-status'] = " ".join(stage)
1643             elif stage is not None:
1644                 db_dict['stage'] = str(stage)
1645
1646             if error_message is not None:
1647                 db_dict['errorMessage'] = error_message
1648             if operation_state is not None:
1649                 db_dict['operationState'] = operation_state
1650                 db_dict["statusEnteredTime"] = time()
1651             self.update_db_2("nslcmops", op_id, db_dict)
1652         except DbException as e:
1653             self.logger.warn('Error writing OPERATION status for op_id: {} -> {}'.format(op_id, e))
1654
1655     def _write_all_config_status(self, db_nsr: dict, status: str):
1656         try:
1657             nsr_id = db_nsr["_id"]
1658             # configurationStatus
1659             config_status = db_nsr.get('configurationStatus')
1660             if config_status:
1661                 db_nsr_update = {"configurationStatus.{}.status".format(index): status for index, v in
1662                                  enumerate(config_status) if v}
1663                 # update status
1664                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1665
1666         except DbException as e:
1667             self.logger.warn('Error writing all configuration status, ns={}: {}'.format(nsr_id, e))
1668
1669     def _write_configuration_status(self, nsr_id: str, vca_index: int, status: str = None,
1670                                     element_under_configuration: str = None, element_type: str = None,
1671                                     other_update: dict = None):
1672
1673         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
1674         #                   .format(vca_index, status))
1675
1676         try:
1677             db_path = 'configurationStatus.{}.'.format(vca_index)
1678             db_dict = other_update or {}
1679             if status:
1680                 db_dict[db_path + 'status'] = status
1681             if element_under_configuration:
1682                 db_dict[db_path + 'elementUnderConfiguration'] = element_under_configuration
1683             if element_type:
1684                 db_dict[db_path + 'elementType'] = element_type
1685             self.update_db_2("nsrs", nsr_id, db_dict)
1686         except DbException as e:
1687             self.logger.warn('Error writing configuration status={}, ns={}, vca_index={}: {}'
1688                              .format(status, nsr_id, vca_index, e))
1689
1690     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
1691         """
1692         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
1693         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
1694         Database is used because the result can be obtained from a different LCM worker in case of HA.
1695         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
1696         :param db_nslcmop: database content of nslcmop
1697         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
1698         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
1699             computed 'vim-account-id'
1700         """
1701         modified = False
1702         nslcmop_id = db_nslcmop['_id']
1703         placement_engine = deep_get(db_nslcmop, ('operationParams', 'placement-engine'))
1704         if placement_engine == "PLA":
1705             self.logger.debug(logging_text + "Invoke and wait for placement optimization")
1706             await self.msg.aiowrite("pla", "get_placement", {'nslcmopId': nslcmop_id}, loop=self.loop)
1707             db_poll_interval = 5
1708             wait = db_poll_interval * 10
1709             pla_result = None
1710             while not pla_result and wait >= 0:
1711                 await asyncio.sleep(db_poll_interval)
1712                 wait -= db_poll_interval
1713                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1714                 pla_result = deep_get(db_nslcmop, ('_admin', 'pla'))
1715
1716             if not pla_result:
1717                 raise LcmException("Placement timeout for nslcmopId={}".format(nslcmop_id))
1718
1719             for pla_vnf in pla_result['vnf']:
1720                 vnfr = db_vnfrs.get(pla_vnf['member-vnf-index'])
1721                 if not pla_vnf.get('vimAccountId') or not vnfr:
1722                     continue
1723                 modified = True
1724                 self.db.set_one("vnfrs", {"_id": vnfr["_id"]}, {"vim-account-id": pla_vnf['vimAccountId']})
1725                 # Modifies db_vnfrs
1726                 vnfr["vim-account-id"] = pla_vnf['vimAccountId']
1727         return modified
1728
1729     def update_nsrs_with_pla_result(self, params):
1730         try:
1731             nslcmop_id = deep_get(params, ('placement', 'nslcmopId'))
1732             self.update_db_2("nslcmops", nslcmop_id, {"_admin.pla": params.get('placement')})
1733         except Exception as e:
1734             self.logger.warn('Update failed for nslcmop_id={}:{}'.format(nslcmop_id, e))
1735
1736     async def instantiate(self, nsr_id, nslcmop_id):
1737         """
1738
1739         :param nsr_id: ns instance to deploy
1740         :param nslcmop_id: operation to run
1741         :return:
1742         """
1743
1744         # Try to lock HA task here
1745         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1746         if not task_is_locked_by_me:
1747             self.logger.debug('instantiate() task is not locked by me, ns={}'.format(nsr_id))
1748             return
1749
1750         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
1751         self.logger.debug(logging_text + "Enter")
1752
1753         # Sync from FSMongo
1754         self.fs.sync()
1755
1756         # get all needed from database
1757
1758         # database nsrs record
1759         db_nsr = None
1760
1761         # database nslcmops record
1762         db_nslcmop = None
1763
1764         # update operation on nsrs
1765         db_nsr_update = {}
1766         # update operation on nslcmops
1767         db_nslcmop_update = {}
1768
1769         nslcmop_operation_state = None
1770         db_vnfrs = {}     # vnf's info indexed by member-index
1771         # n2vc_info = {}
1772         tasks_dict_info = {}  # from task to info text
1773         exc = None
1774         error_list = []
1775         stage = ['Stage 1/5: preparation of the environment.', "Waiting for previous operations to terminate.", ""]
1776         # ^ stage, step, VIM progress
1777         try:
1778             # wait for any previous tasks in process
1779             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
1780
1781             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
1782             stage[1] = "Reading from database,"
1783             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
1784             db_nsr_update["detailed-status"] = "creating"
1785             db_nsr_update["operational-status"] = "init"
1786             self._write_ns_status(
1787                 nsr_id=nsr_id,
1788                 ns_state="BUILDING",
1789                 current_operation="INSTANTIATING",
1790                 current_operation_id=nslcmop_id,
1791                 other_update=db_nsr_update
1792             )
1793             self._write_op_status(
1794                 op_id=nslcmop_id,
1795                 stage=stage,
1796                 queuePosition=0
1797             )
1798
1799             # read from db: operation
1800             stage[1] = "Getting nslcmop={} from db".format(nslcmop_id)
1801             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1802             ns_params = db_nslcmop.get("operationParams")
1803             if ns_params and ns_params.get("timeout_ns_deploy"):
1804                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1805             else:
1806                 timeout_ns_deploy = self.timeout.get("ns_deploy", self.timeout_ns_deploy)
1807
1808             # read from db: ns
1809             stage[1] = "Getting nsr={} from db".format(nsr_id)
1810             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1811             stage[1] = "Getting nsd={} from db".format(db_nsr["nsd-id"])
1812             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1813             db_nsr["nsd"] = nsd
1814             # nsr_name = db_nsr["name"]   # TODO short-name??
1815
1816             # read from db: vnf's of this ns
1817             stage[1] = "Getting vnfrs from db"
1818             self.logger.debug(logging_text + stage[1])
1819             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1820
1821             # read from db: vnfd's for every vnf
1822             db_vnfds_ref = {}     # every vnfd data indexed by vnf name
1823             db_vnfds = {}         # every vnfd data indexed by vnf id
1824             db_vnfds_index = {}   # every vnfd data indexed by vnf member-index
1825
1826             # for each vnf in ns, read vnfd
1827             for vnfr in db_vnfrs_list:
1828                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr   # vnf's dict indexed by member-index: '1', '2', etc
1829                 vnfd_id = vnfr["vnfd-id"]                       # vnfd uuid for this vnf
1830                 vnfd_ref = vnfr["vnfd-ref"]                     # vnfd name for this vnf
1831                 # if we haven't this vnfd, read it from db
1832                 if vnfd_id not in db_vnfds:
1833                     # read from db
1834                     stage[1] = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
1835                     self.logger.debug(logging_text + stage[1])
1836                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1837
1838                     # store vnfd
1839                     db_vnfds_ref[vnfd_ref] = vnfd     # vnfd's indexed by name
1840                     db_vnfds[vnfd_id] = vnfd          # vnfd's indexed by id
1841                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]  # vnfd's indexed by member-index
1842
1843             # Get or generates the _admin.deployed.VCA list
1844             vca_deployed_list = None
1845             if db_nsr["_admin"].get("deployed"):
1846                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
1847             if vca_deployed_list is None:
1848                 vca_deployed_list = []
1849                 configuration_status_list = []
1850                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1851                 db_nsr_update["configurationStatus"] = configuration_status_list
1852                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
1853                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1854             elif isinstance(vca_deployed_list, dict):
1855                 # maintain backward compatibility. Change a dict to list at database
1856                 vca_deployed_list = list(vca_deployed_list.values())
1857                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
1858                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
1859
1860             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
1861                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
1862                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
1863
1864             # set state to INSTANTIATED. When instantiated NBI will not delete directly
1865             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1866             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1867
1868             # n2vc_redesign STEP 2 Deploy Network Scenario
1869             stage[0] = 'Stage 2/5: deployment of KDUs, VMs and execution environments.'
1870             self._write_op_status(
1871                 op_id=nslcmop_id,
1872                 stage=stage
1873             )
1874
1875             stage[1] = "Deploying KDUs,"
1876             # self.logger.debug(logging_text + "Before deploy_kdus")
1877             # Call to deploy_kdus in case exists the "vdu:kdu" param
1878             await self.deploy_kdus(
1879                 logging_text=logging_text,
1880                 nsr_id=nsr_id,
1881                 nslcmop_id=nslcmop_id,
1882                 db_vnfrs=db_vnfrs,
1883                 db_vnfds=db_vnfds,
1884                 task_instantiation_info=tasks_dict_info,
1885             )
1886
1887             stage[1] = "Getting VCA public key."
1888             # n2vc_redesign STEP 1 Get VCA public ssh-key
1889             # feature 1429. Add n2vc public key to needed VMs
1890             n2vc_key = self.n2vc.get_public_key()
1891             n2vc_key_list = [n2vc_key]
1892             if self.vca_config.get("public_key"):
1893                 n2vc_key_list.append(self.vca_config["public_key"])
1894
1895             stage[1] = "Deploying NS at VIM."
1896             task_ro = asyncio.ensure_future(
1897                 self.instantiate_RO(
1898                     logging_text=logging_text,
1899                     nsr_id=nsr_id,
1900                     nsd=nsd,
1901                     db_nsr=db_nsr,
1902                     db_nslcmop=db_nslcmop,
1903                     db_vnfrs=db_vnfrs,
1904                     db_vnfds_ref=db_vnfds_ref,
1905                     n2vc_key_list=n2vc_key_list,
1906                     stage=stage
1907                 )
1908             )
1909             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
1910             tasks_dict_info[task_ro] = "Deploying at VIM"
1911
1912             # n2vc_redesign STEP 3 to 6 Deploy N2VC
1913             stage[1] = "Deploying Execution Environments."
1914             self.logger.debug(logging_text + stage[1])
1915
1916             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
1917             # get_iterable() returns a value from a dict or empty tuple if key does not exist
1918             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1919                 vnfd_id = c_vnf["vnfd-id-ref"]
1920                 vnfd = db_vnfds_ref[vnfd_id]
1921                 member_vnf_index = str(c_vnf["member-vnf-index"])
1922                 db_vnfr = db_vnfrs[member_vnf_index]
1923                 base_folder = vnfd["_admin"]["storage"]
1924                 vdu_id = None
1925                 vdu_index = 0
1926                 vdu_name = None
1927                 kdu_name = None
1928
1929                 # Get additional parameters
1930                 deploy_params = {}
1931                 if db_vnfr.get("additionalParamsForVnf"):
1932                     deploy_params = self._format_additional_params(db_vnfr["additionalParamsForVnf"].copy())
1933
1934                 descriptor_config = vnfd.get("vnf-configuration")
1935                 if descriptor_config and descriptor_config.get("juju"):
1936                     self._deploy_n2vc(
1937                         logging_text=logging_text + "member_vnf_index={} ".format(member_vnf_index),
1938                         db_nsr=db_nsr,
1939                         db_vnfr=db_vnfr,
1940                         nslcmop_id=nslcmop_id,
1941                         nsr_id=nsr_id,
1942                         nsi_id=nsi_id,
1943                         vnfd_id=vnfd_id,
1944                         vdu_id=vdu_id,
1945                         kdu_name=kdu_name,
1946                         member_vnf_index=member_vnf_index,
1947                         vdu_index=vdu_index,
1948                         vdu_name=vdu_name,
1949                         deploy_params=deploy_params,
1950                         descriptor_config=descriptor_config,
1951                         base_folder=base_folder,
1952                         task_instantiation_info=tasks_dict_info,
1953                         stage=stage
1954                     )
1955
1956                 # Deploy charms for each VDU that supports one.
1957                 for vdud in get_iterable(vnfd, 'vdu'):
1958                     vdu_id = vdud["id"]
1959                     descriptor_config = vdud.get('vdu-configuration')
1960                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
1961                     if vdur.get("additionalParams"):
1962                         deploy_params_vdu = self._format_additional_params(vdur["additionalParams"])
1963                     else:
1964                         deploy_params_vdu = deploy_params
1965                     if descriptor_config and descriptor_config.get("juju"):
1966                         # look for vdu index in the db_vnfr["vdu"] section
1967                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
1968                         #     if vdur["vdu-id-ref"] == vdu_id:
1969                         #         break
1970                         # else:
1971                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
1972                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
1973                         # vdu_name = vdur.get("name")
1974                         vdu_name = None
1975                         kdu_name = None
1976                         for vdu_index in range(int(vdud.get("count", 1))):
1977                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1978                             self._deploy_n2vc(
1979                                 logging_text=logging_text + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
1980                                     member_vnf_index, vdu_id, vdu_index),
1981                                 db_nsr=db_nsr,
1982                                 db_vnfr=db_vnfr,
1983                                 nslcmop_id=nslcmop_id,
1984                                 nsr_id=nsr_id,
1985                                 nsi_id=nsi_id,
1986                                 vnfd_id=vnfd_id,
1987                                 vdu_id=vdu_id,
1988                                 kdu_name=kdu_name,
1989                                 member_vnf_index=member_vnf_index,
1990                                 vdu_index=vdu_index,
1991                                 vdu_name=vdu_name,
1992                                 deploy_params=deploy_params_vdu,
1993                                 descriptor_config=descriptor_config,
1994                                 base_folder=base_folder,
1995                                 task_instantiation_info=tasks_dict_info,
1996                                 stage=stage
1997                             )
1998                 for kdud in get_iterable(vnfd, 'kdu'):
1999                     kdu_name = kdud["name"]
2000                     descriptor_config = kdud.get('kdu-configuration')
2001                     if descriptor_config and descriptor_config.get("juju"):
2002                         vdu_id = None
2003                         vdu_index = 0
2004                         vdu_name = None
2005                         # look for vdu index in the db_vnfr["vdu"] section
2006                         # for vdur_index, vdur in enumerate(db_vnfr["vdur"]):
2007                         #     if vdur["vdu-id-ref"] == vdu_id:
2008                         #         break
2009                         # else:
2010                         #     raise LcmException("Mismatch vdu_id={} not found in the vnfr['vdur'] list for "
2011                         #                        "member_vnf_index={}".format(vdu_id, member_vnf_index))
2012                         # vdu_name = vdur.get("name")
2013                         # vdu_name = None
2014
2015                         self._deploy_n2vc(
2016                             logging_text=logging_text,
2017                             db_nsr=db_nsr,
2018                             db_vnfr=db_vnfr,
2019                             nslcmop_id=nslcmop_id,
2020                             nsr_id=nsr_id,
2021                             nsi_id=nsi_id,
2022                             vnfd_id=vnfd_id,
2023                             vdu_id=vdu_id,
2024                             kdu_name=kdu_name,
2025                             member_vnf_index=member_vnf_index,
2026                             vdu_index=vdu_index,
2027                             vdu_name=vdu_name,
2028                             deploy_params=deploy_params,
2029                             descriptor_config=descriptor_config,
2030                             base_folder=base_folder,
2031                             task_instantiation_info=tasks_dict_info,
2032                             stage=stage
2033                         )
2034
2035             # Check if this NS has a charm configuration
2036             descriptor_config = nsd.get("ns-configuration")
2037             if descriptor_config and descriptor_config.get("juju"):
2038                 vnfd_id = None
2039                 db_vnfr = None
2040                 member_vnf_index = None
2041                 vdu_id = None
2042                 kdu_name = None
2043                 vdu_index = 0
2044                 vdu_name = None
2045
2046                 # Get additional parameters
2047                 deploy_params = {}
2048                 if db_nsr.get("additionalParamsForNs"):
2049                     deploy_params = self._format_additional_params(db_nsr["additionalParamsForNs"].copy())
2050                 base_folder = nsd["_admin"]["storage"]
2051                 self._deploy_n2vc(
2052                     logging_text=logging_text,
2053                     db_nsr=db_nsr,
2054                     db_vnfr=db_vnfr,
2055                     nslcmop_id=nslcmop_id,
2056                     nsr_id=nsr_id,
2057                     nsi_id=nsi_id,
2058                     vnfd_id=vnfd_id,
2059                     vdu_id=vdu_id,
2060                     kdu_name=kdu_name,
2061                     member_vnf_index=member_vnf_index,
2062                     vdu_index=vdu_index,
2063                     vdu_name=vdu_name,
2064                     deploy_params=deploy_params,
2065                     descriptor_config=descriptor_config,
2066                     base_folder=base_folder,
2067                     task_instantiation_info=tasks_dict_info,
2068                     stage=stage
2069                 )
2070
2071             # rest of staff will be done at finally
2072
2073         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
2074             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(stage[1], e))
2075             exc = e
2076         except asyncio.CancelledError:
2077             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
2078             exc = "Operation was cancelled"
2079         except Exception as e:
2080             exc = traceback.format_exc()
2081             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
2082         finally:
2083             if exc:
2084                 error_list.append(str(exc))
2085             try:
2086                 # wait for pending tasks
2087                 if tasks_dict_info:
2088                     stage[1] = "Waiting for instantiate pending tasks."
2089                     self.logger.debug(logging_text + stage[1])
2090                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_deploy,
2091                                                              stage, nslcmop_id, nsr_id=nsr_id)
2092                 stage[1] = stage[2] = ""
2093             except asyncio.CancelledError:
2094                 error_list.append("Cancelled")
2095                 # TODO cancel all tasks
2096             except Exception as exc:
2097                 error_list.append(str(exc))
2098
2099             # update operation-status
2100             db_nsr_update["operational-status"] = "running"
2101             # let's begin with VCA 'configured' status (later we can change it)
2102             db_nsr_update["config-status"] = "configured"
2103             for task, task_name in tasks_dict_info.items():
2104                 if not task.done() or task.cancelled() or task.exception():
2105                     if task_name.startswith(self.task_name_deploy_vca):
2106                         # A N2VC task is pending
2107                         db_nsr_update["config-status"] = "failed"
2108                     else:
2109                         # RO or KDU task is pending
2110                         db_nsr_update["operational-status"] = "failed"
2111
2112             # update status at database
2113             if error_list:
2114                 error_detail = ". ".join(error_list)
2115                 self.logger.error(logging_text + error_detail)
2116                 error_description_nslcmop = 'Stage: {}. Detail: {}'.format(stage[0], error_detail)
2117                 error_description_nsr = 'Operation: INSTANTIATING.{}, Stage {}'.format(nslcmop_id, stage[0])
2118
2119                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
2120                 db_nslcmop_update["detailed-status"] = error_detail
2121                 nslcmop_operation_state = "FAILED"
2122                 ns_state = "BROKEN"
2123             else:
2124                 error_detail = None
2125                 error_description_nsr = error_description_nslcmop = None
2126                 ns_state = "READY"
2127                 db_nsr_update["detailed-status"] = "Done"
2128                 db_nslcmop_update["detailed-status"] = "Done"
2129                 nslcmop_operation_state = "COMPLETED"
2130
2131             if db_nsr:
2132                 self._write_ns_status(
2133                     nsr_id=nsr_id,
2134                     ns_state=ns_state,
2135                     current_operation="IDLE",
2136                     current_operation_id=None,
2137                     error_description=error_description_nsr,
2138                     error_detail=error_detail,
2139                     other_update=db_nsr_update
2140                 )
2141             self._write_op_status(
2142                 op_id=nslcmop_id,
2143                 stage="",
2144                 error_message=error_description_nslcmop,
2145                 operation_state=nslcmop_operation_state,
2146                 other_update=db_nslcmop_update,
2147             )
2148
2149             if nslcmop_operation_state:
2150                 try:
2151                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2152                                                                    "operationState": nslcmop_operation_state},
2153                                             loop=self.loop)
2154                 except Exception as e:
2155                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2156
2157             self.logger.debug(logging_text + "Exit")
2158             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
2159
2160     async def _add_vca_relations(self, logging_text, nsr_id, vca_index: int, timeout: int = 3600) -> bool:
2161
2162         # steps:
2163         # 1. find all relations for this VCA
2164         # 2. wait for other peers related
2165         # 3. add relations
2166
2167         try:
2168
2169             # STEP 1: find all relations for this VCA
2170
2171             # read nsr record
2172             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2173             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2174
2175             # this VCA data
2176             my_vca = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))[vca_index]
2177
2178             # read all ns-configuration relations
2179             ns_relations = list()
2180             db_ns_relations = deep_get(nsd, ('ns-configuration', 'relation'))
2181             if db_ns_relations:
2182                 for r in db_ns_relations:
2183                     # check if this VCA is in the relation
2184                     if my_vca.get('member-vnf-index') in\
2185                             (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
2186                         ns_relations.append(r)
2187
2188             # read all vnf-configuration relations
2189             vnf_relations = list()
2190             db_vnfd_list = db_nsr.get('vnfd-id')
2191             if db_vnfd_list:
2192                 for vnfd in db_vnfd_list:
2193                     db_vnfd = self.db.get_one("vnfds", {"_id": vnfd})
2194                     db_vnf_relations = deep_get(db_vnfd, ('vnf-configuration', 'relation'))
2195                     if db_vnf_relations:
2196                         for r in db_vnf_relations:
2197                             # check if this VCA is in the relation
2198                             if my_vca.get('vdu_id') in (r.get('entities')[0].get('id'), r.get('entities')[1].get('id')):
2199                                 vnf_relations.append(r)
2200
2201             # if no relations, terminate
2202             if not ns_relations and not vnf_relations:
2203                 self.logger.debug(logging_text + ' No relations')
2204                 return True
2205
2206             self.logger.debug(logging_text + ' adding relations\n    {}\n    {}'.format(ns_relations, vnf_relations))
2207
2208             # add all relations
2209             start = time()
2210             while True:
2211                 # check timeout
2212                 now = time()
2213                 if now - start >= timeout:
2214                     self.logger.error(logging_text + ' : timeout adding relations')
2215                     return False
2216
2217                 # reload nsr from database (we need to update record: _admin.deloyed.VCA)
2218                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2219
2220                 # for each defined NS relation, find the VCA's related
2221                 for r in ns_relations:
2222                     from_vca_ee_id = None
2223                     to_vca_ee_id = None
2224                     from_vca_endpoint = None
2225                     to_vca_endpoint = None
2226                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2227                     for vca in vca_list:
2228                         if vca.get('member-vnf-index') == r.get('entities')[0].get('id') \
2229                                 and vca.get('config_sw_installed'):
2230                             from_vca_ee_id = vca.get('ee_id')
2231                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2232                         if vca.get('member-vnf-index') == r.get('entities')[1].get('id') \
2233                                 and vca.get('config_sw_installed'):
2234                             to_vca_ee_id = vca.get('ee_id')
2235                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2236                     if from_vca_ee_id and to_vca_ee_id:
2237                         # add relation
2238                         await self.n2vc.add_relation(
2239                             ee_id_1=from_vca_ee_id,
2240                             ee_id_2=to_vca_ee_id,
2241                             endpoint_1=from_vca_endpoint,
2242                             endpoint_2=to_vca_endpoint)
2243                         # remove entry from relations list
2244                         ns_relations.remove(r)
2245                     else:
2246                         # check failed peers
2247                         try:
2248                             vca_status_list = db_nsr.get('configurationStatus')
2249                             if vca_status_list:
2250                                 for i in range(len(vca_list)):
2251                                     vca = vca_list[i]
2252                                     vca_status = vca_status_list[i]
2253                                     if vca.get('member-vnf-index') == r.get('entities')[0].get('id'):
2254                                         if vca_status.get('status') == 'BROKEN':
2255                                             # peer broken: remove relation from list
2256                                             ns_relations.remove(r)
2257                                     if vca.get('member-vnf-index') == r.get('entities')[1].get('id'):
2258                                         if vca_status.get('status') == 'BROKEN':
2259                                             # peer broken: remove relation from list
2260                                             ns_relations.remove(r)
2261                         except Exception:
2262                             # ignore
2263                             pass
2264
2265                 # for each defined VNF relation, find the VCA's related
2266                 for r in vnf_relations:
2267                     from_vca_ee_id = None
2268                     to_vca_ee_id = None
2269                     from_vca_endpoint = None
2270                     to_vca_endpoint = None
2271                     vca_list = deep_get(db_nsr, ('_admin', 'deployed', 'VCA'))
2272                     for vca in vca_list:
2273                         if vca.get('vdu_id') == r.get('entities')[0].get('id') and vca.get('config_sw_installed'):
2274                             from_vca_ee_id = vca.get('ee_id')
2275                             from_vca_endpoint = r.get('entities')[0].get('endpoint')
2276                         if vca.get('vdu_id') == r.get('entities')[1].get('id') and vca.get('config_sw_installed'):
2277                             to_vca_ee_id = vca.get('ee_id')
2278                             to_vca_endpoint = r.get('entities')[1].get('endpoint')
2279                     if from_vca_ee_id and to_vca_ee_id:
2280                         # add relation
2281                         await self.n2vc.add_relation(
2282                             ee_id_1=from_vca_ee_id,
2283                             ee_id_2=to_vca_ee_id,
2284                             endpoint_1=from_vca_endpoint,
2285                             endpoint_2=to_vca_endpoint)
2286                         # remove entry from relations list
2287                         vnf_relations.remove(r)
2288                     else:
2289                         # check failed peers
2290                         try:
2291                             vca_status_list = db_nsr.get('configurationStatus')
2292                             if vca_status_list:
2293                                 for i in range(len(vca_list)):
2294                                     vca = vca_list[i]
2295                                     vca_status = vca_status_list[i]
2296                                     if vca.get('vdu_id') == r.get('entities')[0].get('id'):
2297                                         if vca_status.get('status') == 'BROKEN':
2298                                             # peer broken: remove relation from list
2299                                             ns_relations.remove(r)
2300                                     if vca.get('vdu_id') == r.get('entities')[1].get('id'):
2301                                         if vca_status.get('status') == 'BROKEN':
2302                                             # peer broken: remove relation from list
2303                                             ns_relations.remove(r)
2304                         except Exception:
2305                             # ignore
2306                             pass
2307
2308                 # wait for next try
2309                 await asyncio.sleep(5.0)
2310
2311                 if not ns_relations and not vnf_relations:
2312                     self.logger.debug('Relations added')
2313                     break
2314
2315             return True
2316
2317         except Exception as e:
2318             self.logger.warn(logging_text + ' ERROR adding relations: {}'.format(e))
2319             return False
2320
2321     def _write_db_callback(self, task, item, _id, on_done=None, on_exc=None):
2322         """
2323         callback for kdu install intended to store the returned kdu_instance at database
2324         :return: None
2325         """
2326         db_update = {}
2327         try:
2328             result = task.result()
2329             if on_done:
2330                 db_update[on_done] = str(result)
2331         except Exception as e:
2332             if on_exc:
2333                 db_update[on_exc] = str(e)
2334         if db_update:
2335             try:
2336                 self.update_db_2(item, _id, db_update)
2337             except Exception:
2338                 pass
2339
2340     async def deploy_kdus(self, logging_text, nsr_id, nslcmop_id, db_vnfrs, db_vnfds, task_instantiation_info):
2341         # Launch kdus if present in the descriptor
2342
2343         k8scluster_id_2_uuic = {"helm-chart": {}, "juju-bundle": {}}
2344
2345         def _get_cluster_id(cluster_id, cluster_type):
2346             nonlocal k8scluster_id_2_uuic
2347             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
2348                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
2349
2350             db_k8scluster = self.db.get_one("k8sclusters", {"_id": cluster_id}, fail_on_empty=False)
2351             if not db_k8scluster:
2352                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
2353             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
2354             if not k8s_id:
2355                 raise LcmException("K8s cluster '{}' has not been initilized for '{}'".format(cluster_id, cluster_type))
2356             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
2357             return k8s_id
2358
2359         logging_text += "Deploy kdus: "
2360         step = ""
2361         try:
2362             db_nsr_update = {"_admin.deployed.K8s": []}
2363             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2364
2365             index = 0
2366             updated_cluster_list = []
2367
2368             for vnfr_data in db_vnfrs.values():
2369                 for kdur in get_iterable(vnfr_data, "kdur"):
2370                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
2371                     vnfd_id = vnfr_data.get('vnfd-id')
2372                     namespace = kdur.get("k8s-namespace")
2373                     if kdur.get("helm-chart"):
2374                         kdumodel = kdur["helm-chart"]
2375                         k8sclustertype = "helm-chart"
2376                     elif kdur.get("juju-bundle"):
2377                         kdumodel = kdur["juju-bundle"]
2378                         k8sclustertype = "juju-bundle"
2379                     else:
2380                         raise LcmException("kdu type for kdu='{}.{}' is neither helm-chart nor "
2381                                            "juju-bundle. Maybe an old NBI version is running".
2382                                            format(vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]))
2383                     # check if kdumodel is a file and exists
2384                     try:
2385                         storage = deep_get(db_vnfds.get(vnfd_id), ('_admin', 'storage'))
2386                         if storage and storage.get('pkg-dir'):  # may be not present if vnfd has not artifacts
2387                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
2388                             filename = '{}/{}/{}s/{}'.format(storage["folder"], storage["pkg-dir"], k8sclustertype,
2389                                                              kdumodel)
2390                             if self.fs.file_exists(filename, mode='file') or self.fs.file_exists(filename, mode='dir'):
2391                                 kdumodel = self.fs.path + filename
2392                     except (asyncio.TimeoutError, asyncio.CancelledError):
2393                         raise
2394                     except Exception:       # it is not a file
2395                         pass
2396
2397                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
2398                     step = "Synchronize repos for k8s cluster '{}'".format(k8s_cluster_id)
2399                     cluster_uuid = _get_cluster_id(k8s_cluster_id, k8sclustertype)
2400
2401                     if k8sclustertype == "helm-chart" and cluster_uuid not in updated_cluster_list:
2402                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
2403                             self.k8sclusterhelm.synchronize_repos(cluster_uuid=cluster_uuid))
2404                         if del_repo_list or added_repo_dict:
2405                             unset = {'_admin.helm_charts_added.' + item: None for item in del_repo_list}
2406                             updated = {'_admin.helm_charts_added.' +
2407                                        item: name for item, name in added_repo_dict.items()}
2408                             self.logger.debug(logging_text + "repos synchronized on k8s cluster '{}' to_delete: {}, "
2409                                                              "to_add: {}".format(k8s_cluster_id, del_repo_list,
2410                                                                                  added_repo_dict))
2411                             self.db.set_one("k8sclusters", {"_id": k8s_cluster_id}, updated, unset=unset)
2412                         updated_cluster_list.append(cluster_uuid)
2413
2414                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(vnfr_data["member-vnf-index-ref"],
2415                                                                               kdur["kdu-name"], k8s_cluster_id)
2416
2417                     k8s_instace_info = {"kdu-instance": None,
2418                                         "k8scluster-uuid": cluster_uuid,
2419                                         "k8scluster-type": k8sclustertype,
2420                                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
2421                                         "kdu-name": kdur["kdu-name"],
2422                                         "kdu-model": kdumodel,
2423                                         "namespace": namespace}
2424                     db_path = "_admin.deployed.K8s.{}".format(index)
2425                     db_nsr_update[db_path] = k8s_instace_info
2426                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2427
2428                     db_dict = {"collection": "nsrs",
2429                                "filter": {"_id": nsr_id},
2430                                "path": db_path}
2431
2432                     task = asyncio.ensure_future(
2433                         self.k8scluster_map[k8sclustertype].install(cluster_uuid=cluster_uuid, kdu_model=kdumodel,
2434                                                                     atomic=True, params=desc_params,
2435                                                                     db_dict=db_dict, timeout=600,
2436                                                                     kdu_name=kdur["kdu-name"], namespace=namespace))
2437
2438                     task.add_done_callback(partial(self._write_db_callback, item="nsrs", _id=nsr_id,
2439                                                    on_done=db_path + ".kdu-instance",
2440                                                    on_exc=db_path + ".detailed-status"))
2441                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_KDU-{}".format(index), task)
2442                     task_instantiation_info[task] = "Deploying KDU {}".format(kdur["kdu-name"])
2443
2444                     index += 1
2445
2446         except (LcmException, asyncio.CancelledError):
2447             raise
2448         except Exception as e:
2449             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
2450             if isinstance(e, (N2VCException, DbException)):
2451                 self.logger.error(logging_text + msg)
2452             else:
2453                 self.logger.critical(logging_text + msg, exc_info=True)
2454             raise LcmException(msg)
2455         finally:
2456             if db_nsr_update:
2457                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2458
2459     def _deploy_n2vc(self, logging_text, db_nsr, db_vnfr, nslcmop_id, nsr_id, nsi_id, vnfd_id, vdu_id,
2460                      kdu_name, member_vnf_index, vdu_index, vdu_name, deploy_params, descriptor_config,
2461                      base_folder, task_instantiation_info, stage):
2462         # launch instantiate_N2VC in a asyncio task and register task object
2463         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
2464         # if not found, create one entry and update database
2465
2466         # fill db_nsr._admin.deployed.VCA.<index>
2467         vca_index = -1
2468         for vca_index, vca_deployed in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
2469             if not vca_deployed:
2470                 continue
2471             if vca_deployed.get("member-vnf-index") == member_vnf_index and \
2472                     vca_deployed.get("vdu_id") == vdu_id and \
2473                     vca_deployed.get("kdu_name") == kdu_name and \
2474                     vca_deployed.get("vdu_count_index", 0) == vdu_index:
2475                 break
2476         else:
2477             # not found, create one.
2478             vca_deployed = {
2479                 "member-vnf-index": member_vnf_index,
2480                 "vdu_id": vdu_id,
2481                 "kdu_name": kdu_name,
2482                 "vdu_count_index": vdu_index,
2483                 "operational-status": "init",  # TODO revise
2484                 "detailed-status": "",  # TODO revise
2485                 "step": "initial-deploy",   # TODO revise
2486                 "vnfd_id": vnfd_id,
2487                 "vdu_name": vdu_name,
2488             }
2489             vca_index += 1
2490
2491             # create VCA and configurationStatus in db
2492             db_dict = {
2493                 "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
2494                 "configurationStatus.{}".format(vca_index): dict()
2495             }
2496             self.update_db_2("nsrs", nsr_id, db_dict)
2497
2498             db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
2499
2500         # Launch task
2501         task_n2vc = asyncio.ensure_future(
2502             self.instantiate_N2VC(
2503                 logging_text=logging_text,
2504                 vca_index=vca_index,
2505                 nsi_id=nsi_id,
2506                 db_nsr=db_nsr,
2507                 db_vnfr=db_vnfr,
2508                 vdu_id=vdu_id,
2509                 kdu_name=kdu_name,
2510                 vdu_index=vdu_index,
2511                 deploy_params=deploy_params,
2512                 config_descriptor=descriptor_config,
2513                 base_folder=base_folder,
2514                 nslcmop_id=nslcmop_id,
2515                 stage=stage
2516             )
2517         )
2518         self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_N2VC-{}".format(vca_index), task_n2vc)
2519         task_instantiation_info[task_n2vc] = self.task_name_deploy_vca + " {}.{}".format(
2520             member_vnf_index or "", vdu_id or "")
2521
2522     # Check if this VNFD has a configured terminate action
2523     def _has_terminate_config_primitive(self, vnfd):
2524         vnf_config = vnfd.get("vnf-configuration")
2525         if vnf_config and vnf_config.get("terminate-config-primitive"):
2526             return True
2527         else:
2528             return False
2529
2530     @staticmethod
2531     def _get_terminate_config_primitive_seq_list(vnfd):
2532         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
2533         # No need to check for existing primitive twice, already done before
2534         vnf_config = vnfd.get("vnf-configuration")
2535         seq_list = vnf_config.get("terminate-config-primitive")
2536         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
2537         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
2538         return seq_list_sorted
2539
2540     @staticmethod
2541     def _create_nslcmop(nsr_id, operation, params):
2542         """
2543         Creates a ns-lcm-opp content to be stored at database.
2544         :param nsr_id: internal id of the instance
2545         :param operation: instantiate, terminate, scale, action, ...
2546         :param params: user parameters for the operation
2547         :return: dictionary following SOL005 format
2548         """
2549         # Raise exception if invalid arguments
2550         if not (nsr_id and operation and params):
2551             raise LcmException(
2552                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
2553         now = time()
2554         _id = str(uuid4())
2555         nslcmop = {
2556             "id": _id,
2557             "_id": _id,
2558             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
2559             "operationState": "PROCESSING",
2560             "statusEnteredTime": now,
2561             "nsInstanceId": nsr_id,
2562             "lcmOperationType": operation,
2563             "startTime": now,
2564             "isAutomaticInvocation": False,
2565             "operationParams": params,
2566             "isCancelPending": False,
2567             "links": {
2568                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
2569                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
2570             }
2571         }
2572         return nslcmop
2573
2574     def _format_additional_params(self, params):
2575         params = params or {}
2576         for key, value in params.items():
2577             if str(value).startswith("!!yaml "):
2578                 params[key] = yaml.safe_load(value[7:])
2579         return params
2580
2581     def _get_terminate_primitive_params(self, seq, vnf_index):
2582         primitive = seq.get('name')
2583         primitive_params = {}
2584         params = {
2585             "member_vnf_index": vnf_index,
2586             "primitive": primitive,
2587             "primitive_params": primitive_params,
2588         }
2589         desc_params = {}
2590         return self._map_primitive_params(seq, params, desc_params)
2591
2592     # sub-operations
2593
2594     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
2595         op = deep_get(db_nslcmop, ('_admin', 'operations'), [])[op_index]
2596         if op.get('operationState') == 'COMPLETED':
2597             # b. Skip sub-operation
2598             # _ns_execute_primitive() or RO.create_action() will NOT be executed
2599             return self.SUBOPERATION_STATUS_SKIP
2600         else:
2601             # c. retry executing sub-operation
2602             # The sub-operation exists, and operationState != 'COMPLETED'
2603             # Update operationState = 'PROCESSING' to indicate a retry.
2604             operationState = 'PROCESSING'
2605             detailed_status = 'In progress'
2606             self._update_suboperation_status(
2607                 db_nslcmop, op_index, operationState, detailed_status)
2608             # Return the sub-operation index
2609             # _ns_execute_primitive() or RO.create_action() will be called from scale()
2610             # with arguments extracted from the sub-operation
2611             return op_index
2612
2613     # Find a sub-operation where all keys in a matching dictionary must match
2614     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
2615     def _find_suboperation(self, db_nslcmop, match):
2616         if db_nslcmop and match:
2617             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
2618             for i, op in enumerate(op_list):
2619                 if all(op.get(k) == match[k] for k in match):
2620                     return i
2621         return self.SUBOPERATION_STATUS_NOT_FOUND
2622
2623     # Update status for a sub-operation given its index
2624     def _update_suboperation_status(self, db_nslcmop, op_index, operationState, detailed_status):
2625         # Update DB for HA tasks
2626         q_filter = {'_id': db_nslcmop['_id']}
2627         update_dict = {'_admin.operations.{}.operationState'.format(op_index): operationState,
2628                        '_admin.operations.{}.detailed-status'.format(op_index): detailed_status}
2629         self.db.set_one("nslcmops",
2630                         q_filter=q_filter,
2631                         update_dict=update_dict,
2632                         fail_on_empty=False)
2633
2634     # Add sub-operation, return the index of the added sub-operation
2635     # Optionally, set operationState, detailed-status, and operationType
2636     # Status and type are currently set for 'scale' sub-operations:
2637     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
2638     # 'detailed-status' : status message
2639     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
2640     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
2641     def _add_suboperation(self, db_nslcmop, vnf_index, vdu_id, vdu_count_index, vdu_name, primitive,
2642                           mapped_primitive_params, operationState=None, detailed_status=None, operationType=None,
2643                           RO_nsr_id=None, RO_scaling_info=None):
2644         if not db_nslcmop:
2645             return self.SUBOPERATION_STATUS_NOT_FOUND
2646         # Get the "_admin.operations" list, if it exists
2647         db_nslcmop_admin = db_nslcmop.get('_admin', {})
2648         op_list = db_nslcmop_admin.get('operations')
2649         # Create or append to the "_admin.operations" list
2650         new_op = {'member_vnf_index': vnf_index,
2651                   'vdu_id': vdu_id,
2652                   'vdu_count_index': vdu_count_index,
2653                   'primitive': primitive,
2654                   'primitive_params': mapped_primitive_params}
2655         if operationState:
2656             new_op['operationState'] = operationState
2657         if detailed_status:
2658             new_op['detailed-status'] = detailed_status
2659         if operationType:
2660             new_op['lcmOperationType'] = operationType
2661         if RO_nsr_id:
2662             new_op['RO_nsr_id'] = RO_nsr_id
2663         if RO_scaling_info:
2664             new_op['RO_scaling_info'] = RO_scaling_info
2665         if not op_list:
2666             # No existing operations, create key 'operations' with current operation as first list element
2667             db_nslcmop_admin.update({'operations': [new_op]})
2668             op_list = db_nslcmop_admin.get('operations')
2669         else:
2670             # Existing operations, append operation to list
2671             op_list.append(new_op)
2672
2673         db_nslcmop_update = {'_admin.operations': op_list}
2674         self.update_db_2("nslcmops", db_nslcmop['_id'], db_nslcmop_update)
2675         op_index = len(op_list) - 1
2676         return op_index
2677
2678     # Helper methods for scale() sub-operations
2679
2680     # pre-scale/post-scale:
2681     # Check for 3 different cases:
2682     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
2683     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
2684     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
2685     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index, vnf_config_primitive, primitive_params,
2686                                          operationType, RO_nsr_id=None, RO_scaling_info=None):
2687         # Find this sub-operation
2688         if RO_nsr_id and RO_scaling_info:
2689             operationType = 'SCALE-RO'
2690             match = {
2691                 'member_vnf_index': vnf_index,
2692                 'RO_nsr_id': RO_nsr_id,
2693                 'RO_scaling_info': RO_scaling_info,
2694             }
2695         else:
2696             match = {
2697                 'member_vnf_index': vnf_index,
2698                 'primitive': vnf_config_primitive,
2699                 'primitive_params': primitive_params,
2700                 'lcmOperationType': operationType
2701             }
2702         op_index = self._find_suboperation(db_nslcmop, match)
2703         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
2704             # a. New sub-operation
2705             # The sub-operation does not exist, add it.
2706             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
2707             # The following parameters are set to None for all kind of scaling:
2708             vdu_id = None
2709             vdu_count_index = None
2710             vdu_name = None
2711             if RO_nsr_id and RO_scaling_info:
2712                 vnf_config_primitive = None
2713                 primitive_params = None
2714             else:
2715                 RO_nsr_id = None
2716                 RO_scaling_info = None
2717             # Initial status for sub-operation
2718             operationState = 'PROCESSING'
2719             detailed_status = 'In progress'
2720             # Add sub-operation for pre/post-scaling (zero or more operations)
2721             self._add_suboperation(db_nslcmop,
2722                                    vnf_index,
2723                                    vdu_id,
2724                                    vdu_count_index,
2725                                    vdu_name,
2726                                    vnf_config_primitive,
2727                                    primitive_params,
2728                                    operationState,
2729                                    detailed_status,
2730                                    operationType,
2731                                    RO_nsr_id,
2732                                    RO_scaling_info)
2733             return self.SUBOPERATION_STATUS_NEW
2734         else:
2735             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
2736             # or op_index (operationState != 'COMPLETED')
2737             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
2738
2739     # Function to return execution_environment id
2740
2741     def _get_ee_id(self, vnf_index, vdu_id, vca_deployed_list):
2742         # TODO vdu_index_count
2743         for vca in vca_deployed_list:
2744             if vca["member-vnf-index"] == vnf_index and vca["vdu_id"] == vdu_id:
2745                 return vca["ee_id"]
2746
2747     async def destroy_N2VC(self, logging_text, db_nslcmop, vca_deployed, config_descriptor, vca_index, destroy_ee=True):
2748         """
2749         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
2750         :param logging_text:
2751         :param db_nslcmop:
2752         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
2753         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
2754         :param vca_index: index in the database _admin.deployed.VCA
2755         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
2756         :return: None or exception
2757         """
2758         # execute terminate_primitives
2759         terminate_primitives = config_descriptor.get("terminate-config-primitive")
2760         vdu_id = vca_deployed.get("vdu_id")
2761         vdu_count_index = vca_deployed.get("vdu_count_index")
2762         vdu_name = vca_deployed.get("vdu_name")
2763         vnf_index = vca_deployed.get("member-vnf-index")
2764         if terminate_primitives and vca_deployed.get("needed_terminate"):
2765             # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
2766             terminate_primitives = sorted(terminate_primitives, key=lambda x: int(x['seq']))
2767             for seq in terminate_primitives:
2768                 # For each sequence in list, get primitive and call _ns_execute_primitive()
2769                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
2770                     vnf_index, seq.get("name"))
2771                 self.logger.debug(logging_text + step)
2772                 # Create the primitive for each sequence, i.e. "primitive": "touch"
2773                 primitive = seq.get('name')
2774                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
2775                 # The following 3 parameters are currently set to None for 'terminate':
2776                 # vdu_id, vdu_count_index, vdu_name
2777
2778                 # Add sub-operation
2779                 self._add_suboperation(db_nslcmop,
2780                                        vnf_index,
2781                                        vdu_id,
2782                                        vdu_count_index,
2783                                        vdu_name,
2784                                        primitive,
2785                                        mapped_primitive_params)
2786                 # Sub-operations: Call _ns_execute_primitive() instead of action()
2787                 try:
2788                     result, result_detail = await self._ns_execute_primitive(vca_deployed["ee_id"], primitive,
2789                                                                              mapped_primitive_params)
2790                 except LcmException:
2791                     # this happens when VCA is not deployed. In this case it is not needed to terminate
2792                     continue
2793                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
2794                 if result not in result_ok:
2795                     raise LcmException("terminate_primitive {}  for vnf_member_index={} fails with "
2796                                        "error {}".format(seq.get("name"), vnf_index, result_detail))
2797             # set that this VCA do not need terminated
2798             db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(vca_index)
2799             self.update_db_2("nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False})
2800
2801         if destroy_ee:
2802             await self.n2vc.delete_execution_environment(vca_deployed["ee_id"])
2803
2804     async def _delete_all_N2VC(self, db_nsr: dict):
2805         self._write_all_config_status(db_nsr=db_nsr, status='TERMINATING')
2806         namespace = "." + db_nsr["_id"]
2807         try:
2808             await self.n2vc.delete_namespace(namespace=namespace, total_timeout=self.timeout_charm_delete)
2809         except N2VCNotFound:  # already deleted. Skip
2810             pass
2811         self._write_all_config_status(db_nsr=db_nsr, status='DELETED')
2812
2813     async def _terminate_RO(self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage):
2814         """
2815         Terminates a deployment from RO
2816         :param logging_text:
2817         :param nsr_deployed: db_nsr._admin.deployed
2818         :param nsr_id:
2819         :param nslcmop_id:
2820         :param stage: list of string with the content to write on db_nslcmop.detailed-status.
2821             this method will update only the index 2, but it will write on database the concatenated content of the list
2822         :return:
2823         """
2824         db_nsr_update = {}
2825         failed_detail = []
2826         ro_nsr_id = ro_delete_action = None
2827         if nsr_deployed and nsr_deployed.get("RO"):
2828             ro_nsr_id = nsr_deployed["RO"].get("nsr_id")
2829             ro_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
2830         try:
2831             if ro_nsr_id:
2832                 stage[2] = "Deleting ns from VIM."
2833                 db_nsr_update["detailed-status"] = " ".join(stage)
2834                 self._write_op_status(nslcmop_id, stage)
2835                 self.logger.debug(logging_text + stage[2])
2836                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2837                 self._write_op_status(nslcmop_id, stage)
2838                 desc = await self.RO.delete("ns", ro_nsr_id)
2839                 ro_delete_action = desc["action_id"]
2840                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = ro_delete_action
2841                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2842                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2843             if ro_delete_action:
2844                 # wait until NS is deleted from VIM
2845                 stage[2] = "Waiting ns deleted from VIM."
2846                 detailed_status_old = None
2847                 self.logger.debug(logging_text + stage[2] + " RO_id={} ro_delete_action={}".format(ro_nsr_id,
2848                                                                                                    ro_delete_action))
2849                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2850                 self._write_op_status(nslcmop_id, stage)
2851
2852                 delete_timeout = 20 * 60  # 20 minutes
2853                 while delete_timeout > 0:
2854                     desc = await self.RO.show(
2855                         "ns",
2856                         item_id_name=ro_nsr_id,
2857                         extra_item="action",
2858                         extra_item_id=ro_delete_action)
2859
2860                     # deploymentStatus
2861                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
2862
2863                     ns_status, ns_status_info = self.RO.check_action_status(desc)
2864                     if ns_status == "ERROR":
2865                         raise ROclient.ROClientException(ns_status_info)
2866                     elif ns_status == "BUILD":
2867                         stage[2] = "Deleting from VIM {}".format(ns_status_info)
2868                     elif ns_status == "ACTIVE":
2869                         db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2870                         db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2871                         break
2872                     else:
2873                         assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2874                     if stage[2] != detailed_status_old:
2875                         detailed_status_old = stage[2]
2876                         db_nsr_update["detailed-status"] = " ".join(stage)
2877                         self._write_op_status(nslcmop_id, stage)
2878                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
2879                     await asyncio.sleep(5, loop=self.loop)
2880                     delete_timeout -= 5
2881                 else:  # delete_timeout <= 0:
2882                     raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
2883
2884         except Exception as e:
2885             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2886             if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2887                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2888                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2889                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2890                 self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(ro_nsr_id))
2891             elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2892                 failed_detail.append("delete conflict: {}".format(e))
2893                 self.logger.debug(logging_text + "RO_ns_id={} delete conflict: {}".format(ro_nsr_id, e))
2894             else:
2895                 failed_detail.append("delete error: {}".format(e))
2896                 self.logger.error(logging_text + "RO_ns_id={} delete error: {}".format(ro_nsr_id, e))
2897
2898         # Delete nsd
2899         if not failed_detail and deep_get(nsr_deployed, ("RO", "nsd_id")):
2900             ro_nsd_id = nsr_deployed["RO"]["nsd_id"]
2901             try:
2902                 stage[2] = "Deleting nsd from RO."
2903                 db_nsr_update["detailed-status"] = " ".join(stage)
2904                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2905                 self._write_op_status(nslcmop_id, stage)
2906                 await self.RO.delete("nsd", ro_nsd_id)
2907                 self.logger.debug(logging_text + "ro_nsd_id={} deleted".format(ro_nsd_id))
2908                 db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2909             except Exception as e:
2910                 if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2911                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2912                     self.logger.debug(logging_text + "ro_nsd_id={} already deleted".format(ro_nsd_id))
2913                 elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2914                     failed_detail.append("ro_nsd_id={} delete conflict: {}".format(ro_nsd_id, e))
2915                     self.logger.debug(logging_text + failed_detail[-1])
2916                 else:
2917                     failed_detail.append("ro_nsd_id={} delete error: {}".format(ro_nsd_id, e))
2918                     self.logger.error(logging_text + failed_detail[-1])
2919
2920         if not failed_detail and deep_get(nsr_deployed, ("RO", "vnfd")):
2921             for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
2922                 if not vnf_deployed or not vnf_deployed["id"]:
2923                     continue
2924                 try:
2925                     ro_vnfd_id = vnf_deployed["id"]
2926                     stage[2] = "Deleting member_vnf_index={} ro_vnfd_id={} from RO.".format(
2927                         vnf_deployed["member-vnf-index"], ro_vnfd_id)
2928                     db_nsr_update["detailed-status"] = " ".join(stage)
2929                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2930                     self._write_op_status(nslcmop_id, stage)
2931                     await self.RO.delete("vnfd", ro_vnfd_id)
2932                     self.logger.debug(logging_text + "ro_vnfd_id={} deleted".format(ro_vnfd_id))
2933                     db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2934                 except Exception as e:
2935                     if isinstance(e, ROclient.ROClientException) and e.http_code == 404:  # not found
2936                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2937                         self.logger.debug(logging_text + "ro_vnfd_id={} already deleted ".format(ro_vnfd_id))
2938                     elif isinstance(e, ROclient.ROClientException) and e.http_code == 409:  # conflict
2939                         failed_detail.append("ro_vnfd_id={} delete conflict: {}".format(ro_vnfd_id, e))
2940                         self.logger.debug(logging_text + failed_detail[-1])
2941                     else:
2942                         failed_detail.append("ro_vnfd_id={} delete error: {}".format(ro_vnfd_id, e))
2943                         self.logger.error(logging_text + failed_detail[-1])
2944
2945         if failed_detail:
2946             stage[2] = "Error deleting from VIM"
2947         else:
2948             stage[2] = "Deleted from VIM"
2949         db_nsr_update["detailed-status"] = " ".join(stage)
2950         self.update_db_2("nsrs", nsr_id, db_nsr_update)
2951         self._write_op_status(nslcmop_id, stage)
2952
2953         if failed_detail:
2954             raise LcmException("; ".join(failed_detail))
2955
2956     async def terminate(self, nsr_id, nslcmop_id):
2957         # Try to lock HA task here
2958         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2959         if not task_is_locked_by_me:
2960             return
2961
2962         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
2963         self.logger.debug(logging_text + "Enter")
2964         timeout_ns_terminate = self.timeout_ns_terminate
2965         db_nsr = None
2966         db_nslcmop = None
2967         operation_params = None
2968         exc = None
2969         error_list = []   # annotates all failed error messages
2970         db_nslcmop_update = {}
2971         autoremove = False  # autoremove after terminated
2972         tasks_dict_info = {}
2973         db_nsr_update = {}
2974         stage = ["Stage 1/3: Preparing task.", "Waiting for previous operations to terminate.", ""]
2975         # ^ contains [stage, step, VIM-status]
2976         try:
2977             # wait for any previous tasks in process
2978             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
2979
2980             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
2981             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2982             operation_params = db_nslcmop.get("operationParams") or {}
2983             if operation_params.get("timeout_ns_terminate"):
2984                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
2985             stage[1] = "Getting nsr={} from db.".format(nsr_id)
2986             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2987
2988             db_nsr_update["operational-status"] = "terminating"
2989             db_nsr_update["config-status"] = "terminating"
2990             self._write_ns_status(
2991                 nsr_id=nsr_id,
2992                 ns_state="TERMINATING",
2993                 current_operation="TERMINATING",
2994                 current_operation_id=nslcmop_id,
2995                 other_update=db_nsr_update
2996             )
2997             self._write_op_status(
2998                 op_id=nslcmop_id,
2999                 queuePosition=0,
3000                 stage=stage
3001             )
3002             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
3003             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
3004                 return
3005
3006             stage[1] = "Getting vnf descriptors from db."
3007             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
3008             db_vnfds_from_id = {}
3009             db_vnfds_from_member_index = {}
3010             # Loop over VNFRs
3011             for vnfr in db_vnfrs_list:
3012                 vnfd_id = vnfr["vnfd-id"]
3013                 if vnfd_id not in db_vnfds_from_id:
3014                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
3015                     db_vnfds_from_id[vnfd_id] = vnfd
3016                 db_vnfds_from_member_index[vnfr["member-vnf-index-ref"]] = db_vnfds_from_id[vnfd_id]
3017
3018             # Destroy individual execution environments when there are terminating primitives.
3019             # Rest of EE will be deleted at once
3020             if not operation_params.get("skip_terminate_primitives"):
3021                 stage[0] = "Stage 2/3 execute terminating primitives."
3022                 stage[1] = "Looking execution environment that needs terminate."
3023                 self.logger.debug(logging_text + stage[1])
3024                 for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
3025                     config_descriptor = None
3026                     if not vca or not vca.get("ee_id") or not vca.get("needed_terminate"):
3027                         continue
3028                     if not vca.get("member-vnf-index"):
3029                         # ns
3030                         config_descriptor = db_nsr.get("ns-configuration")
3031                     elif vca.get("vdu_id"):
3032                         db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
3033                         vdud = next((vdu for vdu in db_vnfd.get("vdu", ()) if vdu["id"] == vca.get("vdu_id")), None)
3034                         if vdud:
3035                             config_descriptor = vdud.get("vdu-configuration")
3036                     elif vca.get("kdu_name"):
3037                         db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
3038                         kdud = next((kdu for kdu in db_vnfd.get("kdu", ()) if kdu["name"] == vca.get("kdu_name")), None)
3039                         if kdud:
3040                             config_descriptor = kdud.get("kdu-configuration")
3041                     else:
3042                         config_descriptor = db_vnfds_from_member_index[vca["member-vnf-index"]].get("vnf-configuration")
3043                     task = asyncio.ensure_future(self.destroy_N2VC(logging_text, db_nslcmop, vca, config_descriptor,
3044                                                                    vca_index, False))
3045                     tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
3046
3047                 # wait for pending tasks of terminate primitives
3048                 if tasks_dict_info:
3049                     self.logger.debug(logging_text + 'Waiting for terminate primitive pending tasks...')
3050                     error_list = await self._wait_for_tasks(logging_text, tasks_dict_info,
3051                                                             min(self.timeout_charm_delete, timeout_ns_terminate),
3052                                                             stage, nslcmop_id)
3053                     if error_list:
3054                         return   # raise LcmException("; ".join(error_list))
3055                     tasks_dict_info.clear()
3056
3057             # remove All execution environments at once
3058             stage[0] = "Stage 3/3 delete all."
3059
3060             if nsr_deployed.get("VCA"):
3061                 stage[1] = "Deleting all execution environments."
3062                 self.logger.debug(logging_text + stage[1])
3063                 task_delete_ee = asyncio.ensure_future(asyncio.wait_for(self._delete_all_N2VC(db_nsr=db_nsr),
3064                                                                         timeout=self.timeout_charm_delete))
3065                 # task_delete_ee = asyncio.ensure_future(self.n2vc.delete_namespace(namespace="." + nsr_id))
3066                 tasks_dict_info[task_delete_ee] = "Terminating all VCA"
3067
3068             # Delete from k8scluster
3069             stage[1] = "Deleting KDUs."
3070             self.logger.debug(logging_text + stage[1])
3071             # print(nsr_deployed)
3072             for kdu in get_iterable(nsr_deployed, "K8s"):
3073                 if not kdu or not kdu.get("kdu-instance"):
3074                     continue
3075                 kdu_instance = kdu.get("kdu-instance")
3076                 if kdu.get("k8scluster-type") in self.k8scluster_map:
3077                     task_delete_kdu_instance = asyncio.ensure_future(
3078                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
3079                             cluster_uuid=kdu.get("k8scluster-uuid"),
3080                             kdu_instance=kdu_instance))
3081                 else:
3082                     self.logger.error(logging_text + "Unknown k8s deployment type {}".
3083                                       format(kdu.get("k8scluster-type")))
3084                     continue
3085                 tasks_dict_info[task_delete_kdu_instance] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
3086
3087             # remove from RO
3088             stage[1] = "Deleting ns from VIM."
3089             if self.ng_ro:
3090                 task_delete_ro = asyncio.ensure_future(
3091                     self._terminate_ng_ro(logging_text, nsr_deployed, nsr_id, nslcmop_id, stage))
3092             else:
3093                 task_delete_ro = asyncio.ensure_future(
3094                     self._terminate_RO(logging_text, nsr_deployed, nsr_id, nslcmop_id, stage))
3095             tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
3096
3097             # rest of staff will be done at finally
3098
3099         except (ROclient.ROClientException, DbException, LcmException, N2VCException) as e:
3100             self.logger.error(logging_text + "Exit Exception {}".format(e))
3101             exc = e
3102         except asyncio.CancelledError:
3103             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(stage[1]))
3104             exc = "Operation was cancelled"
3105         except Exception as e:
3106             exc = traceback.format_exc()
3107             self.logger.critical(logging_text + "Exit Exception while '{}': {}".format(stage[1], e), exc_info=True)
3108         finally:
3109             if exc:
3110                 error_list.append(str(exc))
3111             try:
3112                 # wait for pending tasks
3113                 if tasks_dict_info:
3114                     stage[1] = "Waiting for terminate pending tasks."
3115                     self.logger.debug(logging_text + stage[1])
3116                     error_list += await self._wait_for_tasks(logging_text, tasks_dict_info, timeout_ns_terminate,
3117                                                              stage, nslcmop_id)
3118                 stage[1] = stage[2] = ""
3119             except asyncio.CancelledError:
3120                 error_list.append("Cancelled")
3121                 # TODO cancell all tasks
3122             except Exception as exc:
3123                 error_list.append(str(exc))
3124             # update status at database
3125             if error_list:
3126                 error_detail = "; ".join(error_list)
3127                 # self.logger.error(logging_text + error_detail)
3128                 error_description_nslcmop = 'Stage: {}. Detail: {}'.format(stage[0], error_detail)
3129                 error_description_nsr = 'Operation: TERMINATING.{}, Stage {}.'.format(nslcmop_id, stage[0])
3130
3131                 db_nsr_update["operational-status"] = "failed"
3132                 db_nsr_update["detailed-status"] = error_description_nsr + " Detail: " + error_detail
3133                 db_nslcmop_update["detailed-status"] = error_detail
3134                 nslcmop_operation_state = "FAILED"
3135                 ns_state = "BROKEN"
3136             else:
3137                 error_detail = None
3138                 error_description_nsr = error_description_nslcmop = None
3139                 ns_state = "NOT_INSTANTIATED"
3140                 db_nsr_update["operational-status"] = "terminated"
3141                 db_nsr_update["detailed-status"] = "Done"
3142                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
3143                 db_nslcmop_update["detailed-status"] = "Done"
3144                 nslcmop_operation_state = "COMPLETED"
3145
3146             if db_nsr:
3147                 self._write_ns_status(
3148                     nsr_id=nsr_id,
3149                     ns_state=ns_state,
3150                     current_operation="IDLE",
3151                     current_operation_id=None,
3152                     error_description=error_description_nsr,
3153                     error_detail=error_detail,
3154                     other_update=db_nsr_update
3155                 )
3156             self._write_op_status(
3157                 op_id=nslcmop_id,
3158                 stage="",
3159                 error_message=error_description_nslcmop,
3160                 operation_state=nslcmop_operation_state,
3161                 other_update=db_nslcmop_update,
3162             )
3163             if operation_params:
3164                 autoremove = operation_params.get("autoremove", False)
3165             if nslcmop_operation_state:
3166                 try:
3167                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3168                                                                  "operationState": nslcmop_operation_state,
3169                                                                  "autoremove": autoremove},
3170                                             loop=self.loop)
3171                 except Exception as e:
3172                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3173
3174             self.logger.debug(logging_text + "Exit")
3175             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
3176
3177     async def _wait_for_tasks(self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None):
3178         time_start = time()
3179         error_detail_list = []
3180         error_list = []
3181         pending_tasks = list(created_tasks_info.keys())
3182         num_tasks = len(pending_tasks)
3183         num_done = 0
3184         stage[1] = "{}/{}.".format(num_done, num_tasks)
3185         self._write_op_status(nslcmop_id, stage)
3186         while pending_tasks:
3187             new_error = None
3188             _timeout = timeout + time_start - time()
3189             done, pending_tasks = await asyncio.wait(pending_tasks, timeout=_timeout,
3190                                                      return_when=asyncio.FIRST_COMPLETED)
3191             num_done += len(done)
3192             if not done:   # Timeout
3193                 for task in pending_tasks:
3194                     new_error = created_tasks_info[task] + ": Timeout"
3195                     error_detail_list.append(new_error)
3196                     error_list.append(new_error)
3197                 break
3198             for task in done:
3199                 if task.cancelled():
3200                     exc = "Cancelled"
3201                 else:
3202                     exc = task.exception()
3203                 if exc:
3204                     if isinstance(exc, asyncio.TimeoutError):
3205                         exc = "Timeout"
3206                     new_error = created_tasks_info[task] + ": {}".format(exc)
3207                     error_list.append(created_tasks_info[task])
3208                     error_detail_list.append(new_error)
3209                     if isinstance(exc, (str, DbException, N2VCException, ROclient.ROClientException, LcmException,
3210                                         K8sException)):
3211                         self.logger.error(logging_text + new_error)
3212                     else:
3213                         exc_traceback = "".join(traceback.format_exception(None, exc, exc.__traceback__))
3214                         self.logger.error(logging_text + created_tasks_info[task] + exc_traceback)
3215                 else:
3216                     self.logger.debug(logging_text + created_tasks_info[task] + ": Done")
3217             stage[1] = "{}/{}.".format(num_done, num_tasks)
3218             if new_error:
3219                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
3220                 if nsr_id:  # update also nsr
3221                     self.update_db_2("nsrs", nsr_id, {"errorDescription": "Error at: " + ", ".join(error_list),
3222                                                       "errorDetail": ". ".join(error_detail_list)})
3223             self._write_op_status(nslcmop_id, stage)
3224         return error_detail_list
3225
3226     @staticmethod
3227     def _map_primitive_params(primitive_desc, params, instantiation_params):
3228         """
3229         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
3230         The default-value is used. If it is between < > it look for a value at instantiation_params
3231         :param primitive_desc: portion of VNFD/NSD that describes primitive
3232         :param params: Params provided by user
3233         :param instantiation_params: Instantiation params provided by user
3234         :return: a dictionary with the calculated params
3235         """
3236         calculated_params = {}
3237         for parameter in primitive_desc.get("parameter", ()):
3238             param_name = parameter["name"]
3239             if param_name in params:
3240                 calculated_params[param_name] = params[param_name]
3241             elif "default-value" in parameter or "value" in parameter:
3242                 if "value" in parameter:
3243                     calculated_params[param_name] = parameter["value"]
3244                 else:
3245                     calculated_params[param_name] = parameter["default-value"]
3246                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
3247                         and calculated_params[param_name].endswith(">"):
3248                     if calculated_params[param_name][1:-1] in instantiation_params:
3249                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
3250                     else:
3251                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
3252                                            format(calculated_params[param_name], primitive_desc["name"]))
3253             else:
3254                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
3255                                    format(param_name, primitive_desc["name"]))
3256
3257             if isinstance(calculated_params[param_name], (dict, list, tuple)):
3258                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
3259                                                                width=256)
3260             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
3261                 calculated_params[param_name] = calculated_params[param_name][7:]
3262
3263         # add always ns_config_info if primitive name is config
3264         if primitive_desc["name"] == "config":
3265             if "ns_config_info" in instantiation_params:
3266                 calculated_params["ns_config_info"] = instantiation_params["ns_config_info"]
3267         return calculated_params
3268
3269     def _look_for_deployed_vca(self, deployed_vca, member_vnf_index, vdu_id, vdu_count_index, kdu_name=None):
3270         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
3271         for vca in deployed_vca:
3272             if not vca:
3273                 continue
3274             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
3275                 continue
3276             if vdu_count_index is not None and vdu_count_index != vca["vdu_count_index"]:
3277                 continue
3278             if kdu_name and kdu_name != vca["kdu_name"]:
3279                 continue
3280             break
3281         else:
3282             # vca_deployed not found
3283             raise LcmException("charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} is not "
3284                                "deployed".format(member_vnf_index, vdu_id, kdu_name, vdu_count_index))
3285
3286         # get ee_id
3287         ee_id = vca.get("ee_id")
3288         if not ee_id:
3289             raise LcmException("charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
3290                                "execution environment"
3291                                .format(member_vnf_index, vdu_id, kdu_name, vdu_count_index))
3292         return ee_id
3293
3294     async def _ns_execute_primitive(self, ee_id, primitive, primitive_params, retries=0,
3295                                     retries_interval=30, timeout=None) -> (str, str):
3296         try:
3297             if primitive == "config":
3298                 primitive_params = {"params": primitive_params}
3299
3300             while retries >= 0:
3301                 try:
3302                     output = await asyncio.wait_for(
3303                         self.n2vc.exec_primitive(
3304                             ee_id=ee_id,
3305                             primitive_name=primitive,
3306                             params_dict=primitive_params,
3307                             progress_timeout=self.timeout_progress_primitive,
3308                             total_timeout=self.timeout_primitive),
3309                         timeout=timeout or self.timeout_primitive)
3310                     # execution was OK
3311                     break
3312                 except asyncio.CancelledError:
3313                     raise
3314                 except Exception as e:  # asyncio.TimeoutError
3315                     if isinstance(e, asyncio.TimeoutError):
3316                         e = "Timeout"
3317                     retries -= 1
3318                     if retries >= 0:
3319                         self.logger.debug('Error executing action {} on {} -> {}'.format(primitive, ee_id, e))
3320                         # wait and retry
3321                         await asyncio.sleep(retries_interval, loop=self.loop)
3322                     else:
3323                         return 'FAILED', str(e)
3324
3325             return 'COMPLETED', output
3326
3327         except (LcmException, asyncio.CancelledError):
3328             raise
3329         except Exception as e:
3330             return 'FAIL', 'Error executing action {}: {}'.format(primitive, e)
3331
3332     async def action(self, nsr_id, nslcmop_id):
3333
3334         # Try to lock HA task here
3335         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3336         if not task_is_locked_by_me:
3337             return
3338
3339         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
3340         self.logger.debug(logging_text + "Enter")
3341         # get all needed from database
3342         db_nsr = None
3343         db_nslcmop = None
3344         db_nsr_update = {}
3345         db_nslcmop_update = {}
3346         nslcmop_operation_state = None
3347         error_description_nslcmop = None
3348         exc = None
3349         try:
3350             # wait for any previous tasks in process
3351             step = "Waiting for previous operations to terminate"
3352             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3353
3354             self._write_ns_status(
3355                 nsr_id=nsr_id,
3356                 ns_state=None,
3357                 current_operation="RUNNING ACTION",
3358                 current_operation_id=nslcmop_id
3359             )
3360
3361             step = "Getting information from database"
3362             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3363             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3364
3365             nsr_deployed = db_nsr["_admin"].get("deployed")
3366             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3367             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3368             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
3369             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3370             primitive = db_nslcmop["operationParams"]["primitive"]
3371             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
3372             timeout_ns_action = db_nslcmop["operationParams"].get("timeout_ns_action", self.timeout_primitive)
3373
3374             if vnf_index:
3375                 step = "Getting vnfr from database"
3376                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3377                 step = "Getting vnfd from database"
3378                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3379             else:
3380                 step = "Getting nsd from database"
3381                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
3382
3383             # for backward compatibility
3384             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3385                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3386                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3387                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3388
3389             # look for primitive
3390             config_primitive_desc = None
3391             if vdu_id:
3392                 for vdu in get_iterable(db_vnfd, "vdu"):
3393                     if vdu_id == vdu["id"]:
3394                         for config_primitive in deep_get(vdu, ("vdu-configuration", "config-primitive"), ()):
3395                             if config_primitive["name"] == primitive:
3396                                 config_primitive_desc = config_primitive
3397                                 break
3398                         break
3399             elif kdu_name:
3400                 for kdu in get_iterable(db_vnfd, "kdu"):
3401                     if kdu_name == kdu["name"]:
3402                         for config_primitive in deep_get(kdu, ("kdu-configuration", "config-primitive"), ()):
3403                             if config_primitive["name"] == primitive:
3404                                 config_primitive_desc = config_primitive
3405                                 break
3406                         break
3407             elif vnf_index:
3408                 for config_primitive in deep_get(db_vnfd, ("vnf-configuration", "config-primitive"), ()):
3409                     if config_primitive["name"] == primitive:
3410                         config_primitive_desc = config_primitive
3411                         break
3412             else:
3413                 for config_primitive in deep_get(db_nsd, ("ns-configuration", "config-primitive"), ()):
3414                     if config_primitive["name"] == primitive:
3415                         config_primitive_desc = config_primitive
3416                         break
3417
3418             if not config_primitive_desc and not (kdu_name and primitive in ("upgrade", "rollback", "status")):
3419                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
3420                                    format(primitive))
3421
3422             if vnf_index:
3423                 if vdu_id:
3424                     vdur = next((x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None)
3425                     desc_params = self._format_additional_params(vdur.get("additionalParams"))
3426                 elif kdu_name:
3427                     kdur = next((x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None)
3428                     desc_params = self._format_additional_params(kdur.get("additionalParams"))
3429                 else:
3430                     desc_params = self._format_additional_params(db_vnfr.get("additionalParamsForVnf"))
3431             else:
3432                 desc_params = self._format_additional_params(db_nsr.get("additionalParamsForNs"))
3433
3434             if kdu_name:
3435                 kdu_action = True if not deep_get(kdu, ("kdu-configuration", "juju")) else False
3436
3437             # TODO check if ns is in a proper status
3438             if kdu_name and (primitive in ("upgrade", "rollback", "status") or kdu_action):
3439                 # kdur and desc_params already set from before
3440                 if primitive_params:
3441                     desc_params.update(primitive_params)
3442                 # TODO Check if we will need something at vnf level
3443                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
3444                     if kdu_name == kdu["kdu-name"] and kdu["member-vnf-index"] == vnf_index:
3445                         break
3446                 else:
3447                     raise LcmException("KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index))
3448
3449                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
3450                     msg = "unknown k8scluster-type '{}'".format(kdu.get("k8scluster-type"))
3451                     raise LcmException(msg)
3452
3453                 db_dict = {"collection": "nsrs",
3454                            "filter": {"_id": nsr_id},
3455                            "path": "_admin.deployed.K8s.{}".format(index)}
3456                 self.logger.debug(logging_text + "Exec k8s {} on {}.{}".format(primitive, vnf_index, kdu_name))
3457                 step = "Executing kdu {}".format(primitive)
3458                 if primitive == "upgrade":
3459                     if desc_params.get("kdu_model"):
3460                         kdu_model = desc_params.get("kdu_model")
3461                         del desc_params["kdu_model"]
3462                     else:
3463                         kdu_model = kdu.get("kdu-model")
3464                         parts = kdu_model.split(sep=":")
3465                         if len(parts) == 2:
3466                             kdu_model = parts[0]
3467
3468                     detailed_status = await asyncio.wait_for(
3469                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
3470                             cluster_uuid=kdu.get("k8scluster-uuid"),
3471                             kdu_instance=kdu.get("kdu-instance"),
3472                             atomic=True, kdu_model=kdu_model,
3473                             params=desc_params, db_dict=db_dict,
3474                             timeout=timeout_ns_action),
3475                         timeout=timeout_ns_action + 10)
3476                     self.logger.debug(logging_text + " Upgrade of kdu {} done".format(detailed_status))
3477                 elif primitive == "rollback":
3478                     detailed_status = await asyncio.wait_for(
3479                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
3480                             cluster_uuid=kdu.get("k8scluster-uuid"),
3481                             kdu_instance=kdu.get("kdu-instance"),
3482                             db_dict=db_dict),
3483                         timeout=timeout_ns_action)
3484                 elif primitive == "status":
3485                     detailed_status = await asyncio.wait_for(
3486                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
3487                             cluster_uuid=kdu.get("k8scluster-uuid"),
3488                             kdu_instance=kdu.get("kdu-instance")),
3489                         timeout=timeout_ns_action)
3490                 else:
3491                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(kdu["kdu-name"], nsr_id)
3492                     params = self._map_primitive_params(config_primitive_desc, primitive_params, desc_params)
3493
3494                     detailed_status = await asyncio.wait_for(
3495                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
3496                             cluster_uuid=kdu.get("k8scluster-uuid"),
3497                             kdu_instance=kdu_instance,
3498                             primitive_name=primitive,
3499                             params=params, db_dict=db_dict,
3500                             timeout=timeout_ns_action),
3501                         timeout=timeout_ns_action)
3502
3503                 if detailed_status:
3504                     nslcmop_operation_state = 'COMPLETED'
3505                 else:
3506                     detailed_status = ''
3507                     nslcmop_operation_state = 'FAILED'
3508             else:
3509                 nslcmop_operation_state, detailed_status = await self._ns_execute_primitive(
3510                     self._look_for_deployed_vca(nsr_deployed["VCA"],
3511                                                 member_vnf_index=vnf_index,
3512                                                 vdu_id=vdu_id,
3513                                                 vdu_count_index=vdu_count_index),
3514                     primitive=primitive,
3515                     primitive_params=self._map_primitive_params(config_primitive_desc, primitive_params, desc_params),
3516                     timeout=timeout_ns_action)
3517
3518             db_nslcmop_update["detailed-status"] = detailed_status
3519             error_description_nslcmop = detailed_status if nslcmop_operation_state == "FAILED" else ""
3520             self.logger.debug(logging_text + " task Done with result {} {}".format(nslcmop_operation_state,
3521                                                                                    detailed_status))
3522             return  # database update is called inside finally
3523
3524         except (DbException, LcmException, N2VCException, K8sException) as e:
3525             self.logger.error(logging_text + "Exit Exception {}".format(e))
3526             exc = e
3527         except asyncio.CancelledError:
3528             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
3529             exc = "Operation was cancelled"
3530         except asyncio.TimeoutError:
3531             self.logger.error(logging_text + "Timeout while '{}'".format(step))
3532             exc = "Timeout"
3533         except Exception as e:
3534             exc = traceback.format_exc()
3535             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
3536         finally:
3537             if exc:
3538                 db_nslcmop_update["detailed-status"] = detailed_status = error_description_nslcmop = \
3539                     "FAILED {}: {}".format(step, exc)
3540                 nslcmop_operation_state = "FAILED"
3541             if db_nsr:
3542                 self._write_ns_status(
3543                     nsr_id=nsr_id,
3544                     ns_state=db_nsr["nsState"],   # TODO check if degraded. For the moment use previous status
3545                     current_operation="IDLE",
3546                     current_operation_id=None,
3547                     # error_description=error_description_nsr,
3548                     # error_detail=error_detail,
3549                     other_update=db_nsr_update
3550                 )
3551
3552             self._write_op_status(
3553                 op_id=nslcmop_id,
3554                 stage="",
3555                 error_message=error_description_nslcmop,
3556                 operation_state=nslcmop_operation_state,
3557                 other_update=db_nslcmop_update,
3558             )
3559
3560             if nslcmop_operation_state:
3561                 try:
3562                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
3563                                                                "operationState": nslcmop_operation_state},
3564                                             loop=self.loop)
3565                 except Exception as e:
3566                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
3567             self.logger.debug(logging_text + "Exit")
3568             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
3569             return nslcmop_operation_state, detailed_status
3570
3571     async def scale(self, nsr_id, nslcmop_id):
3572
3573         # Try to lock HA task here
3574         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
3575         if not task_is_locked_by_me:
3576             return
3577
3578         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
3579         self.logger.debug(logging_text + "Enter")
3580         # get all needed from database
3581         db_nsr = None
3582         db_nslcmop = None
3583         db_nslcmop_update = {}
3584         nslcmop_operation_state = None
3585         db_nsr_update = {}
3586         exc = None
3587         # in case of error, indicates what part of scale was failed to put nsr at error status
3588         scale_process = None
3589         old_operational_status = ""
3590         old_config_status = ""
3591         vnfr_scaled = False
3592         try:
3593             # wait for any previous tasks in process
3594             step = "Waiting for previous operations to terminate"
3595             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
3596
3597             self._write_ns_status(
3598                 nsr_id=nsr_id,
3599                 ns_state=None,
3600                 current_operation="SCALING",
3601                 current_operation_id=nslcmop_id
3602             )
3603
3604             step = "Getting nslcmop from database"
3605             self.logger.debug(step + " after having waited for previous tasks to be completed")
3606             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
3607             step = "Getting nsr from database"
3608             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3609
3610             old_operational_status = db_nsr["operational-status"]
3611             old_config_status = db_nsr["config-status"]
3612             step = "Parsing scaling parameters"
3613             # self.logger.debug(step)
3614             db_nsr_update["operational-status"] = "scaling"
3615             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3616             nsr_deployed = db_nsr["_admin"].get("deployed")
3617
3618             #######
3619             nsr_deployed = db_nsr["_admin"].get("deployed")
3620             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
3621             # vdu_id = db_nslcmop["operationParams"].get("vdu_id")
3622             # vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
3623             # vdu_name = db_nslcmop["operationParams"].get("vdu_name")
3624             #######
3625
3626             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
3627             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
3628             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
3629             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
3630             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
3631
3632             # for backward compatibility
3633             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
3634                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
3635                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
3636                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3637
3638             step = "Getting vnfr from database"
3639             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
3640             step = "Getting vnfd from database"
3641             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
3642
3643             step = "Getting scaling-group-descriptor"
3644             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
3645                 if scaling_descriptor["name"] == scaling_group:
3646                     break
3647             else:
3648                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
3649                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
3650
3651             # cooldown_time = 0
3652             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
3653             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
3654             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
3655             #         break
3656
3657             # TODO check if ns is in a proper status
3658             step = "Sending scale order to VIM"
3659             nb_scale_op = 0
3660             if not db_nsr["_admin"].get("scaling-group"):
3661                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
3662                 admin_scale_index = 0
3663             else:
3664                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
3665                     if admin_scale_info["name"] == scaling_group:
3666                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
3667                         break
3668                 else:  # not found, set index one plus last element and add new entry with the name
3669                     admin_scale_index += 1
3670                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
3671             RO_scaling_info = []
3672             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
3673             if scaling_type == "SCALE_OUT":
3674                 # count if max-instance-count is reached
3675                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
3676                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
3677                 if nb_scale_op >= max_instance_count:
3678                     raise LcmException("reached the limit of {} (max-instance-count) "
3679                                        "scaling-out operations for the "
3680                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3681
3682                 nb_scale_op += 1
3683                 vdu_scaling_info["scaling_direction"] = "OUT"
3684                 vdu_scaling_info["vdu-create"] = {}
3685                 for vdu_scale_info in scaling_descriptor["vdu"]:
3686                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3687                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
3688                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3689
3690             elif scaling_type == "SCALE_IN":
3691                 # count if min-instance-count is reached
3692                 min_instance_count = 0
3693                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
3694                     min_instance_count = int(scaling_descriptor["min-instance-count"])
3695                 if nb_scale_op <= min_instance_count:
3696                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
3697                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
3698                 nb_scale_op -= 1
3699                 vdu_scaling_info["scaling_direction"] = "IN"
3700                 vdu_scaling_info["vdu-delete"] = {}
3701                 for vdu_scale_info in scaling_descriptor["vdu"]:
3702                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
3703                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
3704                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
3705
3706             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
3707             vdu_create = vdu_scaling_info.get("vdu-create")
3708             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
3709             if vdu_scaling_info["scaling_direction"] == "IN":
3710                 for vdur in reversed(db_vnfr["vdur"]):
3711                     if vdu_delete.get(vdur["vdu-id-ref"]):
3712                         vdu_delete[vdur["vdu-id-ref"]] -= 1
3713                         vdu_scaling_info["vdu"].append({
3714                             "name": vdur["name"],
3715                             "vdu_id": vdur["vdu-id-ref"],
3716                             "interface": []
3717                         })
3718                         for interface in vdur["interfaces"]:
3719                             vdu_scaling_info["vdu"][-1]["interface"].append({
3720                                 "name": interface["name"],
3721                                 "ip_address": interface["ip-address"],
3722                                 "mac_address": interface.get("mac-address"),
3723                             })
3724                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
3725
3726             # PRE-SCALE BEGIN
3727             step = "Executing pre-scale vnf-config-primitive"
3728             if scaling_descriptor.get("scaling-config-action"):
3729                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
3730                     if (scaling_config_action.get("trigger") == "pre-scale-in" and scaling_type == "SCALE_IN") \
3731                        or (scaling_config_action.get("trigger") == "pre-scale-out" and scaling_type == "SCALE_OUT"):
3732                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
3733                         step = db_nslcmop_update["detailed-status"] = \
3734                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
3735
3736                         # look for primitive
3737                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
3738                             if config_primitive["name"] == vnf_config_primitive:
3739                                 break
3740                         else:
3741                             raise LcmException(
3742                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
3743                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
3744                                 "primitive".format(scaling_group, config_primitive))
3745
3746                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
3747                         if db_vnfr.get("additionalParamsForVnf"):
3748                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
3749
3750                         scale_process = "VCA"
3751                         db_nsr_update["config-status"] = "configuring pre-scaling"
3752                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
3753
3754                         # Pre-scale retry check: Check if this sub-operation has been executed before
3755                         op_index = self._check_or_add_scale_suboperation(
3756                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
3757                         if op_index == self.SUBOPERATION_STATUS_SKIP:
3758                             # Skip sub-operation
3759                             result = 'COMPLETED'
3760                             result_detail = 'Done'
3761                             self.logger.debug(logging_text +
3762                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
3763                                                   vnf_config_primitive, result, result_detail))
3764                         else:
3765                             if op_index == self.SUBOPERATION_STATUS_NEW:
3766                                 # New sub-operation: Get index of this sub-operation
3767                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3768                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
3769                                                   format(vnf_config_primitive))
3770                             else:
3771                                 # retry:  Get registered params for this existing sub-operation
3772                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3773                                 vnf_index = op.get('member_vnf_index')
3774                                 vnf_config_primitive = op.get('primitive')
3775                                 primitive_params = op.get('primitive_params')
3776                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
3777                                                   format(vnf_config_primitive))
3778                             # Execute the primitive, either with new (first-time) or registered (retry) args
3779                             result, result_detail = await self._ns_execute_primitive(
3780                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
3781                                                             member_vnf_index=vnf_index,
3782                                                             vdu_id=None,
3783                                                             vdu_count_index=None),
3784                                 vnf_config_primitive, primitive_params)
3785                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
3786                                 vnf_config_primitive, result, result_detail))
3787                             # Update operationState = COMPLETED | FAILED
3788                             self._update_suboperation_status(
3789                                 db_nslcmop, op_index, result, result_detail)
3790
3791                         if result == "FAILED":
3792                             raise LcmException(result_detail)
3793                         db_nsr_update["config-status"] = old_config_status
3794                         scale_process = None
3795             # PRE-SCALE END
3796
3797             # SCALE RO - BEGIN
3798             # Should this block be skipped if 'RO_nsr_id' == None ?
3799             # if (RO_nsr_id and RO_scaling_info):
3800             if RO_scaling_info:
3801                 scale_process = "RO"
3802                 # Scale RO retry check: Check if this sub-operation has been executed before
3803                 op_index = self._check_or_add_scale_suboperation(
3804                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
3805                 if op_index == self.SUBOPERATION_STATUS_SKIP:
3806                     # Skip sub-operation
3807                     result = 'COMPLETED'
3808                     result_detail = 'Done'
3809                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
3810                         result, result_detail))
3811                 else:
3812                     if op_index == self.SUBOPERATION_STATUS_NEW:
3813                         # New sub-operation: Get index of this sub-operation
3814                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3815                         self.logger.debug(logging_text + "New sub-operation RO")
3816                     else:
3817                         # retry:  Get registered params for this existing sub-operation
3818                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3819                         RO_nsr_id = op.get('RO_nsr_id')
3820                         RO_scaling_info = op.get('RO_scaling_info')
3821                         self.logger.debug(logging_text + "Sub-operation RO retry for primitive {}".format(
3822                             vnf_config_primitive))
3823
3824                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
3825                     db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
3826                     db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
3827                     # wait until ready
3828                     RO_nslcmop_id = RO_desc["instance_action_id"]
3829                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
3830
3831                     RO_task_done = False
3832                     step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
3833                     detailed_status_old = None
3834                     self.logger.debug(logging_text + step)
3835
3836                     deployment_timeout = 1 * 3600   # One hour
3837                     while deployment_timeout > 0:
3838                         if not RO_task_done:
3839                             desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
3840                                                       extra_item_id=RO_nslcmop_id)
3841
3842                             # deploymentStatus
3843                             self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
3844
3845                             ns_status, ns_status_info = self.RO.check_action_status(desc)
3846                             if ns_status == "ERROR":
3847                                 raise ROclient.ROClientException(ns_status_info)
3848                             elif ns_status == "BUILD":
3849                                 detailed_status = step + "; {}".format(ns_status_info)
3850                             elif ns_status == "ACTIVE":
3851                                 RO_task_done = True
3852                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
3853                                 self.logger.debug(logging_text + step)
3854                             else:
3855                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
3856                         else:
3857
3858                             if ns_status == "ERROR":
3859                                 raise ROclient.ROClientException(ns_status_info)
3860                             elif ns_status == "BUILD":
3861                                 detailed_status = step + "; {}".format(ns_status_info)
3862                             elif ns_status == "ACTIVE":
3863                                 step = detailed_status = \
3864                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
3865                                 if not vnfr_scaled:
3866                                     self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
3867                                     vnfr_scaled = True
3868                                 try:
3869                                     desc = await self.RO.show("ns", RO_nsr_id)
3870
3871                                     # deploymentStatus
3872                                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
3873
3874                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
3875                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
3876                                     break
3877                                 except LcmExceptionNoMgmtIP:
3878                                     pass
3879                             else:
3880                                 assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
3881                         if detailed_status != detailed_status_old:
3882                             self._update_suboperation_status(
3883                                 db_nslcmop, op_index, 'COMPLETED', detailed_status)
3884                             detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
3885                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
3886
3887                         await asyncio.sleep(5, loop=self.loop)
3888                         deployment_timeout -= 5
3889                     if deployment_timeout <= 0:
3890                         self._update_suboperation_status(
3891                             db_nslcmop, nslcmop_id, op_index, 'FAILED', "Timeout when waiting for ns to get ready")
3892                         raise ROclient.ROClientException("Timeout waiting ns to be ready")
3893
3894                     # update VDU_SCALING_INFO with the obtained ip_addresses
3895                     if vdu_scaling_info["scaling_direction"] == "OUT":
3896                         for vdur in reversed(db_vnfr["vdur"]):
3897                             if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
3898                                 vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
3899                                 vdu_scaling_info["vdu"].append({
3900                                     "name": vdur["name"],
3901                                     "vdu_id": vdur["vdu-id-ref"],
3902                                     "interface": []
3903                                 })
3904                                 for interface in vdur["interfaces"]:
3905                                     vdu_scaling_info["vdu"][-1]["interface"].append({
3906                                         "name": interface["name"],
3907                                         "ip_address": interface["ip-address"],
3908                                         "mac_address": interface.get("mac-address"),
3909                                     })
3910                         del vdu_scaling_info["vdu-create"]
3911
3912                     self._update_suboperation_status(db_nslcmop, op_index, 'COMPLETED', 'Done')
3913             # SCALE RO - END
3914
3915             scale_process = None
3916             if db_nsr_update:
3917                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3918
3919             # POST-SCALE BEGIN
3920             # execute primitive service POST-SCALING
3921             step = "Executing post-scale vnf-config-primitive"
3922             if scaling_descriptor.get("scaling-config-action"):
3923                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
3924                     if (scaling_config_action.get("trigger") == "post-scale-in" and scaling_type == "SCALE_IN") \
3925                        or (scaling_config_action.get("trigger") == "post-scale-out" and scaling_type == "SCALE_OUT"):
3926                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
3927                         step = db_nslcmop_update["detailed-status"] = \
3928                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
3929
3930                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
3931                         if db_vnfr.get("additionalParamsForVnf"):
3932                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
3933
3934                         # look for primitive
3935                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
3936                             if config_primitive["name"] == vnf_config_primitive:
3937                                 break
3938                         else:
3939                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
3940                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
3941                                                "match any vnf-configuration:config-primitive".format(scaling_group,
3942                                                                                                      config_primitive))
3943                         scale_process = "VCA"
3944                         db_nsr_update["config-status"] = "configuring post-scaling"
3945                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
3946
3947                         # Post-scale retry check: Check if this sub-operation has been executed before
3948                         op_index = self._check_or_add_scale_suboperation(
3949                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
3950                         if op_index == self.SUBOPERATION_STATUS_SKIP:
3951                             # Skip sub-operation
3952                             result = 'COMPLETED'
3953                             result_detail = 'Done'
3954                             self.logger.debug(logging_text +
3955                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".
3956                                               format(vnf_config_primitive, result, result_detail))
3957                         else:
3958                             if op_index == self.SUBOPERATION_STATUS_NEW:
3959                                 # New sub-operation: Get index of this sub-operation
3960                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
3961                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
3962                                                   format(vnf_config_primitive))
3963                             else:
3964                                 # retry:  Get registered params for this existing sub-operation
3965                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
3966                                 vnf_index = op.get('member_vnf_index')
3967                                 vnf_config_primitive = op.get('primitive')
3968                                 primitive_params = op.get('primitive_params')
3969                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation retry".
3970                                                   format(vnf_config_primitive))
3971                             # Execute the primitive, either with new (first-time) or registered (retry) args
3972                             result, result_detail = await self._ns_execute_primitive(
3973                                 self._look_for_deployed_vca(nsr_deployed["VCA"],
3974                                                             member_vnf_index=vnf_index,
3975                                                             vdu_id=None,
3976                                                             vdu_count_index=None),
3977                                 vnf_config_primitive, primitive_params)
3978                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
3979                                 vnf_config_primitive, result, result_detail))
3980                             # Update operationState = COMPLETED | FAILED
3981                             self._update_suboperation_status(
3982                                 db_nslcmop, op_index, result, result_detail)
3983
3984                         if result == "FAILED":
3985                             raise LcmException(result_detail)
3986                         db_nsr_update["config-status"] = old_config_status
3987                         scale_process = None
3988             # POST-SCALE END
3989
3990             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
3991             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
3992                 else old_operational_status
3993             db_nsr_update["config-status"] = old_config_status
3994             return
3995         except (ROclient.ROClientException, DbException, LcmException) as e:
3996             self.logger.error(logging_text + "Exit Exception {}".format(e))
3997             exc = e
3998         except asyncio.CancelledError:
3999             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
4000             exc = "Operation was cancelled"
4001         except Exception as e:
4002             exc = traceback.format_exc()
4003             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
4004         finally:
4005             self._write_ns_status(
4006                 nsr_id=nsr_id,
4007                 ns_state=None,
4008                 current_operation="IDLE",
4009                 current_operation_id=None
4010             )
4011             if exc:
4012                 db_nslcmop_update["detailed-status"] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
4013                 nslcmop_operation_state = "FAILED"
4014                 if db_nsr:
4015                     db_nsr_update["operational-status"] = old_operational_status
4016                     db_nsr_update["config-status"] = old_config_status
4017                     db_nsr_update["detailed-status"] = ""
4018                     if scale_process:
4019                         if "VCA" in scale_process:
4020                             db_nsr_update["config-status"] = "failed"
4021                         if "RO" in scale_process:
4022                             db_nsr_update["operational-status"] = "failed"
4023                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
4024                                                                                                      exc)
4025             else:
4026                 error_description_nslcmop = None
4027                 nslcmop_operation_state = "COMPLETED"
4028                 db_nslcmop_update["detailed-status"] = "Done"
4029
4030             self._write_op_status(
4031                 op_id=nslcmop_id,
4032                 stage="",
4033                 error_message=error_description_nslcmop,
4034                 operation_state=nslcmop_operation_state,
4035                 other_update=db_nslcmop_update,
4036             )
4037             if db_nsr:
4038                 self._write_ns_status(
4039                     nsr_id=nsr_id,
4040                     ns_state=None,
4041                     current_operation="IDLE",
4042                     current_operation_id=None,
4043                     other_update=db_nsr_update
4044                 )
4045
4046             if nslcmop_operation_state:
4047                 try:
4048                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
4049                                                              "operationState": nslcmop_operation_state},
4050                                             loop=self.loop)
4051                     # if cooldown_time:
4052                     #     await asyncio.sleep(cooldown_time, loop=self.loop)
4053                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
4054                 except Exception as e:
4055                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
4056             self.logger.debug(logging_text + "Exit")
4057             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")