osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Update target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 def deep_get(target_dict, key_list):
  71     """
  72     Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
  73     Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
  74     :param target_dict: dictionary to be read
  75     :param key_list: list of keys to read from  target_dict
  76     :return: The wanted value if exist, None otherwise
  77     """
  78     for key in key_list:
  79         if not isinstance(target_dict, dict) or key not in target_dict:
  80             return None
  81         target_dict = target_dict[key]
  82     return target_dict
  83
  84
  85 class NsLcm(LcmBase):
  86     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  87     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  88     timeout_charm_delete = 10 * 60
  89     timeout_primitive = 10 * 60  # timeout for primitive execution
  90
  91     SUBOPERATION_STATUS_NOT_FOUND = -1
  92     SUBOPERATION_STATUS_NEW = -2
  93     SUBOPERATION_STATUS_SKIP = -3
  94
  95     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  96         """
  97         Init, Connect to database, filesystem storage, and messaging
  98         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  99         :return: None
 100         """
 101         # logging
 102         self.logger = logging.getLogger('lcm.ns')
 103         self.loop = loop
 104         self.lcm_tasks = lcm_tasks
 105
 106         super().__init__(db, msg, fs, self.logger)
 107
 108         self.ro_config = ro_config
 109
 110         self.n2vc = N2VC(
 111             log=self.logger,
 112             server=vca_config['host'],
 113             port=vca_config['port'],
 114             user=vca_config['user'],
 115             secret=vca_config['secret'],
 116             # TODO: This should point to the base folder where charms are stored,
 117             # if there is a common one (like object storage). Otherwise, leave
 118             # it unset and pass it via DeployCharms
 119             # artifacts=vca_config[''],
 120             artifacts=None,
 121             juju_public_key=vca_config.get('pubkey'),
 122             ca_cert=vca_config.get('cacert'),
 123             api_proxy=vca_config.get('apiproxy')
 124         )
 125         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 126
 127     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 128         """
 129         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 130         :param vnfd: input vnfd
 131         :param new_id: overrides vnf id if provided
 132         :param additionalParams: Instantiation params for VNFs provided
 133         :param nsrId: Id of the NSR
 134         :return: copy of vnfd
 135         """
 136         try:
 137             vnfd_RO = deepcopy(vnfd)
 138             # remove unused by RO configuration, monitoring, scaling and internal keys
 139             vnfd_RO.pop("_id", None)
 140             vnfd_RO.pop("_admin", None)
 141             vnfd_RO.pop("vnf-configuration", None)
 142             vnfd_RO.pop("monitoring-param", None)
 143             vnfd_RO.pop("scaling-group-descriptor", None)
 144             if new_id:
 145                 vnfd_RO["id"] = new_id
 146
 147             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 148             for vdu in get_iterable(vnfd_RO, "vdu"):
 149                 cloud_init_file = None
 150                 if vdu.get("cloud-init-file"):
 151                     base_folder = vnfd["_admin"]["storage"]
 152                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 153                                                                    vdu["cloud-init-file"])
 154                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 155                         cloud_init_content = ci_file.read()
 156                     vdu.pop("cloud-init-file", None)
 157                 elif vdu.get("cloud-init"):
 158                     cloud_init_content = vdu["cloud-init"]
 159                 else:
 160                     continue
 161
 162                 env = Environment()
 163                 ast = env.parse(cloud_init_content)
 164                 mandatory_vars = meta.find_undeclared_variables(ast)
 165                 if mandatory_vars:
 166                     for var in mandatory_vars:
 167                         if not additionalParams or var not in additionalParams.keys():
 168                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 169                                                "file, must be provided in the instantiation parameters inside the "
 170                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 171                 template = Template(cloud_init_content)
 172                 cloud_init_content = template.render(additionalParams or {})
 173                 vdu["cloud-init"] = cloud_init_content
 174
 175             return vnfd_RO
 176         except FsException as e:
 177             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 178                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 179         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 180             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 181                                format(vnfd["id"], vdu["id"], e))
 182
 183     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 184         """
 185         Callback both for charm status change and task completion
 186         :param model_name: Charm model name
 187         :param application_name: Charm application name
 188         :param status: Can be
 189             - blocked: The unit needs manual intervention
 190             - maintenance: The unit is actively deploying/configuring
 191             - waiting: The unit is waiting for another charm to be ready
 192             - active: The unit is deployed, configured, and ready
 193             - error: The charm has failed and needs attention.
 194             - terminated: The charm has been destroyed
 195             - removing,
 196             - removed
 197         :param message: detailed message error
 198         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 199             nsr_id:
 200             nslcmop_id:
 201             lcmOperationType: currently "instantiate"
 202             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 203             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 204             n2vc_event: event used to notify instantiation task that some change has been produced
 205         :param task: None for charm status change, or task for completion task callback
 206         :return:
 207         """
 208         try:
 209             nsr_id = n2vc_info["nsr_id"]
 210             deployed = n2vc_info["deployed"]
 211             db_nsr_update = n2vc_info["db_update"]
 212             nslcmop_id = n2vc_info["nslcmop_id"]
 213             ns_operation = n2vc_info["lcmOperationType"]
 214             n2vc_event = n2vc_info["n2vc_event"]
 215             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 216                                                                                     application_name)
 217             for vca_index, vca_deployed in enumerate(deployed):
 218                 if not vca_deployed:
 219                     continue
 220                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 221                     break
 222             else:
 223                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 224                                   format(model_name))
 225                 return
 226             if task:
 227                 if task.cancelled():
 228                     self.logger.debug(logging_text + " task Cancelled")
 229                     vca_deployed['operational-status'] = "error"
 230                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 231                     vca_deployed['detailed-status'] = "Task Cancelled"
 232                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 233
 234                 elif task.done():
 235                     exc = task.exception()
 236                     if exc:
 237                         self.logger.error(logging_text + " task Exception={}".format(exc))
 238                         vca_deployed['operational-status'] = "error"
 239                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 240                         vca_deployed['detailed-status'] = str(exc)
 241                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 242                     else:
 243                         self.logger.debug(logging_text + " task Done")
 244                         # task is Done, but callback is still ongoing. So ignore
 245                         return
 246             elif status:
 247                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 248                 if vca_deployed['operational-status'] == status:
 249                     return  # same status, ignore
 250                 vca_deployed['operational-status'] = status
 251                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 252                 vca_deployed['detailed-status'] = str(message)
 253                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 254             else:
 255                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 256                 return
 257             # wake up instantiate task
 258             n2vc_event.set()
 259         except Exception as e:
 260             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 261
 262     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 263         """
 264         Creates a RO ns descriptor from OSM ns_instantiate params
 265         :param ns_params: OSM instantiate params
 266         :return: The RO ns descriptor
 267         """
 268         vim_2_RO = {}
 269         wim_2_RO = {}
 270         # TODO feature 1417: Check that no instantiation is set over PDU
 271         # check if PDU forces a concrete vim-network-id and add it
 272         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 273
 274         def vim_account_2_RO(vim_account):
 275             if vim_account in vim_2_RO:
 276                 return vim_2_RO[vim_account]
 277
 278             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 279             if db_vim["_admin"]["operationalState"] != "ENABLED":
 280                 raise LcmException("VIM={} is not available. operationalState={}".format(
 281                     vim_account, db_vim["_admin"]["operationalState"]))
 282             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 283             vim_2_RO[vim_account] = RO_vim_id
 284             return RO_vim_id
 285
 286         def wim_account_2_RO(wim_account):
 287             if isinstance(wim_account, str):
 288                 if wim_account in wim_2_RO:
 289                     return wim_2_RO[wim_account]
 290
 291                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 292                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 293                     raise LcmException("WIM={} is not available. operationalState={}".format(
 294                         wim_account, db_wim["_admin"]["operationalState"]))
 295                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 296                 wim_2_RO[wim_account] = RO_wim_id
 297                 return RO_wim_id
 298             else:
 299                 return wim_account
 300
 301         def ip_profile_2_RO(ip_profile):
 302             RO_ip_profile = deepcopy((ip_profile))
 303             if "dns-server" in RO_ip_profile:
 304                 if isinstance(RO_ip_profile["dns-server"], list):
 305                     RO_ip_profile["dns-address"] = []
 306                     for ds in RO_ip_profile.pop("dns-server"):
 307                         RO_ip_profile["dns-address"].append(ds['address'])
 308                 else:
 309                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 310             if RO_ip_profile.get("ip-version") == "ipv4":
 311                 RO_ip_profile["ip-version"] = "IPv4"
 312             if RO_ip_profile.get("ip-version") == "ipv6":
 313                 RO_ip_profile["ip-version"] = "IPv6"
 314             if "dhcp-params" in RO_ip_profile:
 315                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 316             return RO_ip_profile
 317
 318         if not ns_params:
 319             return None
 320         RO_ns_params = {
 321             # "name": ns_params["nsName"],
 322             # "description": ns_params.get("nsDescription"),
 323             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 324             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 325             # "scenario": ns_params["nsdId"],
 326         }
 327         n2vc_key_list = n2vc_key_list or []
 328         for vnfd_ref, vnfd in vnfd_dict.items():
 329             vdu_needed_access = []
 330             mgmt_cp = None
 331             if vnfd.get("vnf-configuration"):
 332                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 333                 if ssh_required and vnfd.get("mgmt-interface"):
 334                     if vnfd["mgmt-interface"].get("vdu-id"):
 335                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 336                     elif vnfd["mgmt-interface"].get("cp"):
 337                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 338
 339             for vdu in vnfd.get("vdu", ()):
 340                 if vdu.get("vdu-configuration"):
 341                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 342                     if ssh_required:
 343                         vdu_needed_access.append(vdu["id"])
 344                 elif mgmt_cp:
 345                     for vdu_interface in vdu.get("interface"):
 346                         if vdu_interface.get("external-connection-point-ref") and \
 347                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 348                             vdu_needed_access.append(vdu["id"])
 349                             mgmt_cp = None
 350                             break
 351
 352             if vdu_needed_access:
 353                 for vnf_member in nsd.get("constituent-vnfd"):
 354                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 355                         continue
 356                     for vdu in vdu_needed_access:
 357                         populate_dict(RO_ns_params,
 358                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 359                                       n2vc_key_list)
 360
 361         if ns_params.get("vduImage"):
 362             RO_ns_params["vduImage"] = ns_params["vduImage"]
 363
 364         if ns_params.get("ssh_keys"):
 365             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 366         for vnf_params in get_iterable(ns_params, "vnf"):
 367             for constituent_vnfd in nsd["constituent-vnfd"]:
 368                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 369                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 370                     break
 371             else:
 372                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 373                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 374             if vnf_params.get("vimAccountId"):
 375                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 376                               vim_account_2_RO(vnf_params["vimAccountId"]))
 377
 378             for vdu_params in get_iterable(vnf_params, "vdu"):
 379                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 380                 if vdu_params.get("volume"):
 381                     for volume_params in vdu_params["volume"]:
 382                         if volume_params.get("vim-volume-id"):
 383                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 384                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 385                                           volume_params["vim-volume-id"])
 386                 if vdu_params.get("interface"):
 387                     for interface_params in vdu_params["interface"]:
 388                         if interface_params.get("ip-address"):
 389                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 390                                                          vdu_params["id"], "interfaces", interface_params["name"],
 391                                                          "ip_address"),
 392                                           interface_params["ip-address"])
 393                         if interface_params.get("mac-address"):
 394                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 395                                                          vdu_params["id"], "interfaces", interface_params["name"],
 396                                                          "mac_address"),
 397                                           interface_params["mac-address"])
 398                         if interface_params.get("floating-ip-required"):
 399                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 400                                                          vdu_params["id"], "interfaces", interface_params["name"],
 401                                                          "floating-ip"),
 402                                           interface_params["floating-ip-required"])
 403
 404             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 405                 if internal_vld_params.get("vim-network-name"):
 406                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 407                                                  internal_vld_params["name"], "vim-network-name"),
 408                                   internal_vld_params["vim-network-name"])
 409                 if internal_vld_params.get("vim-network-id"):
 410                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 411                                                  internal_vld_params["name"], "vim-network-id"),
 412                                   internal_vld_params["vim-network-id"])
 413                 if internal_vld_params.get("ip-profile"):
 414                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 415                                                  internal_vld_params["name"], "ip-profile"),
 416                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 417                 if internal_vld_params.get("provider-network"):
 418
 419                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 420                                                  internal_vld_params["name"], "provider-network"),
 421                                   internal_vld_params["provider-network"].copy())
 422
 423                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 424                     # look for interface
 425                     iface_found = False
 426                     for vdu_descriptor in vnf_descriptor["vdu"]:
 427                         for vdu_interface in vdu_descriptor["interface"]:
 428                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 429                                 if icp_params.get("ip-address"):
 430                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 431                                                                  vdu_descriptor["id"], "interfaces",
 432                                                                  vdu_interface["name"], "ip_address"),
 433                                                   icp_params["ip-address"])
 434
 435                                 if icp_params.get("mac-address"):
 436                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 437                                                                  vdu_descriptor["id"], "interfaces",
 438                                                                  vdu_interface["name"], "mac_address"),
 439                                                   icp_params["mac-address"])
 440                                 iface_found = True
 441                                 break
 442                         if iface_found:
 443                             break
 444                     else:
 445                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 446                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 447                                            "connection-point".format(vnf_params["member-vnf-index"],
 448                                                                      icp_params["id-ref"]))
 449
 450         for vld_params in get_iterable(ns_params, "vld"):
 451             if "ip-profile" in vld_params:
 452                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 453                               ip_profile_2_RO(vld_params["ip-profile"]))
 454
 455             if vld_params.get("provider-network"):
 456
 457                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "provider-network"),
 458                               vld_params["provider-network"].copy())
 459
 460             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 461                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 462                               wim_account_2_RO(vld_params["wimAccountId"])),
 463             if vld_params.get("vim-network-name"):
 464                 RO_vld_sites = []
 465                 if isinstance(vld_params["vim-network-name"], dict):
 466                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 467                         RO_vld_sites.append({
 468                             "netmap-use": vim_net,
 469                             "datacenter": vim_account_2_RO(vim_account)
 470                         })
 471                 else:  # isinstance str
 472                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 473                 if RO_vld_sites:
 474                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 475
 476             if vld_params.get("vim-network-id"):
 477                 RO_vld_sites = []
 478                 if isinstance(vld_params["vim-network-id"], dict):
 479                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 480                         RO_vld_sites.append({
 481                             "netmap-use": vim_net,
 482                             "datacenter": vim_account_2_RO(vim_account)
 483                         })
 484                 else:  # isinstance str
 485                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 486                 if RO_vld_sites:
 487                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 488             if vld_params.get("ns-net"):
 489                 if isinstance(vld_params["ns-net"], dict):
 490                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 491                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 492                 if RO_vld_ns_net:
 493                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 494             if "vnfd-connection-point-ref" in vld_params:
 495                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 496                     # look for interface
 497                     for constituent_vnfd in nsd["constituent-vnfd"]:
 498                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 499                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 500                             break
 501                     else:
 502                         raise LcmException(
 503                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 504                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 505                     match_cp = False
 506                     for vdu_descriptor in vnf_descriptor["vdu"]:
 507                         for interface_descriptor in vdu_descriptor["interface"]:
 508                             if interface_descriptor.get("external-connection-point-ref") == \
 509                                     cp_params["vnfd-connection-point-ref"]:
 510                                 match_cp = True
 511                                 break
 512                         if match_cp:
 513                             break
 514                     else:
 515                         raise LcmException(
 516                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 517                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 518                                 cp_params["member-vnf-index-ref"],
 519                                 cp_params["vnfd-connection-point-ref"],
 520                                 vnf_descriptor["id"]))
 521                     if cp_params.get("ip-address"):
 522                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 523                                                      vdu_descriptor["id"], "interfaces",
 524                                                      interface_descriptor["name"], "ip_address"),
 525                                       cp_params["ip-address"])
 526                     if cp_params.get("mac-address"):
 527                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 528                                                      vdu_descriptor["id"], "interfaces",
 529                                                      interface_descriptor["name"], "mac_address"),
 530                                       cp_params["mac-address"])
 531         return RO_ns_params
 532
 533     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 534         # make a copy to do not change
 535         vdu_create = copy(vdu_create)
 536         vdu_delete = copy(vdu_delete)
 537
 538         vdurs = db_vnfr.get("vdur")
 539         if vdurs is None:
 540             vdurs = []
 541         vdu_index = len(vdurs)
 542         while vdu_index:
 543             vdu_index -= 1
 544             vdur = vdurs[vdu_index]
 545             if vdur.get("pdu-type"):
 546                 continue
 547             vdu_id_ref = vdur["vdu-id-ref"]
 548             if vdu_create and vdu_create.get(vdu_id_ref):
 549                 for index in range(0, vdu_create[vdu_id_ref]):
 550                     vdur = deepcopy(vdur)
 551                     vdur["_id"] = str(uuid4())
 552                     vdur["count-index"] += 1
 553                     vdurs.insert(vdu_index+1+index, vdur)
 554                 del vdu_create[vdu_id_ref]
 555             if vdu_delete and vdu_delete.get(vdu_id_ref):
 556                 del vdurs[vdu_index]
 557                 vdu_delete[vdu_id_ref] -= 1
 558                 if not vdu_delete[vdu_id_ref]:
 559                     del vdu_delete[vdu_id_ref]
 560         # check all operations are done
 561         if vdu_create or vdu_delete:
 562             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 563                 vdu_create))
 564         if vdu_delete:
 565             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 566                 vdu_delete))
 567
 568         vnfr_update = {"vdur": vdurs}
 569         db_vnfr["vdur"] = vdurs
 570         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 571
 572     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 573         """
 574         Updates database nsr with the RO info for the created vld
 575         :param ns_update_nsr: dictionary to be filled with the updated info
 576         :param db_nsr: content of db_nsr. This is also modified
 577         :param nsr_desc_RO: nsr descriptor from RO
 578         :return: Nothing, LcmException is raised on errors
 579         """
 580
 581         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 582             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 583                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 584                     continue
 585                 vld["vim-id"] = net_RO.get("vim_net_id")
 586                 vld["name"] = net_RO.get("vim_name")
 587                 vld["status"] = net_RO.get("status")
 588                 vld["status-detailed"] = net_RO.get("error_msg")
 589                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 590                 break
 591             else:
 592                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 593
 594     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 595         """
 596         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 597         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 598         :param nsr_desc_RO: nsr descriptor from RO
 599         :return: Nothing, LcmException is raised on errors
 600         """
 601         for vnf_index, db_vnfr in db_vnfrs.items():
 602             for vnf_RO in nsr_desc_RO["vnfs"]:
 603                 if vnf_RO["member_vnf_index"] != vnf_index:
 604                     continue
 605                 vnfr_update = {}
 606                 if vnf_RO.get("ip_address"):
 607                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 608                 elif not db_vnfr.get("ip-address"):
 609                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 610
 611                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 612                     vdur_RO_count_index = 0
 613                     if vdur.get("pdu-type"):
 614                         continue
 615                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 616                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 617                             continue
 618                         if vdur["count-index"] != vdur_RO_count_index:
 619                             vdur_RO_count_index += 1
 620                             continue
 621                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 622                         if vdur_RO.get("ip_address"):
 623                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 624                         else:
 625                             vdur["ip-address"] = None
 626                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 627                         vdur["name"] = vdur_RO.get("vim_name")
 628                         vdur["status"] = vdur_RO.get("status")
 629                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 630                         for ifacer in get_iterable(vdur, "interfaces"):
 631                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 632                                 if ifacer["name"] == interface_RO.get("internal_name"):
 633                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 634                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 635                                     break
 636                             else:
 637                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 638                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 639                                                                           ifacer["name"]))
 640                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 641                         break
 642                     else:
 643                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 644                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 645
 646                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 647                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 648                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 649                             continue
 650                         vld["vim-id"] = net_RO.get("vim_net_id")
 651                         vld["name"] = net_RO.get("vim_name")
 652                         vld["status"] = net_RO.get("status")
 653                         vld["status-detailed"] = net_RO.get("error_msg")
 654                         vnfr_update["vld.{}".format(vld_index)] = vld
 655                         break
 656                     else:
 657                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 658                             vnf_index, vld["id"]))
 659
 660                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 661                 break
 662
 663             else:
 664                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 665
 666     @staticmethod
 667     def _get_ns_config_info(vca_deployed_list):
 668         """
 669         Generates a mapping between vnf,vdu elements and the N2VC id
 670         :param vca_deployed_list: List of database _admin.deploy.VCA that contains this list
 671         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 672             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 673             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 674         """
 675         mapping = {}
 676         ns_config_info = {"osm-config-mapping": mapping}
 677         for vca in vca_deployed_list:
 678             if not vca["member-vnf-index"]:
 679                 continue
 680             if not vca["vdu_id"]:
 681                 mapping[vca["member-vnf-index"]] = vca["application"]
 682             else:
 683                 mapping["{}.{}.{}".format(vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"])] =\
 684                     vca["application"]
 685         return ns_config_info
 686
 687     @staticmethod
 688     def _get_initial_config_primitive_list(desc_primitive_list, vca_deployed):
 689         """
 690         Generates a list of initial-config-primitive based on the list provided by the descriptor. It includes internal
 691         primitives as verify-ssh-credentials, or config when needed
 692         :param desc_primitive_list: information of the descriptor
 693         :param vca_deployed: information of the deployed, needed for known if it is related to an NS, VNF, VDU and if
 694             this element contains a ssh public key
 695         :return: The modified list. Can ba an empty list, but always a list
 696         """
 697         if desc_primitive_list:
 698             primitive_list = desc_primitive_list.copy()
 699         else:
 700             primitive_list = []
 701         # look for primitive config, and get the position. None if not present
 702         config_position = None
 703         for index, primitive in enumerate(primitive_list):
 704             if primitive["name"] == "config":
 705                 config_position = index
 706                 break
 707
 708         # for NS, add always a config primitive if not present (bug 874)
 709         if not vca_deployed["member-vnf-index"] and config_position is None:
 710             primitive_list.insert(0, {"name": "config", "parameter": []})
 711             config_position = 0
 712         # for VNF/VDU add verify-ssh-credentials after config
 713         if vca_deployed["member-vnf-index"] and config_position is not None and vca_deployed.get("ssh-public-key"):
 714             primitive_list.insert(config_position + 1, {"name": "verify-ssh-credentials", "parameter": []})
 715         return primitive_list
 716
 717     async def instantiate(self, nsr_id, nslcmop_id):
 718
 719         # Try to lock HA task here
 720         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 721         if not task_is_locked_by_me:
 722             return
 723
 724         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 725         self.logger.debug(logging_text + "Enter")
 726         # get all needed from database
 727         start_deploy = time()
 728         db_nsr = None
 729         db_nslcmop = None
 730         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 731         db_nslcmop_update = {}
 732         nslcmop_operation_state = None
 733         db_vnfrs = {}
 734         RO_descriptor_number = 0   # number of descriptors created at RO
 735         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 736         n2vc_info = {}
 737         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 738         exc = None
 739         try:
 740             # wait for any previous tasks in process
 741             step = "Waiting for previous operations to terminate"
 742             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 743
 744             step = "Getting nslcmop={} from db".format(nslcmop_id)
 745             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 746             step = "Getting nsr={} from db".format(nsr_id)
 747             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 748             ns_params = db_nslcmop.get("operationParams")
 749             nsd = db_nsr["nsd"]
 750             nsr_name = db_nsr["name"]   # TODO short-name??
 751
 752             step = "Getting vnfrs from db"
 753             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 754             db_vnfds_ref = {}
 755             db_vnfds = {}
 756             db_vnfds_index = {}
 757             for vnfr in db_vnfrs_list:
 758                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 759                 vnfd_id = vnfr["vnfd-id"]
 760                 vnfd_ref = vnfr["vnfd-ref"]
 761                 if vnfd_id not in db_vnfds:
 762                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 763                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 764                     db_vnfds_ref[vnfd_ref] = vnfd
 765                     db_vnfds[vnfd_id] = vnfd
 766                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 767
 768             # Get or generates the _admin.deployed,VCA list
 769             vca_deployed_list = None
 770             vca_model_name = None
 771             if db_nsr["_admin"].get("deployed"):
 772                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 773                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 774             if vca_deployed_list is None:
 775                 vca_deployed_list = []
 776                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 777                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 778             elif isinstance(vca_deployed_list, dict):
 779                 # maintain backward compatibility. Change a dict to list at database
 780                 vca_deployed_list = list(vca_deployed_list.values())
 781                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 782                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 783
 784             db_nsr_update["detailed-status"] = "creating"
 785             db_nsr_update["operational-status"] = "init"
 786             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
 787                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 788                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 789
 790             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 791             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 792             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 793
 794             # Deploy charms
 795             # The parameters we'll need to deploy a charm
 796             number_to_configure = 0
 797
 798             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 799                 """An inner function to deploy the charm from either ns, vnf or vdu
 800                 For ns both vnf_index and vdu_id are None.
 801                 For vnf only vdu_id is None
 802                 For vdu both vnf_index and vdu_id contain a value
 803                 """
 804                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 805                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 806
 807                 machine_spec = {}
 808                 if native_charm:
 809                     machine_spec["username"] = charm_params.get("username")
 810                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 811
 812                 # Note: The charm needs to exist on disk at the location
 813                 # specified by charm_path.
 814                 descriptor = vnfd if vnf_index else nsd
 815                 base_folder = descriptor["_admin"]["storage"]
 816                 storage_params = self.fs.get_params()
 817                 charm_path = "{}{}/{}/charms/{}".format(
 818                     storage_params["path"],
 819                     base_folder["folder"],
 820                     base_folder["pkg-dir"],
 821                     proxy_charm
 822                 )
 823
 824                 # ns_name will be ignored in the current version of N2VC
 825                 # but will be implemented for the next point release.
 826                 model_name = nsr_id
 827                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 828                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 829                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 830
 831                 vca_index = len(vca_deployed_list)
 832                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 833                 # 26*26 charm in the same NS
 834                 application_name = application_name[0:48]
 835                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 836                 vca_deployed_ = {
 837                     "member-vnf-index": vnf_index,
 838                     "vdu_id": vdu_id,
 839                     "model": model_name,
 840                     "application": application_name,
 841                     "operational-status": "init",
 842                     "detailed-status": "",
 843                     "step": "initial-deploy",
 844                     "vnfd_id": vnfd_id,
 845                     "vdu_name": vdu_name,
 846                     "vdu_count_index": vdu_count_index,
 847                 }
 848                 vca_deployed_list.append(vca_deployed_)
 849                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 850                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 851
 852                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 853                                                                                                 proxy_charm))
 854                 if not n2vc_info:
 855                     n2vc_info["nsr_id"] = nsr_id
 856                     n2vc_info["nslcmop_id"] = nslcmop_id
 857                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 858                     n2vc_info["lcmOperationType"] = "instantiate"
 859                     n2vc_info["deployed"] = vca_deployed_list
 860                     n2vc_info["db_update"] = db_nsr_update
 861                 task = asyncio.ensure_future(
 862                     self.n2vc.DeployCharms(
 863                         model_name,          # The network service name
 864                         application_name,    # The application name
 865                         descriptor,          # The vnf/nsd descriptor
 866                         charm_path,          # Path to charm
 867                         charm_params,        # Runtime params, like mgmt ip
 868                         machine_spec,        # for native charms only
 869                         self.n2vc_callback,  # Callback for status changes
 870                         n2vc_info,           # Callback parameter
 871                         None,                # Callback parameter (task)
 872                     )
 873                 )
 874                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 875                                                          n2vc_info))
 876                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 877
 878             step = "Looking for needed vnfd to configure with proxy charm"
 879             self.logger.debug(logging_text + step)
 880
 881             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 882                 vnfd_id = c_vnf["vnfd-id-ref"]
 883                 vnf_index = str(c_vnf["member-vnf-index"])
 884                 vnfd = db_vnfds_ref[vnfd_id]
 885
 886                 # Get additional parameters
 887                 vnfr_params = {}
 888                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 889                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 890                 for k, v in vnfr_params.items():
 891                     if isinstance(v, str) and v.startswith("!!yaml "):
 892                         vnfr_params[k] = yaml.safe_load(v[7:])
 893
 894                 step = "deploying proxy charms for configuration"
 895                 # Check if this VNF has a charm configuration
 896                 vnf_config = vnfd.get("vnf-configuration")
 897                 if vnf_config and vnf_config.get("juju"):
 898                     proxy_charm = vnf_config["juju"]["charm"]
 899                     if vnf_config["juju"].get("proxy") is False:
 900                         # native_charm, will be deployed after VM. Skip
 901                         proxy_charm = None
 902
 903                     if proxy_charm:
 904                         if not vca_model_name:
 905                             step = "creating VCA model name '{}'".format(nsr_id)
 906                             self.logger.debug(logging_text + step)
 907                             await self.n2vc.CreateNetworkService(nsr_id)
 908                             vca_model_name = nsr_id
 909                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 910                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 911                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 912                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 913                         charm_params = {
 914                             "user_values": vnfr_params,
 915                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 916                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 917                         }
 918
 919                         # Login to the VCA. If there are multiple calls to login(),
 920                         # subsequent calls will be a nop and return immediately.
 921                         await self.n2vc.login()
 922
 923                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 924                         number_to_configure += 1
 925
 926                 # Deploy charms for each VDU that supports one.
 927                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 928                     vdu_config = vdu.get('vdu-configuration')
 929                     proxy_charm = None
 930
 931                     if vdu_config and vdu_config.get("juju"):
 932                         proxy_charm = vdu_config["juju"]["charm"]
 933                         if vdu_config["juju"].get("proxy") is False:
 934                             # native_charm, will be deployed after VM. Skip
 935                             proxy_charm = None
 936                         if proxy_charm:
 937                             if not vca_model_name:
 938                                 step = "creating VCA model name"
 939                                 await self.n2vc.CreateNetworkService(nsr_id)
 940                                 vca_model_name = nsr_id
 941                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 942                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 943                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 944                                                                                                           vdu["id"])
 945                             await self.n2vc.login()
 946                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 947                             # TODO for the moment only first vdu_id contains a charm deployed
 948                             if vdur["vdu-id-ref"] != vdu["id"]:
 949                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 950                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 951                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 952                             charm_params = {
 953                                 "user_values": vnfr_params,
 954                                 "rw_mgmt_ip": vdur["ip-address"],
 955                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 956                             }
 957                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 958                                          charm_params, n2vc_info)
 959                             number_to_configure += 1
 960
 961             # Check if this NS has a charm configuration
 962
 963             ns_config = nsd.get("ns-configuration")
 964             if ns_config and ns_config.get("juju"):
 965                 proxy_charm = ns_config["juju"]["charm"]
 966                 if ns_config["juju"].get("proxy") is False:
 967                     # native_charm, will be deployed after VM. Skip
 968                     proxy_charm = None
 969                 if proxy_charm:
 970                     step = "deploying proxy charm to configure ns"
 971                     # TODO is NS magmt IP address needed?
 972
 973                     # Get additional parameters
 974                     additional_params = {}
 975                     if db_nsr.get("additionalParamsForNs"):
 976                         additional_params = db_nsr["additionalParamsForNs"].copy()
 977                     for k, v in additional_params.items():
 978                         if isinstance(v, str) and v.startswith("!!yaml "):
 979                             additional_params[k] = yaml.safe_load(v[7:])
 980
 981                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 982                     charm_params = {
 983                         "user_values": additional_params,
 984                         # "rw_mgmt_ip": db_nsr["ip-address"],
 985                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 986                     }
 987
 988                     # Login to the VCA. If there are multiple calls to login(),
 989                     # subsequent calls will be a nop and return immediately.
 990                     await self.n2vc.login()
 991                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 992                     number_to_configure += 1
 993
 994             db_nsr_update["operational-status"] = "running"
 995
 996             # Wait until all charms has reached blocked or active status
 997             step = "waiting proxy charms to be ready"
 998             if number_to_configure:
 999                 # wait until all charms are configured.
1000                 # steps are:
1001                 #       initial-deploy
1002                 #       get-ssh-public-key
1003                 #       generate-ssh-key
1004                 #       retry-get-ssh-public-key
1005                 #       ssh-public-key-obtained
1006                 while time() <= start_deploy + self.total_deploy_timeout:
1007                     if db_nsr_update:
1008                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1009                     if db_nslcmop_update:
1010                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1011
1012                     all_active = True
1013                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
1014                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1015                         if vca_deployed["step"] == "initial-deploy":
1016                             if vca_deployed["operational-status"] in ("active", "blocked"):
1017                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
1018                                        .format(vca_deployed["member-vnf-index"],
1019                                                vca_deployed["vdu_id"])
1020                                 self.logger.debug(logging_text + step)
1021                                 try:
1022                                     primitive_id = await self.n2vc.ExecutePrimitive(
1023                                         vca_deployed["model"],
1024                                         vca_deployed["application"],
1025                                         "get-ssh-public-key",
1026                                         None,
1027                                     )
1028                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
1029                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1030                                         primitive_id
1031                                     db_nsr_update[database_entry + "operational-status"] =\
1032                                         vca_deployed["operational-status"]
1033                                 except PrimitiveDoesNotExist:
1034                                     ssh_public_key = None
1035                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1036                                         "ssh-public-key-obtained"
1037                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
1038                                         ssh_public_key
1039                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
1040                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
1041                                     self.logger.debug(logging_text + step)
1042
1043                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
1044                             primitive_id = vca_deployed["primitive_id"]
1045                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1046                                                                                   primitive_id)
1047                             if primitive_status in ("completed", "failed"):
1048                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1049                                                                                       primitive_id)
1050                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
1051                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
1052                                         primitive_result.get("pubkey"):
1053                                     ssh_public_key = primitive_result.get("pubkey")
1054                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1055                                         "ssh-public-key-obtained"
1056                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
1057                                         ssh_public_key
1058                                     n2vc_key_list.append(ssh_public_key)
1059                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
1060                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
1061                                     self.logger.debug(logging_text + step)
1062                                 else:  # primitive_status == "failed":
1063                                     if vca_deployed["step"] == "get-ssh-public-key":
1064                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
1065                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
1066                                                                      vca_deployed["vdu_id"])
1067                                         self.logger.debug(logging_text + step)
1068                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1069                                             "generate-ssh-key"
1070                                         primitive_id = await self.n2vc.ExecutePrimitive(
1071                                             vca_deployed["model"],
1072                                             vca_deployed["application"],
1073                                             "generate-ssh-key",
1074                                             None,
1075                                         )
1076                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1077                                             primitive_id
1078                                     else:  # failed for second time
1079                                         raise LcmException(
1080                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
1081
1082                         elif vca_deployed["step"] == "generate-ssh-key":
1083                             primitive_id = vca_deployed["primitive_id"]
1084                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1085                                                                                   primitive_id)
1086                             if primitive_status in ("completed", "failed"):
1087                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1088                                                                                       primitive_id)
1089                                 vca_deployed["primitive_id"] = db_nsr_update[
1090                                     database_entry + "primitive_id"] = None
1091                                 if primitive_status == "completed":
1092                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1093                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1094                                                               vca_deployed["vdu_id"])
1095                                     self.logger.debug(logging_text + step)
1096                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1097                                         "retry-get-ssh-public-key"
1098                                     primitive_id = await self.n2vc.ExecutePrimitive(
1099                                         vca_deployed["model"],
1100                                         vca_deployed["application"],
1101                                         "get-ssh-public-key",
1102                                         None,
1103                                     )
1104                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1105                                         primitive_id
1106
1107                                 else:  # primitive_status == "failed":
1108                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1109                                                        .format(primitive_result))
1110
1111                         if vca_deployed["step"] != "ssh-public-key-obtained":
1112                             all_active = False
1113
1114                     if all_active:
1115                         break
1116                     await asyncio.sleep(5)
1117                 else:   # total_deploy_timeout
1118                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1119                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1120
1121             # deploy RO
1122             # get vnfds, instantiate at RO
1123             for c_vnf in nsd.get("constituent-vnfd", ()):
1124                 member_vnf_index = c_vnf["member-vnf-index"]
1125                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1126                 vnfd_ref = vnfd["id"]
1127                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1128                     vnfd_ref, member_vnf_index)
1129                 # self.logger.debug(logging_text + step)
1130                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1131                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1132                 RO_descriptor_number += 1
1133
1134                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1135                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1136                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1137                         break
1138                 else:
1139                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1140                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1141
1142                 # look if present
1143                 RO_update = {"member-vnf-index": member_vnf_index}
1144                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1145                 if vnfd_list:
1146                     RO_update["id"] = vnfd_list[0]["uuid"]
1147                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1148                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1149                 else:
1150                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1151                                            get("additionalParamsForVnf"), nsr_id)
1152                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1153                     RO_update["id"] = desc["uuid"]
1154                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1155                         vnfd_ref, member_vnf_index, desc["uuid"]))
1156                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1157                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1158                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1159
1160             # create nsd at RO
1161             nsd_ref = nsd["id"]
1162             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1163             # self.logger.debug(logging_text + step)
1164
1165             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1166             RO_descriptor_number += 1
1167             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1168             if nsd_list:
1169                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1170                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1171                     nsd_ref, RO_nsd_uuid))
1172             else:
1173                 nsd_RO = deepcopy(nsd)
1174                 nsd_RO["id"] = RO_osm_nsd_id
1175                 nsd_RO.pop("_id", None)
1176                 nsd_RO.pop("_admin", None)
1177                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1178                     member_vnf_index = c_vnf["member-vnf-index"]
1179                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1180                 for c_vld in nsd_RO.get("vld", ()):
1181                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1182                         member_vnf_index = cp["member-vnf-index-ref"]
1183                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1184
1185                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1186                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1187                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1188                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1189             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1190
1191             # Crate ns at RO
1192             # if present use it unless in error status
1193             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1194             if RO_nsr_id:
1195                 try:
1196                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1197                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1198                     desc = await self.RO.show("ns", RO_nsr_id)
1199                 except ROclient.ROClientException as e:
1200                     if e.http_code != HTTPStatus.NOT_FOUND:
1201                         raise
1202                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1203                 if RO_nsr_id:
1204                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1205                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1206                     if ns_status == "ERROR":
1207                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1208                         self.logger.debug(logging_text + step)
1209                         await self.RO.delete("ns", RO_nsr_id)
1210                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1211             if not RO_nsr_id:
1212                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1213                 # self.logger.debug(logging_text + step)
1214
1215                 # check if VIM is creating and wait  look if previous tasks in process
1216                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1217                 if task_dependency:
1218                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1219                     self.logger.debug(logging_text + step)
1220                     await asyncio.wait(task_dependency, timeout=3600)
1221                 if ns_params.get("vnf"):
1222                     for vnf in ns_params["vnf"]:
1223                         if "vimAccountId" in vnf:
1224                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1225                                                                                         vnf["vimAccountId"])
1226                         if task_dependency:
1227                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1228                             self.logger.debug(logging_text + step)
1229                             await asyncio.wait(task_dependency, timeout=3600)
1230
1231                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1232
1233                 # feature 1429. Add n2vc public key to needed VMs
1234                 n2vc_key = await self.n2vc.GetPublicKey()
1235                 n2vc_key_list.append(n2vc_key)
1236                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1237
1238                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1239                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1240                                             name=db_nsr["name"],
1241                                             scenario=RO_nsd_uuid)
1242                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1243                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1244                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1245                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1246             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1247
1248             # wait until NS is ready
1249             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1250             detailed_status_old = None
1251             self.logger.debug(logging_text + step)
1252
1253             while time() <= start_deploy + self.total_deploy_timeout:
1254                 desc = await self.RO.show("ns", RO_nsr_id)
1255                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1256                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1257                 if ns_status == "ERROR":
1258                     raise ROclient.ROClientException(ns_status_info)
1259                 elif ns_status == "BUILD":
1260                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1261                 elif ns_status == "ACTIVE":
1262                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1263                     try:
1264                         self.ns_update_vnfr(db_vnfrs, desc)
1265                         break
1266                     except LcmExceptionNoMgmtIP:
1267                         pass
1268                 else:
1269                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1270                 if detailed_status != detailed_status_old:
1271                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1272                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1273                 await asyncio.sleep(5, loop=self.loop)
1274             else:   # total_deploy_timeout
1275                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1276
1277             step = "Updating NSR"
1278             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1279
1280             db_nsr_update["operational-status"] = "running"
1281             db_nsr["detailed-status"] = "Configuring vnfr"
1282             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1283
1284             # Configure proxy charms once VMs are up
1285             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1286                 vnf_index = vca_deployed.get("member-vnf-index")
1287                 vdu_id = vca_deployed.get("vdu_id")
1288                 vdu_name = None
1289                 vdu_count_index = None
1290
1291                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1292                                                                                                            vdu_id)
1293                 add_params = {}
1294                 initial_config_primitive_list = None
1295                 if vnf_index:
1296                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1297                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1298                     vnfd = db_vnfds_index[vnf_index]
1299
1300                     if vdu_id:
1301                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1302                             if vdu["id"] == vdu_id:
1303                                 initial_config_primitive_list = vdu['vdu-configuration'].get('initial-config-primitive')
1304                                 break
1305                         else:
1306                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1307                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1308                         # TODO for the moment only first vdu_id contains a charm deployed
1309                         if vdur["vdu-id-ref"] != vdu["id"]:
1310                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1311                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1312                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1313                     else:
1314                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1315                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive')
1316                 else:
1317                     if db_nsr.get("additionalParamsForNs"):
1318                         add_params = db_nsr["additionalParamsForNs"].copy()
1319                     for k, v in add_params.items():
1320                         if isinstance(v, str) and v.startswith("!!yaml "):
1321                             add_params[k] = yaml.safe_load(v[7:])
1322                     add_params["rw_mgmt_ip"] = None
1323                     add_params["ns_config_info"] = self._get_ns_config_info(vca_deployed_list)
1324                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive')
1325
1326                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1327                 # add config if not present for NS charm
1328                 initial_config_primitive_list = self._get_initial_config_primitive_list(initial_config_primitive_list,
1329                                                                                         vca_deployed)
1330
1331                 for initial_config_primitive in initial_config_primitive_list:
1332                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1333                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1334                                       .format(initial_config_primitive["name"], primitive_params_))
1335                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1336                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1337                         initial_config_primitive["name"],
1338                         primitive_params_,
1339                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1340                         retries_interval=30)
1341                     if primitive_result != "COMPLETED":
1342                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1343                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1344                                                    vca_deployed["vdu_id"], primitive_detail))
1345
1346             # Deploy native charms
1347             step = "Looking for needed vnfd to configure with native charm"
1348             self.logger.debug(logging_text + step)
1349
1350             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1351                 vnfd_id = c_vnf["vnfd-id-ref"]
1352                 vnf_index = str(c_vnf["member-vnf-index"])
1353                 vnfd = db_vnfds_ref[vnfd_id]
1354
1355                 # Get additional parameters
1356                 vnfr_params = {}
1357                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1358                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1359                 for k, v in vnfr_params.items():
1360                     if isinstance(v, str) and v.startswith("!!yaml "):
1361                         vnfr_params[k] = yaml.safe_load(v[7:])
1362
1363                 # Check if this VNF has a charm configuration
1364                 vnf_config = vnfd.get("vnf-configuration")
1365                 if vnf_config and vnf_config.get("juju"):
1366                     native_charm = vnf_config["juju"].get("proxy") is False
1367                     proxy_charm = vnf_config["juju"]["charm"]
1368                     if native_charm and proxy_charm:
1369                         if not vca_model_name:
1370                             step = "creating VCA model name '{}'".format(nsr_id)
1371                             self.logger.debug(logging_text + step)
1372                             await self.n2vc.CreateNetworkService(nsr_id)
1373                             vca_model_name = nsr_id
1374                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1375                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1376                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1377                         self.logger.debug(logging_text + step)
1378
1379                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1380                         charm_params = {
1381                             "user_values": vnfr_params,
1382                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1383                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1384                         }
1385
1386                         # get username
1387                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1388                         #  merged. Meanwhile let's get username from initial-config-primitive
1389                         if vnf_config.get("initial-config-primitive"):
1390                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1391                                 if param["name"] == "ssh-username":
1392                                     charm_params["username"] = param["value"]
1393                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1394                             if vnf_config["config-access"]["ssh-access"].get("required"):
1395                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1396
1397                         # Login to the VCA. If there are multiple calls to login(),
1398                         # subsequent calls will be a nop and return immediately.
1399                         await self.n2vc.login()
1400
1401                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1402                         number_to_configure += 1
1403
1404                 # Deploy charms for each VDU that supports one.
1405                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1406                     vdu_config = vdu.get('vdu-configuration')
1407                     native_charm = False
1408
1409                     if vdu_config and vdu_config.get("juju"):
1410                         native_charm = vdu_config["juju"].get("proxy") is False
1411                         proxy_charm = vdu_config["juju"]["charm"]
1412                         if native_charm and proxy_charm:
1413                             if not vca_model_name:
1414                                 step = "creating VCA model name"
1415                                 await self.n2vc.CreateNetworkService(nsr_id)
1416                                 vca_model_name = nsr_id
1417                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1418                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1419                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1420                                                                                                      vdu["id"])
1421
1422                             self.logger.debug(logging_text + step)
1423
1424                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1425
1426                             # TODO for the moment only first vdu_id contains a charm deployed
1427                             if vdur["vdu-id-ref"] != vdu["id"]:
1428                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1429                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1430                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1431                             charm_params = {
1432                                 "user_values": vnfr_params,
1433                                 "rw_mgmt_ip": vdur["ip-address"],
1434                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1435                             }
1436
1437                             # get username
1438                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1439                             #  merged. Meanwhile let's get username from initial-config-primitive
1440                             if vdu_config.get("initial-config-primitive"):
1441                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1442                                     if param["name"] == "ssh-username":
1443                                         charm_params["username"] = param["value"]
1444                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1445                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1446                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1447                                         "default-user")
1448
1449                             await self.n2vc.login()
1450
1451                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1452                                          charm_params, n2vc_info, native_charm)
1453                             number_to_configure += 1
1454
1455             # Check if this NS has a charm configuration
1456
1457             ns_config = nsd.get("ns-configuration")
1458             if ns_config and ns_config.get("juju"):
1459                 native_charm = ns_config["juju"].get("proxy") is False
1460                 proxy_charm = ns_config["juju"]["charm"]
1461                 if native_charm and proxy_charm:
1462                     step = "deploying native charm to configure ns"
1463                     # TODO is NS magmt IP address needed?
1464
1465                     # Get additional parameters
1466                     additional_params = {}
1467                     if db_nsr.get("additionalParamsForNs"):
1468                         additional_params = db_nsr["additionalParamsForNs"].copy()
1469                     for k, v in additional_params.items():
1470                         if isinstance(v, str) and v.startswith("!!yaml "):
1471                             additional_params[k] = yaml.safe_load(v[7:])
1472
1473                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1474                     charm_params = {
1475                         "user_values": additional_params,
1476                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1477                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1478                     }
1479
1480                     # get username
1481                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1482                     #  merged. Meanwhile let's get username from initial-config-primitive
1483                     if ns_config.get("initial-config-primitive"):
1484                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1485                             if param["name"] == "ssh-username":
1486                                 charm_params["username"] = param["value"]
1487                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1488                         if ns_config["config-access"]["ssh-access"].get("required"):
1489                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1490
1491                     # Login to the VCA. If there are multiple calls to login(),
1492                     # subsequent calls will be a nop and return immediately.
1493                     await self.n2vc.login()
1494                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1495                     number_to_configure += 1
1496
1497             # waiting all charms are ok
1498             configuration_failed = False
1499             if number_to_configure:
1500                 step = "Waiting all charms are active"
1501                 old_status = "configuring: init: {}".format(number_to_configure)
1502                 db_nsr_update["config-status"] = old_status
1503                 db_nsr_update["detailed-status"] = old_status
1504                 db_nslcmop_update["detailed-status"] = old_status
1505
1506                 # wait until all are configured.
1507                 while time() <= start_deploy + self.total_deploy_timeout:
1508                     if db_nsr_update:
1509                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1510                     if db_nslcmop_update:
1511                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1512                     # TODO add a fake task that set n2vc_event after some time
1513                     await n2vc_info["n2vc_event"].wait()
1514                     n2vc_info["n2vc_event"].clear()
1515                     all_active = True
1516                     status_map = {}
1517                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1518                     now = time()
1519                     for vca_deployed in vca_deployed_list:
1520                         vca_status = vca_deployed["operational-status"]
1521                         if vca_status not in status_map:
1522                             # Initialize it
1523                             status_map[vca_status] = 0
1524                         status_map[vca_status] += 1
1525
1526                         if vca_status == "active":
1527                             vca_deployed.pop("time_first_error", None)
1528                             vca_deployed.pop("status_first_error", None)
1529                             continue
1530
1531                         all_active = False
1532                         if vca_status in ("error", "blocked"):
1533                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1534                             # if not first time in this status error
1535                             if not vca_deployed.get("time_first_error"):
1536                                 vca_deployed["time_first_error"] = now
1537                                 continue
1538                         if vca_deployed.get("time_first_error") and \
1539                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1540                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1541                                                    .format(vca_deployed["member-vnf-index"],
1542                                                            vca_deployed["vdu_id"], vca_status,
1543                                                            vca_deployed["detailed-status-error"]))
1544
1545                     if all_active:
1546                         break
1547                     elif n2vc_error_text:
1548                         db_nsr_update["config-status"] = "failed"
1549                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1550                         db_nsr_update["detailed-status"] = error_text
1551                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1552                         db_nslcmop_update["detailed-status"] = error_text
1553                         db_nslcmop_update["statusEnteredTime"] = time()
1554                         configuration_failed = True
1555                         break
1556                     else:
1557                         cs = "configuring: "
1558                         separator = ""
1559                         for status, num in status_map.items():
1560                             cs += separator + "{}: {}".format(status, num)
1561                             separator = ", "
1562                         if old_status != cs:
1563                             db_nsr_update["config-status"] = cs
1564                             db_nsr_update["detailed-status"] = cs
1565                             db_nslcmop_update["detailed-status"] = cs
1566                             old_status = cs
1567                 else:   # total_deploy_timeout
1568                     raise LcmException("Timeout waiting ns to be configured")
1569
1570             if not configuration_failed:
1571                 # all is done
1572                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1573                 db_nslcmop_update["statusEnteredTime"] = time()
1574                 db_nslcmop_update["detailed-status"] = "done"
1575                 db_nsr_update["config-status"] = "configured"
1576                 db_nsr_update["detailed-status"] = "done"
1577
1578             return
1579
1580         except (ROclient.ROClientException, DbException, LcmException) as e:
1581             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1582             exc = e
1583         except asyncio.CancelledError:
1584             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1585             exc = "Operation was cancelled"
1586         except Exception as e:
1587             exc = traceback.format_exc()
1588             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1589                                  exc_info=True)
1590         finally:
1591             if exc:
1592                 if db_nsr:
1593                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1594                     db_nsr_update["operational-status"] = "failed"
1595                 if db_nslcmop:
1596                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1597                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1598                     db_nslcmop_update["statusEnteredTime"] = time()
1599             try:
1600                 if db_nsr:
1601                     db_nsr_update["_admin.nslcmop"] = None
1602                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1603                 if db_nslcmop_update:
1604                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1605             except DbException as e:
1606                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1607             if nslcmop_operation_state:
1608                 try:
1609                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1610                                                                    "operationState": nslcmop_operation_state},
1611                                             loop=self.loop)
1612                 except Exception as e:
1613                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1614
1615             self.logger.debug(logging_text + "Exit")
1616             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1617
1618     async def _destroy_charm(self, model, application):
1619         """
1620         Order N2VC destroy a charm
1621         :param model:
1622         :param application:
1623         :return: True if charm does not exist. False if it exist
1624         """
1625         if not await self.n2vc.HasApplication(model, application):
1626             return True  # Already removed
1627         await self.n2vc.RemoveCharms(model, application)
1628         return False
1629
1630     async def _wait_charm_destroyed(self, model, application, timeout):
1631         """
1632         Wait until charm does not exist
1633         :param model:
1634         :param application:
1635         :param timeout:
1636         :return: True if not exist, False if timeout
1637         """
1638         while True:
1639             if not await self.n2vc.HasApplication(model, application):
1640                 return True
1641             if timeout < 0:
1642                 return False
1643             await asyncio.sleep(10)
1644             timeout -= 10
1645
1646     # Check if this VNFD has a configured terminate action
1647     def _has_terminate_config_primitive(self, vnfd):
1648         vnf_config = vnfd.get("vnf-configuration")
1649         if vnf_config and vnf_config.get("terminate-config-primitive"):
1650             return True
1651         else:
1652             return False
1653
1654     @staticmethod
1655     def _get_terminate_config_primitive_seq_list(vnfd):
1656         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1657         # No need to check for existing primitive twice, already done before
1658         vnf_config = vnfd.get("vnf-configuration")
1659         seq_list = vnf_config.get("terminate-config-primitive")
1660         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1661         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1662         return seq_list_sorted
1663
1664     @staticmethod
1665     def _create_nslcmop(nsr_id, operation, params):
1666         """
1667         Creates a ns-lcm-opp content to be stored at database.
1668         :param nsr_id: internal id of the instance
1669         :param operation: instantiate, terminate, scale, action, ...
1670         :param params: user parameters for the operation
1671         :return: dictionary following SOL005 format
1672         """
1673         # Raise exception if invalid arguments
1674         if not (nsr_id and operation and params):
1675             raise LcmException(
1676                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1677         now = time()
1678         _id = str(uuid4())
1679         nslcmop = {
1680             "id": _id,
1681             "_id": _id,
1682             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1683             "operationState": "PROCESSING",
1684             "statusEnteredTime": now,
1685             "nsInstanceId": nsr_id,
1686             "lcmOperationType": operation,
1687             "startTime": now,
1688             "isAutomaticInvocation": False,
1689             "operationParams": params,
1690             "isCancelPending": False,
1691             "links": {
1692                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1693                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1694             }
1695         }
1696         return nslcmop
1697
1698     def _get_terminate_primitive_params(self, seq, vnf_index):
1699         primitive = seq.get('name')
1700         primitive_params = {}
1701         params = {
1702             "member_vnf_index": vnf_index,
1703             "primitive": primitive,
1704             "primitive_params": primitive_params,
1705         }
1706         desc_params = {}
1707         return self._map_primitive_params(seq, params, desc_params)
1708
1709     # sub-operations
1710
1711     def _reintent_or_skip_suboperation(self, db_nslcmop, op_index):
1712         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
1713         if (op.get('operationState') == 'COMPLETED'):
1714             # b. Skip sub-operation
1715             # _ns_execute_primitive() or RO.create_action() will NOT be executed
1716             return self.SUBOPERATION_STATUS_SKIP
1717         else:
1718             # c. Reintent executing sub-operation
1719             # The sub-operation exists, and operationState != 'COMPLETED'
1720             # Update operationState = 'PROCESSING' to indicate a reintent.
1721             operationState = 'PROCESSING'
1722             detailed_status = 'In progress'
1723             self._update_suboperation_status(
1724                 db_nslcmop, op_index, operationState, detailed_status)
1725             # Return the sub-operation index
1726             # _ns_execute_primitive() or RO.create_action() will be called from scale()
1727             # with arguments extracted from the sub-operation
1728             return op_index
1729
1730     # Find a sub-operation where all keys in a matching dictionary must match
1731     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
1732     def _find_suboperation(self, db_nslcmop, match):
1733         if (db_nslcmop and match):
1734             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
1735             for i, op in enumerate(op_list):
1736                 if all(op.get(k) == match[k] for k in match):
1737                     return i
1738         return self.SUBOPERATION_STATUS_NOT_FOUND
1739
1740     # Update status for a sub-operation given its index
1741     def _update_suboperation_status(self, db_nslcmop, op_index, operationState, detailed_status):
1742         # Update DB for HA tasks
1743         q_filter = {'_id': db_nslcmop['_id']}
1744         update_dict = {'_admin.operations.{}.operationState'.format(op_index): operationState,
1745                        '_admin.operations.{}.detailed-status'.format(op_index): detailed_status}
1746         self.db.set_one("nslcmops",
1747                         q_filter=q_filter,
1748                         update_dict=update_dict,
1749                         fail_on_empty=False)
1750
1751     # Add sub-operation, return the index of the added sub-operation
1752     # Optionally, set operationState, detailed-status, and operationType
1753     # Status and type are currently set for 'scale' sub-operations:
1754     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
1755     # 'detailed-status' : status message
1756     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
1757     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
1758     def _add_suboperation(self, db_nslcmop, vnf_index, vdu_id, vdu_count_index,
1759                           vdu_name, primitive, mapped_primitive_params,
1760                           operationState=None, detailed_status=None, operationType=None,
1761                           RO_nsr_id=None, RO_scaling_info=None):
1762         if not (db_nslcmop):
1763             return self.SUBOPERATION_STATUS_NOT_FOUND
1764         # Get the "_admin.operations" list, if it exists
1765         db_nslcmop_admin = db_nslcmop.get('_admin', {})
1766         op_list = db_nslcmop_admin.get('operations')
1767         # Create or append to the "_admin.operations" list
1768         new_op = {'member_vnf_index': vnf_index,
1769                   'vdu_id': vdu_id,
1770                   'vdu_count_index': vdu_count_index,
1771                   'primitive': primitive,
1772                   'primitive_params': mapped_primitive_params}
1773         if operationState:
1774             new_op['operationState'] = operationState
1775         if detailed_status:
1776             new_op['detailed-status'] = detailed_status
1777         if operationType:
1778             new_op['lcmOperationType'] = operationType
1779         if RO_nsr_id:
1780             new_op['RO_nsr_id'] = RO_nsr_id
1781         if RO_scaling_info:
1782             new_op['RO_scaling_info'] = RO_scaling_info
1783         if not op_list:
1784             # No existing operations, create key 'operations' with current operation as first list element
1785             db_nslcmop_admin.update({'operations': [new_op]})
1786             op_list = db_nslcmop_admin.get('operations')
1787         else:
1788             # Existing operations, append operation to list
1789             op_list.append(new_op)
1790
1791         db_nslcmop_update = {'_admin.operations': op_list}
1792         self.update_db_2("nslcmops", db_nslcmop['_id'], db_nslcmop_update)
1793         op_index = len(op_list) - 1
1794         return op_index
1795
1796     # Helper methods for scale() sub-operations
1797
1798     # pre-scale/post-scale:
1799     # Check for 3 different cases:
1800     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
1801     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
1802     # c. Reintent: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
1803     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index,
1804                                          vnf_config_primitive, primitive_params, operationType,
1805                                          RO_nsr_id=None, RO_scaling_info=None):
1806         # Find this sub-operation
1807         if (RO_nsr_id and RO_scaling_info):
1808             operationType = 'SCALE-RO'
1809             match = {
1810                 'member_vnf_index': vnf_index,
1811                 'RO_nsr_id': RO_nsr_id,
1812                 'RO_scaling_info': RO_scaling_info,
1813             }
1814         else:
1815             match = {
1816                 'member_vnf_index': vnf_index,
1817                 'primitive': vnf_config_primitive,
1818                 'primitive_params': primitive_params,
1819                 'lcmOperationType': operationType
1820             }
1821         op_index = self._find_suboperation(db_nslcmop, match)
1822         if (op_index == self.SUBOPERATION_STATUS_NOT_FOUND):
1823             # a. New sub-operation
1824             # The sub-operation does not exist, add it.
1825             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
1826             # The following parameters are set to None for all kind of scaling:
1827             vdu_id = None
1828             vdu_count_index = None
1829             vdu_name = None
1830             if (RO_nsr_id and RO_scaling_info):
1831                 vnf_config_primitive = None
1832                 primitive_params = None
1833             else:
1834                 RO_nsr_id = None
1835                 RO_scaling_info = None
1836             # Initial status for sub-operation
1837             operationState = 'PROCESSING'
1838             detailed_status = 'In progress'
1839             # Add sub-operation for pre/post-scaling (zero or more operations)
1840             self._add_suboperation(db_nslcmop,
1841                                    vnf_index,
1842                                    vdu_id,
1843                                    vdu_count_index,
1844                                    vdu_name,
1845                                    vnf_config_primitive,
1846                                    primitive_params,
1847                                    operationState,
1848                                    detailed_status,
1849                                    operationType,
1850                                    RO_nsr_id,
1851                                    RO_scaling_info)
1852             return self.SUBOPERATION_STATUS_NEW
1853         else:
1854             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
1855             # or op_index (operationState != 'COMPLETED')
1856             return self._reintent_or_skip_suboperation(db_nslcmop, op_index)
1857
1858     # Helper methods for terminate()
1859
1860     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1861         """ Create a primitive with params from VNFD
1862             Called from terminate() before deleting instance
1863             Calls action() to execute the primitive """
1864         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1865         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1866         db_vnfds = {}
1867         # Loop over VNFRs
1868         for vnfr in db_vnfrs_list:
1869             vnfd_id = vnfr["vnfd-id"]
1870             vnf_index = vnfr["member-vnf-index-ref"]
1871             if vnfd_id not in db_vnfds:
1872                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1873                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1874                 db_vnfds[vnfd_id] = vnfd
1875             vnfd = db_vnfds[vnfd_id]
1876             if not self._has_terminate_config_primitive(vnfd):
1877                 continue
1878             # Get the primitive's sorted sequence list
1879             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1880             for seq in seq_list:
1881                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1882                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1883                     vnf_index, seq.get("name"))
1884                 self.logger.debug(logging_text + step)
1885                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1886                 primitive = seq.get('name')
1887                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1888                 # The following 3 parameters are currently set to None for 'terminate':
1889                 # vdu_id, vdu_count_index, vdu_name
1890                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1891                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1892                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1893                 # Add sub-operation
1894                 self._add_suboperation(db_nslcmop,
1895                                        nslcmop_id,
1896                                        vnf_index,
1897                                        vdu_id,
1898                                        vdu_count_index,
1899                                        vdu_name,
1900                                        primitive,
1901                                        mapped_primitive_params)
1902                 # Sub-operations: Call _ns_execute_primitive() instead of action()
1903                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1904                 nsr_deployed = db_nsr["_admin"]["deployed"]
1905                 result, result_detail = await self._ns_execute_primitive(
1906                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1907                     mapped_primitive_params)
1908
1909                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1910                 #    nsr_id, nslcmop_terminate_action_id)
1911                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1912                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1913                 if result not in result_ok:
1914                     raise LcmException(
1915                         "terminate_primitive_action for vnf_member_index={}",
1916                         " primitive={} fails with error {}".format(
1917                             vnf_index, seq.get("name"), result_detail))
1918
1919     async def terminate(self, nsr_id, nslcmop_id):
1920
1921         # Try to lock HA task here
1922         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1923         if not task_is_locked_by_me:
1924             return
1925
1926         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1927         self.logger.debug(logging_text + "Enter")
1928         db_nsr = None
1929         db_nslcmop = None
1930         exc = None
1931         failed_detail = []   # annotates all failed error messages
1932         vca_time_destroy = None   # time of where destroy charm order
1933         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1934         db_nslcmop_update = {}
1935         nslcmop_operation_state = None
1936         autoremove = False  # autoremove after terminated
1937         try:
1938             # wait for any previous tasks in process
1939             step = "Waiting for previous operations to terminate"
1940             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1941
1942             step = "Getting nslcmop={} from db".format(nslcmop_id)
1943             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1944             step = "Getting nsr={} from db".format(nsr_id)
1945             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1946             # nsd = db_nsr["nsd"]
1947             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1948             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1949                 return
1950             # #TODO check if VIM is creating and wait
1951             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1952             # Call internal terminate action
1953             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1954
1955             db_nsr_update["operational-status"] = "terminating"
1956             db_nsr_update["config-status"] = "terminating"
1957
1958             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1959                 vca_model_name = nsr_deployed["VCA-model-name"]
1960                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1961                 self.logger.debug(logging_text + step)
1962                 try:
1963                     await self.n2vc.DestroyNetworkService(vca_model_name)
1964                 except NetworkServiceDoesNotExist:
1965                     pass
1966                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1967                 if nsr_deployed.get("VCA"):
1968                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1969                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1970                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1971             # for backward compatibility if charm have been created with "default"  model name delete one by one
1972             elif nsr_deployed and nsr_deployed.get("VCA"):
1973                 try:
1974                     step = "Scheduling configuration charms removing"
1975                     db_nsr_update["detailed-status"] = "Deleting charms"
1976                     self.logger.debug(logging_text + step)
1977                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1978                     # for backward compatibility
1979                     if isinstance(nsr_deployed["VCA"], dict):
1980                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1981                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1982                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1983
1984                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1985                         if vca_deployed:
1986                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1987                                 vca_deployed.clear()
1988                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1989                             else:
1990                                 vca_time_destroy = time()
1991                 except Exception as e:
1992                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1993
1994             # remove from RO
1995             RO_fail = False
1996
1997             # Delete ns
1998             RO_nsr_id = RO_delete_action = None
1999             if nsr_deployed and nsr_deployed.get("RO"):
2000                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
2001                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
2002             try:
2003                 if RO_nsr_id:
2004                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
2005                         "Deleting ns from VIM"
2006                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2007                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2008                     self.logger.debug(logging_text + step)
2009                     desc = await self.RO.delete("ns", RO_nsr_id)
2010                     RO_delete_action = desc["action_id"]
2011                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
2012                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2013                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2014                 if RO_delete_action:
2015                     # wait until NS is deleted from VIM
2016                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
2017                         format(RO_nsr_id, RO_delete_action)
2018                     detailed_status_old = None
2019                     self.logger.debug(logging_text + step)
2020
2021                     delete_timeout = 20 * 60   # 20 minutes
2022                     while delete_timeout > 0:
2023                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2024                                                   extra_item_id=RO_delete_action)
2025                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2026                         if ns_status == "ERROR":
2027                             raise ROclient.ROClientException(ns_status_info)
2028                         elif ns_status == "BUILD":
2029                             detailed_status = step + "; {}".format(ns_status_info)
2030                         elif ns_status == "ACTIVE":
2031                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2032                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2033                             break
2034                         else:
2035                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2036                         if detailed_status != detailed_status_old:
2037                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
2038                                 db_nsr_update["detailed-status"] = detailed_status
2039                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2040                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2041                         await asyncio.sleep(5, loop=self.loop)
2042                         delete_timeout -= 5
2043                     else:  # delete_timeout <= 0:
2044                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
2045
2046             except ROclient.ROClientException as e:
2047                 if e.http_code == 404:  # not found
2048                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
2049                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
2050                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
2051                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
2052                 elif e.http_code == 409:   # conflict
2053                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
2054                     self.logger.debug(logging_text + failed_detail[-1])
2055                     RO_fail = True
2056                 else:
2057                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
2058                     self.logger.error(logging_text + failed_detail[-1])
2059                     RO_fail = True
2060
2061             # Delete nsd
2062             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
2063                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
2064                 try:
2065                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
2066                         "Deleting nsd from RO"
2067                     await self.RO.delete("nsd", RO_nsd_id)
2068                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
2069                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2070                 except ROclient.ROClientException as e:
2071                     if e.http_code == 404:  # not found
2072                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2073                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
2074                     elif e.http_code == 409:   # conflict
2075                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
2076                         self.logger.debug(logging_text + failed_detail[-1])
2077                         RO_fail = True
2078                     else:
2079                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
2080                         self.logger.error(logging_text + failed_detail[-1])
2081                         RO_fail = True
2082
2083             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
2084                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
2085                     if not vnf_deployed or not vnf_deployed["id"]:
2086                         continue
2087                     try:
2088                         RO_vnfd_id = vnf_deployed["id"]
2089                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
2090                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
2091                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
2092                         await self.RO.delete("vnfd", RO_vnfd_id)
2093                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
2094                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2095                     except ROclient.ROClientException as e:
2096                         if e.http_code == 404:  # not found
2097                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2098                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
2099                         elif e.http_code == 409:   # conflict
2100                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
2101                             self.logger.debug(logging_text + failed_detail[-1])
2102                         else:
2103                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
2104                             self.logger.error(logging_text + failed_detail[-1])
2105
2106             # wait until charm deleted
2107             if vca_time_destroy:
2108                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
2109                     "Waiting for deletion of configuration charms"
2110                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2111                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2112                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
2113                     if not vca_deployed:
2114                         continue
2115                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
2116                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
2117                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
2118                                                             timeout):
2119                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
2120                             vca_deployed["application"]))
2121                     else:
2122                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
2123
2124             if failed_detail:
2125                 self.logger.error(logging_text + " ;".join(failed_detail))
2126                 db_nsr_update["operational-status"] = "failed"
2127                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
2128                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
2129                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2130                 db_nslcmop_update["statusEnteredTime"] = time()
2131             else:
2132                 db_nsr_update["operational-status"] = "terminated"
2133                 db_nsr_update["detailed-status"] = "Done"
2134                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
2135                 db_nslcmop_update["detailed-status"] = "Done"
2136                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2137                 db_nslcmop_update["statusEnteredTime"] = time()
2138                 if db_nslcmop["operationParams"].get("autoremove"):
2139                     autoremove = True
2140
2141         except (ROclient.ROClientException, DbException, LcmException) as e:
2142             self.logger.error(logging_text + "Exit Exception {}".format(e))
2143             exc = e
2144         except asyncio.CancelledError:
2145             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2146             exc = "Operation was cancelled"
2147         except Exception as e:
2148             exc = traceback.format_exc()
2149             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
2150         finally:
2151             if exc and db_nslcmop:
2152                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2153                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2154                 db_nslcmop_update["statusEnteredTime"] = time()
2155             try:
2156                 if db_nslcmop and db_nslcmop_update:
2157                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2158                 if db_nsr:
2159                     db_nsr_update["_admin.nslcmop"] = None
2160                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2161             except DbException as e:
2162                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2163             if nslcmop_operation_state:
2164                 try:
2165                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2166                                                                  "operationState": nslcmop_operation_state,
2167                                                                  "autoremove": autoremove},
2168                                             loop=self.loop)
2169                 except Exception as e:
2170                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2171             self.logger.debug(logging_text + "Exit")
2172             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
2173
2174     @staticmethod
2175     def _map_primitive_params(primitive_desc, params, instantiation_params):
2176         """
2177         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
2178         The default-value is used. If it is between < > it look for a value at instantiation_params
2179         :param primitive_desc: portion of VNFD/NSD that describes primitive
2180         :param params: Params provided by user
2181         :param instantiation_params: Instantiation params provided by user
2182         :return: a dictionary with the calculated params
2183         """
2184         calculated_params = {}
2185         for parameter in primitive_desc.get("parameter", ()):
2186             param_name = parameter["name"]
2187             if param_name in params:
2188                 calculated_params[param_name] = params[param_name]
2189             elif "default-value" in parameter or "value" in parameter:
2190                 if "value" in parameter:
2191                     calculated_params[param_name] = parameter["value"]
2192                 else:
2193                     calculated_params[param_name] = parameter["default-value"]
2194                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
2195                         and calculated_params[param_name].endswith(">"):
2196                     if calculated_params[param_name][1:-1] in instantiation_params:
2197                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
2198                     else:
2199                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
2200                                            format(parameter["default-value"], primitive_desc["name"]))
2201             else:
2202                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
2203                                    format(param_name, primitive_desc["name"]))
2204
2205             if isinstance(calculated_params[param_name], (dict, list, tuple)):
2206                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
2207                                                                width=256)
2208             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
2209                 calculated_params[param_name] = calculated_params[param_name][7:]
2210
2211         # add always ns_config_info if primitive name is config
2212         if primitive_desc["name"] == "config":
2213             if "ns_config_info" in instantiation_params:
2214                 calculated_params["ns_config_info"] = instantiation_params["ns_config_info"]
2215         return calculated_params
2216
2217     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
2218                                     primitive, primitive_params, retries=0, retries_interval=30):
2219         start_primitive_time = time()
2220         try:
2221             for vca_deployed in db_deployed["VCA"]:
2222                 if not vca_deployed:
2223                     continue
2224                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2225                     continue
2226                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2227                     continue
2228                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2229                     continue
2230                 break
2231             else:
2232                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2233                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2234             model_name = vca_deployed.get("model")
2235             application_name = vca_deployed.get("application")
2236             if not model_name or not application_name:
2237                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2238                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2239                                                                        vdu_count_index))
2240             # if vca_deployed["operational-status"] != "active":
2241             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2242             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2243             callback = None  # self.n2vc_callback
2244             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2245             await self.n2vc.login()
2246             if primitive == "config":
2247                 primitive_params = {"params": primitive_params}
2248             while retries >= 0:
2249                 primitive_id = await self.n2vc.ExecutePrimitive(
2250                     model_name,
2251                     application_name,
2252                     primitive,
2253                     callback,
2254                     *callback_args,
2255                     **primitive_params
2256                 )
2257                 while time() - start_primitive_time < self.timeout_primitive:
2258                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2259                     if primitive_result_ in ("completed", "failed"):
2260                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2261                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2262                         break
2263                     elif primitive_result_ is None and primitive == "config":
2264                         primitive_result = "COMPLETED"
2265                         detailed_result = None
2266                         break
2267                     else:  # ("running", "pending", None):
2268                         pass
2269                     await asyncio.sleep(5)
2270                 else:
2271                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2272                 if primitive_result == "COMPLETED":
2273                     break
2274                 retries -= 1
2275                 if retries >= 0:
2276                     await asyncio.sleep(retries_interval)
2277
2278             return primitive_result, detailed_result
2279         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2280             return "FAILED", str(e)
2281
2282     async def action(self, nsr_id, nslcmop_id):
2283
2284         # Try to lock HA task here
2285         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2286         if not task_is_locked_by_me:
2287             return
2288
2289         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2290         self.logger.debug(logging_text + "Enter")
2291         # get all needed from database
2292         db_nsr = None
2293         db_nslcmop = None
2294         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2295         db_nslcmop_update = {}
2296         nslcmop_operation_state = None
2297         nslcmop_operation_state_detail = None
2298         exc = None
2299         try:
2300             # wait for any previous tasks in process
2301             step = "Waiting for previous operations to terminate"
2302             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2303
2304             step = "Getting information from database"
2305             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2306             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2307
2308             nsr_deployed = db_nsr["_admin"].get("deployed")
2309             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2310             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2311             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2312             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2313
2314             if vnf_index:
2315                 step = "Getting vnfr from database"
2316                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2317                 step = "Getting vnfd from database"
2318                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2319             else:
2320                 if db_nsr.get("nsd"):
2321                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2322                 else:
2323                     step = "Getting nsd from database"
2324                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2325
2326             # for backward compatibility
2327             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2328                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2329                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2330                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2331
2332             primitive = db_nslcmop["operationParams"]["primitive"]
2333             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2334
2335             # look for primitive
2336             config_primitive_desc = None
2337             if vdu_id:
2338                 for vdu in get_iterable(db_vnfd, "vdu"):
2339                     if vdu_id == vdu["id"]:
2340                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2341                             if config_primitive["name"] == primitive:
2342                                 config_primitive_desc = config_primitive
2343                                 break
2344             elif vnf_index:
2345                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2346                     if config_primitive["name"] == primitive:
2347                         config_primitive_desc = config_primitive
2348                         break
2349             else:
2350                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2351                     if config_primitive["name"] == primitive:
2352                         config_primitive_desc = config_primitive
2353                         break
2354
2355             if not config_primitive_desc:
2356                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2357                                    format(primitive))
2358
2359             desc_params = {}
2360             if vnf_index:
2361                 if db_vnfr.get("additionalParamsForVnf"):
2362                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2363             else:
2364                 if db_nsr.get("additionalParamsForVnf"):
2365                     desc_params.update(db_nsr["additionalParamsForNs"])
2366
2367             # TODO check if ns is in a proper status
2368             result, result_detail = await self._ns_execute_primitive(
2369                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2370                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2371             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2372             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2373             db_nslcmop_update["statusEnteredTime"] = time()
2374             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2375             return  # database update is called inside finally
2376
2377         except (DbException, LcmException) as e:
2378             self.logger.error(logging_text + "Exit Exception {}".format(e))
2379             exc = e
2380         except asyncio.CancelledError:
2381             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2382             exc = "Operation was cancelled"
2383         except Exception as e:
2384             exc = traceback.format_exc()
2385             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2386         finally:
2387             if exc and db_nslcmop:
2388                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2389                     "FAILED {}: {}".format(step, exc)
2390                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2391                 db_nslcmop_update["statusEnteredTime"] = time()
2392             try:
2393                 if db_nslcmop_update:
2394                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2395                 if db_nsr:
2396                     db_nsr_update["_admin.nslcmop"] = None
2397                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2398             except DbException as e:
2399                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2400             self.logger.debug(logging_text + "Exit")
2401             if nslcmop_operation_state:
2402                 try:
2403                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2404                                                                "operationState": nslcmop_operation_state},
2405                                             loop=self.loop)
2406                 except Exception as e:
2407                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2408             self.logger.debug(logging_text + "Exit")
2409             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2410             return nslcmop_operation_state, nslcmop_operation_state_detail
2411
2412     async def scale(self, nsr_id, nslcmop_id):
2413
2414         # Try to lock HA task here
2415         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2416         if not task_is_locked_by_me:
2417             return
2418
2419         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2420         self.logger.debug(logging_text + "Enter")
2421         # get all needed from database
2422         db_nsr = None
2423         db_nslcmop = None
2424         db_nslcmop_update = {}
2425         nslcmop_operation_state = None
2426         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2427         exc = None
2428         # in case of error, indicates what part of scale was failed to put nsr at error status
2429         scale_process = None
2430         old_operational_status = ""
2431         old_config_status = ""
2432         vnfr_scaled = False
2433         try:
2434             # wait for any previous tasks in process
2435             step = "Waiting for previous operations to terminate"
2436             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2437
2438             step = "Getting nslcmop from database"
2439             self.logger.debug(step + " after having waited for previous tasks to be completed")
2440             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2441             step = "Getting nsr from database"
2442             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2443
2444             old_operational_status = db_nsr["operational-status"]
2445             old_config_status = db_nsr["config-status"]
2446             step = "Parsing scaling parameters"
2447             # self.logger.debug(step)
2448             db_nsr_update["operational-status"] = "scaling"
2449             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2450             nsr_deployed = db_nsr["_admin"].get("deployed")
2451             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2452             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2453             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2454             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2455             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2456
2457             # for backward compatibility
2458             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2459                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2460                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2461                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2462
2463             step = "Getting vnfr from database"
2464             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2465             step = "Getting vnfd from database"
2466             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2467
2468             step = "Getting scaling-group-descriptor"
2469             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2470                 if scaling_descriptor["name"] == scaling_group:
2471                     break
2472             else:
2473                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2474                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2475
2476             # cooldown_time = 0
2477             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2478             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2479             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2480             #         break
2481
2482             # TODO check if ns is in a proper status
2483             step = "Sending scale order to VIM"
2484             nb_scale_op = 0
2485             if not db_nsr["_admin"].get("scaling-group"):
2486                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2487                 admin_scale_index = 0
2488             else:
2489                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2490                     if admin_scale_info["name"] == scaling_group:
2491                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2492                         break
2493                 else:  # not found, set index one plus last element and add new entry with the name
2494                     admin_scale_index += 1
2495                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2496             RO_scaling_info = []
2497             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2498             if scaling_type == "SCALE_OUT":
2499                 # count if max-instance-count is reached
2500                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2501                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2502                 if nb_scale_op >= max_instance_count:
2503                     raise LcmException("reached the limit of {} (max-instance-count) "
2504                                        "scaling-out operations for the "
2505                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2506
2507                 nb_scale_op += 1
2508                 vdu_scaling_info["scaling_direction"] = "OUT"
2509                 vdu_scaling_info["vdu-create"] = {}
2510                 for vdu_scale_info in scaling_descriptor["vdu"]:
2511                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2512                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2513                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2514
2515             elif scaling_type == "SCALE_IN":
2516                 # count if min-instance-count is reached
2517                 min_instance_count = 0
2518                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2519                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2520                 if nb_scale_op <= min_instance_count:
2521                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2522                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2523                 nb_scale_op -= 1
2524                 vdu_scaling_info["scaling_direction"] = "IN"
2525                 vdu_scaling_info["vdu-delete"] = {}
2526                 for vdu_scale_info in scaling_descriptor["vdu"]:
2527                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2528                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2529                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2530
2531             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2532             vdu_create = vdu_scaling_info.get("vdu-create")
2533             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2534             if vdu_scaling_info["scaling_direction"] == "IN":
2535                 for vdur in reversed(db_vnfr["vdur"]):
2536                     if vdu_delete.get(vdur["vdu-id-ref"]):
2537                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2538                         vdu_scaling_info["vdu"].append({
2539                             "name": vdur["name"],
2540                             "vdu_id": vdur["vdu-id-ref"],
2541                             "interface": []
2542                         })
2543                         for interface in vdur["interfaces"]:
2544                             vdu_scaling_info["vdu"][-1]["interface"].append({
2545                                 "name": interface["name"],
2546                                 "ip_address": interface["ip-address"],
2547                                 "mac_address": interface.get("mac-address"),
2548                             })
2549                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2550
2551             # PRE-SCALE BEGIN
2552             step = "Executing pre-scale vnf-config-primitive"
2553             if scaling_descriptor.get("scaling-config-action"):
2554                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2555                     if (scaling_config_action.get("trigger") == "pre-scale-in" and scaling_type == "SCALE_IN") \
2556                        or (scaling_config_action.get("trigger") == "pre-scale-out" and scaling_type == "SCALE_OUT"):
2557                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2558                         step = db_nslcmop_update["detailed-status"] = \
2559                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2560
2561                         # look for primitive
2562                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2563                             if config_primitive["name"] == vnf_config_primitive:
2564                                 break
2565                         else:
2566                             raise LcmException(
2567                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2568                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2569                                 "primitive".format(scaling_group, config_primitive))
2570
2571                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2572                         if db_vnfr.get("additionalParamsForVnf"):
2573                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2574                         scale_process = "VCA"
2575                         db_nsr_update["config-status"] = "configuring pre-scaling"
2576                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
2577
2578                         # Pre-scale reintent check: Check if this sub-operation has been executed before
2579                         op_index = self._check_or_add_scale_suboperation(
2580                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
2581                         if (op_index == self.SUBOPERATION_STATUS_SKIP):
2582                             # Skip sub-operation
2583                             result = 'COMPLETED'
2584                             result_detail = 'Done'
2585                             self.logger.debug(logging_text +
2586                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
2587                                                   vnf_config_primitive, result, result_detail))
2588                         else:
2589                             if (op_index == self.SUBOPERATION_STATUS_NEW):
2590                                 # New sub-operation: Get index of this sub-operation
2591                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2592                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
2593                                                   format(vnf_config_primitive))
2594                             else:
2595                                 # Reintent:  Get registered params for this existing sub-operation
2596                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2597                                 vnf_index = op.get('member_vnf_index')
2598                                 vnf_config_primitive = op.get('primitive')
2599                                 primitive_params = op.get('primitive_params')
2600                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
2601                                                   format(vnf_config_primitive))
2602                             # Execute the primitive, either with new (first-time) or registered (reintent) args
2603                             result, result_detail = await self._ns_execute_primitive(
2604                                 nsr_deployed, vnf_index, None, None, None, vnf_config_primitive, primitive_params)
2605                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2606                                 vnf_config_primitive, result, result_detail))
2607                             # Update operationState = COMPLETED | FAILED
2608                             self._update_suboperation_status(
2609                                 db_nslcmop, op_index, result, result_detail)
2610
2611                         if result == "FAILED":
2612                             raise LcmException(result_detail)
2613                         db_nsr_update["config-status"] = old_config_status
2614                         scale_process = None
2615             # PRE-SCALE END
2616
2617             # SCALE RO - BEGIN
2618             # Should this block be skipped if 'RO_nsr_id' == None ?
2619             # if (RO_nsr_id and RO_scaling_info):
2620             if RO_scaling_info:
2621                 scale_process = "RO"
2622                 # Scale RO reintent check: Check if this sub-operation has been executed before
2623                 op_index = self._check_or_add_scale_suboperation(
2624                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
2625                 if (op_index == self.SUBOPERATION_STATUS_SKIP):
2626                     # Skip sub-operation
2627                     result = 'COMPLETED'
2628                     result_detail = 'Done'
2629                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
2630                         result, result_detail))
2631                 else:
2632                     if (op_index == self.SUBOPERATION_STATUS_NEW):
2633                         # New sub-operation: Get index of this sub-operation
2634                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2635                         self.logger.debug(logging_text + "New sub-operation RO")
2636                     else:
2637                         # Reintent:  Get registered params for this existing sub-operation
2638                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2639                         RO_nsr_id = op.get('RO_nsr_id')
2640                         RO_scaling_info = op.get('RO_scaling_info')
2641                         self.logger.debug(logging_text + "Sub-operation RO reintent".format(
2642                             vnf_config_primitive))
2643
2644                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2645                     db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2646                     db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2647                     # wait until ready
2648                     RO_nslcmop_id = RO_desc["instance_action_id"]
2649                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2650
2651                     RO_task_done = False
2652                     step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2653                     detailed_status_old = None
2654                     self.logger.debug(logging_text + step)
2655
2656                     deployment_timeout = 1 * 3600   # One hour
2657                     while deployment_timeout > 0:
2658                         if not RO_task_done:
2659                             desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2660                                                       extra_item_id=RO_nslcmop_id)
2661                             ns_status, ns_status_info = self.RO.check_action_status(desc)
2662                             if ns_status == "ERROR":
2663                                 raise ROclient.ROClientException(ns_status_info)
2664                             elif ns_status == "BUILD":
2665                                 detailed_status = step + "; {}".format(ns_status_info)
2666                             elif ns_status == "ACTIVE":
2667                                 RO_task_done = True
2668                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2669                                 self.logger.debug(logging_text + step)
2670                             else:
2671                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2672                         else:
2673                             if ns_status == "ERROR":
2674                                 raise ROclient.ROClientException(ns_status_info)
2675                             elif ns_status == "BUILD":
2676                                 detailed_status = step + "; {}".format(ns_status_info)
2677                             elif ns_status == "ACTIVE":
2678                                 step = detailed_status = \
2679                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
2680                                 if not vnfr_scaled:
2681                                     self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2682                                     vnfr_scaled = True
2683                                 try:
2684                                     desc = await self.RO.show("ns", RO_nsr_id)
2685                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2686                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2687                                     break
2688                                 except LcmExceptionNoMgmtIP:
2689                                     pass
2690                             else:
2691                                 assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2692                         if detailed_status != detailed_status_old:
2693                             self._update_suboperation_status(
2694                                 db_nslcmop, op_index, 'COMPLETED', detailed_status)
2695                             detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2696                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2697
2698                         await asyncio.sleep(5, loop=self.loop)
2699                         deployment_timeout -= 5
2700                     if deployment_timeout <= 0:
2701                         self._update_suboperation_status(
2702                             db_nslcmop, nslcmop_id, op_index, 'FAILED', "Timeout when waiting for ns to get ready")
2703                         raise ROclient.ROClientException("Timeout waiting ns to be ready")
2704
2705                     # update VDU_SCALING_INFO with the obtained ip_addresses
2706                     if vdu_scaling_info["scaling_direction"] == "OUT":
2707                         for vdur in reversed(db_vnfr["vdur"]):
2708                             if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2709                                 vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2710                                 vdu_scaling_info["vdu"].append({
2711                                     "name": vdur["name"],
2712                                     "vdu_id": vdur["vdu-id-ref"],
2713                                     "interface": []
2714                                 })
2715                                 for interface in vdur["interfaces"]:
2716                                     vdu_scaling_info["vdu"][-1]["interface"].append({
2717                                         "name": interface["name"],
2718                                         "ip_address": interface["ip-address"],
2719                                         "mac_address": interface.get("mac-address"),
2720                                     })
2721                         del vdu_scaling_info["vdu-create"]
2722
2723                     self._update_suboperation_status(db_nslcmop, op_index, 'COMPLETED', 'Done')
2724             # SCALE RO - END
2725
2726             scale_process = None
2727             if db_nsr_update:
2728                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2729
2730             # POST-SCALE BEGIN
2731             # execute primitive service POST-SCALING
2732             step = "Executing post-scale vnf-config-primitive"
2733             if scaling_descriptor.get("scaling-config-action"):
2734                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2735                     if (scaling_config_action.get("trigger") == "post-scale-in" and scaling_type == "SCALE_IN") \
2736                        or (scaling_config_action.get("trigger") == "post-scale-out" and scaling_type == "SCALE_OUT"):
2737                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2738                         step = db_nslcmop_update["detailed-status"] = \
2739                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2740
2741                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2742                         if db_vnfr.get("additionalParamsForVnf"):
2743                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2744
2745                         # look for primitive
2746                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2747                             if config_primitive["name"] == vnf_config_primitive:
2748                                 break
2749                         else:
2750                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2751                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2752                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2753                                                                                                      config_primitive))
2754                         scale_process = "VCA"
2755                         db_nsr_update["config-status"] = "configuring post-scaling"
2756                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
2757
2758                         # Post-scale reintent check: Check if this sub-operation has been executed before
2759                         op_index = self._check_or_add_scale_suboperation(
2760                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
2761                         if (op_index == self.SUBOPERATION_STATUS_SKIP):
2762                             # Skip sub-operation
2763                             result = 'COMPLETED'
2764                             result_detail = 'Done'
2765                             self.logger.debug(logging_text +
2766                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".
2767                                               format(vnf_config_primitive, result, result_detail))
2768                         else:
2769                             if (op_index == self.SUBOPERATION_STATUS_NEW):
2770                                 # New sub-operation: Get index of this sub-operation
2771                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2772                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
2773                                                   format(vnf_config_primitive))
2774                             else:
2775                                 # Reintent:  Get registered params for this existing sub-operation
2776                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2777                                 vnf_index = op.get('member_vnf_index')
2778                                 vnf_config_primitive = op.get('primitive')
2779                                 primitive_params = op.get('primitive_params')
2780                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
2781                                                   format(vnf_config_primitive))
2782                             # Execute the primitive, either with new (first-time) or registered (reintent) args
2783                             result, result_detail = await self._ns_execute_primitive(
2784                                 nsr_deployed, vnf_index, None, None, None, vnf_config_primitive, primitive_params)
2785                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2786                                 vnf_config_primitive, result, result_detail))
2787                             # Update operationState = COMPLETED | FAILED
2788                             self._update_suboperation_status(
2789                                 db_nslcmop, op_index, result, result_detail)
2790
2791                         if result == "FAILED":
2792                             raise LcmException(result_detail)
2793                         db_nsr_update["config-status"] = old_config_status
2794                         scale_process = None
2795             # POST-SCALE END
2796
2797             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2798             db_nslcmop_update["statusEnteredTime"] = time()
2799             db_nslcmop_update["detailed-status"] = "done"
2800             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2801             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2802                 else old_operational_status
2803             db_nsr_update["config-status"] = old_config_status
2804             return
2805         except (ROclient.ROClientException, DbException, LcmException) as e:
2806             self.logger.error(logging_text + "Exit Exception {}".format(e))
2807             exc = e
2808         except asyncio.CancelledError:
2809             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2810             exc = "Operation was cancelled"
2811         except Exception as e:
2812             exc = traceback.format_exc()
2813             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2814         finally:
2815             if exc:
2816                 if db_nslcmop:
2817                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2818                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2819                     db_nslcmop_update["statusEnteredTime"] = time()
2820                 if db_nsr:
2821                     db_nsr_update["operational-status"] = old_operational_status
2822                     db_nsr_update["config-status"] = old_config_status
2823                     db_nsr_update["detailed-status"] = ""
2824                     db_nsr_update["_admin.nslcmop"] = None
2825                     if scale_process:
2826                         if "VCA" in scale_process:
2827                             db_nsr_update["config-status"] = "failed"
2828                         if "RO" in scale_process:
2829                             db_nsr_update["operational-status"] = "failed"
2830                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2831                                                                                                      exc)
2832             try:
2833                 if db_nslcmop and db_nslcmop_update:
2834                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2835                 if db_nsr:
2836                     db_nsr_update["_admin.nslcmop"] = None
2837                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2838             except DbException as e:
2839                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2840             if nslcmop_operation_state:
2841                 try:
2842                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2843                                                              "operationState": nslcmop_operation_state},
2844                                             loop=self.loop)
2845                     # if cooldown_time:
2846                     #     await asyncio.sleep(cooldown_time)
2847                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2848                 except Exception as e:
2849                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2850             self.logger.debug(logging_text + "Exit")
2851             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")