osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Update target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 def deep_get(target_dict, key_list):
  71     """
  72     Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
  73     Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
  74     :param target_dict: dictionary to be read
  75     :param key_list: list of keys to read from  target_dict
  76     :return: The wanted value if exist, None otherwise
  77     """
  78     for key in key_list:
  79         if not isinstance(target_dict, dict) or key not in target_dict:
  80             return None
  81         target_dict = target_dict[key]
  82     return target_dict
  83
  84
  85 class NsLcm(LcmBase):
  86     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  87     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  88     timeout_charm_delete = 10 * 60
  89     timeout_primitive = 10 * 60  # timeout for primitive execution
  90
  91     SUBOPERATION_STATUS_NOT_FOUND = -1
  92     SUBOPERATION_STATUS_NEW = -2
  93     SUBOPERATION_STATUS_SKIP = -3
  94
  95     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  96         """
  97         Init, Connect to database, filesystem storage, and messaging
  98         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  99         :return: None
 100         """
 101         # logging
 102         self.logger = logging.getLogger('lcm.ns')
 103         self.loop = loop
 104         self.lcm_tasks = lcm_tasks
 105
 106         super().__init__(db, msg, fs, self.logger)
 107
 108         self.ro_config = ro_config
 109
 110         self.n2vc = N2VC(
 111             log=self.logger,
 112             server=vca_config['host'],
 113             port=vca_config['port'],
 114             user=vca_config['user'],
 115             secret=vca_config['secret'],
 116             # TODO: This should point to the base folder where charms are stored,
 117             # if there is a common one (like object storage). Otherwise, leave
 118             # it unset and pass it via DeployCharms
 119             # artifacts=vca_config[''],
 120             artifacts=None,
 121             juju_public_key=vca_config.get('pubkey'),
 122             ca_cert=vca_config.get('cacert'),
 123             api_proxy=vca_config.get('apiproxy')
 124         )
 125         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 126
 127     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 128         """
 129         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 130         :param vnfd: input vnfd
 131         :param new_id: overrides vnf id if provided
 132         :param additionalParams: Instantiation params for VNFs provided
 133         :param nsrId: Id of the NSR
 134         :return: copy of vnfd
 135         """
 136         try:
 137             vnfd_RO = deepcopy(vnfd)
 138             # remove unused by RO configuration, monitoring, scaling and internal keys
 139             vnfd_RO.pop("_id", None)
 140             vnfd_RO.pop("_admin", None)
 141             vnfd_RO.pop("vnf-configuration", None)
 142             vnfd_RO.pop("monitoring-param", None)
 143             vnfd_RO.pop("scaling-group-descriptor", None)
 144             if new_id:
 145                 vnfd_RO["id"] = new_id
 146
 147             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 148             for vdu in get_iterable(vnfd_RO, "vdu"):
 149                 cloud_init_file = None
 150                 if vdu.get("cloud-init-file"):
 151                     base_folder = vnfd["_admin"]["storage"]
 152                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 153                                                                    vdu["cloud-init-file"])
 154                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 155                         cloud_init_content = ci_file.read()
 156                     vdu.pop("cloud-init-file", None)
 157                 elif vdu.get("cloud-init"):
 158                     cloud_init_content = vdu["cloud-init"]
 159                 else:
 160                     continue
 161
 162                 env = Environment()
 163                 ast = env.parse(cloud_init_content)
 164                 mandatory_vars = meta.find_undeclared_variables(ast)
 165                 if mandatory_vars:
 166                     for var in mandatory_vars:
 167                         if not additionalParams or var not in additionalParams.keys():
 168                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 169                                                "file, must be provided in the instantiation parameters inside the "
 170                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 171                 template = Template(cloud_init_content)
 172                 cloud_init_content = template.render(additionalParams or {})
 173                 vdu["cloud-init"] = cloud_init_content
 174
 175             return vnfd_RO
 176         except FsException as e:
 177             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 178                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 179         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 180             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 181                                format(vnfd["id"], vdu["id"], e))
 182
 183     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 184         """
 185         Callback both for charm status change and task completion
 186         :param model_name: Charm model name
 187         :param application_name: Charm application name
 188         :param status: Can be
 189             - blocked: The unit needs manual intervention
 190             - maintenance: The unit is actively deploying/configuring
 191             - waiting: The unit is waiting for another charm to be ready
 192             - active: The unit is deployed, configured, and ready
 193             - error: The charm has failed and needs attention.
 194             - terminated: The charm has been destroyed
 195             - removing,
 196             - removed
 197         :param message: detailed message error
 198         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 199             nsr_id:
 200             nslcmop_id:
 201             lcmOperationType: currently "instantiate"
 202             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 203             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 204             n2vc_event: event used to notify instantiation task that some change has been produced
 205         :param task: None for charm status change, or task for completion task callback
 206         :return:
 207         """
 208         try:
 209             nsr_id = n2vc_info["nsr_id"]
 210             deployed = n2vc_info["deployed"]
 211             db_nsr_update = n2vc_info["db_update"]
 212             nslcmop_id = n2vc_info["nslcmop_id"]
 213             ns_operation = n2vc_info["lcmOperationType"]
 214             n2vc_event = n2vc_info["n2vc_event"]
 215             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 216                                                                                     application_name)
 217             for vca_index, vca_deployed in enumerate(deployed):
 218                 if not vca_deployed:
 219                     continue
 220                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 221                     break
 222             else:
 223                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 224                                   format(model_name))
 225                 return
 226             if task:
 227                 if task.cancelled():
 228                     self.logger.debug(logging_text + " task Cancelled")
 229                     vca_deployed['operational-status'] = "error"
 230                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 231                     vca_deployed['detailed-status'] = "Task Cancelled"
 232                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 233
 234                 elif task.done():
 235                     exc = task.exception()
 236                     if exc:
 237                         self.logger.error(logging_text + " task Exception={}".format(exc))
 238                         vca_deployed['operational-status'] = "error"
 239                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 240                         vca_deployed['detailed-status'] = str(exc)
 241                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 242                     else:
 243                         self.logger.debug(logging_text + " task Done")
 244                         # task is Done, but callback is still ongoing. So ignore
 245                         return
 246             elif status:
 247                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 248                 if vca_deployed['operational-status'] == status:
 249                     return  # same status, ignore
 250                 vca_deployed['operational-status'] = status
 251                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 252                 vca_deployed['detailed-status'] = str(message)
 253                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 254             else:
 255                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 256                 return
 257             # wake up instantiate task
 258             n2vc_event.set()
 259         except Exception as e:
 260             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 261
 262     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 263         """
 264         Creates a RO ns descriptor from OSM ns_instantiate params
 265         :param ns_params: OSM instantiate params
 266         :return: The RO ns descriptor
 267         """
 268         vim_2_RO = {}
 269         wim_2_RO = {}
 270         # TODO feature 1417: Check that no instantiation is set over PDU
 271         # check if PDU forces a concrete vim-network-id and add it
 272         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 273
 274         def vim_account_2_RO(vim_account):
 275             if vim_account in vim_2_RO:
 276                 return vim_2_RO[vim_account]
 277
 278             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 279             if db_vim["_admin"]["operationalState"] != "ENABLED":
 280                 raise LcmException("VIM={} is not available. operationalState={}".format(
 281                     vim_account, db_vim["_admin"]["operationalState"]))
 282             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 283             vim_2_RO[vim_account] = RO_vim_id
 284             return RO_vim_id
 285
 286         def wim_account_2_RO(wim_account):
 287             if isinstance(wim_account, str):
 288                 if wim_account in wim_2_RO:
 289                     return wim_2_RO[wim_account]
 290
 291                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 292                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 293                     raise LcmException("WIM={} is not available. operationalState={}".format(
 294                         wim_account, db_wim["_admin"]["operationalState"]))
 295                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 296                 wim_2_RO[wim_account] = RO_wim_id
 297                 return RO_wim_id
 298             else:
 299                 return wim_account
 300
 301         def ip_profile_2_RO(ip_profile):
 302             RO_ip_profile = deepcopy((ip_profile))
 303             if "dns-server" in RO_ip_profile:
 304                 if isinstance(RO_ip_profile["dns-server"], list):
 305                     RO_ip_profile["dns-address"] = []
 306                     for ds in RO_ip_profile.pop("dns-server"):
 307                         RO_ip_profile["dns-address"].append(ds['address'])
 308                 else:
 309                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 310             if RO_ip_profile.get("ip-version") == "ipv4":
 311                 RO_ip_profile["ip-version"] = "IPv4"
 312             if RO_ip_profile.get("ip-version") == "ipv6":
 313                 RO_ip_profile["ip-version"] = "IPv6"
 314             if "dhcp-params" in RO_ip_profile:
 315                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 316             return RO_ip_profile
 317
 318         if not ns_params:
 319             return None
 320         RO_ns_params = {
 321             # "name": ns_params["nsName"],
 322             # "description": ns_params.get("nsDescription"),
 323             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 324             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 325             # "scenario": ns_params["nsdId"],
 326         }
 327         n2vc_key_list = n2vc_key_list or []
 328         for vnfd_ref, vnfd in vnfd_dict.items():
 329             vdu_needed_access = []
 330             mgmt_cp = None
 331             if vnfd.get("vnf-configuration"):
 332                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 333                 if ssh_required and vnfd.get("mgmt-interface"):
 334                     if vnfd["mgmt-interface"].get("vdu-id"):
 335                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 336                     elif vnfd["mgmt-interface"].get("cp"):
 337                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 338
 339             for vdu in vnfd.get("vdu", ()):
 340                 if vdu.get("vdu-configuration"):
 341                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 342                     if ssh_required:
 343                         vdu_needed_access.append(vdu["id"])
 344                 elif mgmt_cp:
 345                     for vdu_interface in vdu.get("interface"):
 346                         if vdu_interface.get("external-connection-point-ref") and \
 347                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 348                             vdu_needed_access.append(vdu["id"])
 349                             mgmt_cp = None
 350                             break
 351
 352             if vdu_needed_access:
 353                 for vnf_member in nsd.get("constituent-vnfd"):
 354                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 355                         continue
 356                     for vdu in vdu_needed_access:
 357                         populate_dict(RO_ns_params,
 358                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 359                                       n2vc_key_list)
 360
 361         if ns_params.get("vduImage"):
 362             RO_ns_params["vduImage"] = ns_params["vduImage"]
 363
 364         if ns_params.get("ssh_keys"):
 365             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 366         for vnf_params in get_iterable(ns_params, "vnf"):
 367             for constituent_vnfd in nsd["constituent-vnfd"]:
 368                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 369                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 370                     break
 371             else:
 372                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 373                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 374             if vnf_params.get("vimAccountId"):
 375                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 376                               vim_account_2_RO(vnf_params["vimAccountId"]))
 377
 378             for vdu_params in get_iterable(vnf_params, "vdu"):
 379                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 380                 if vdu_params.get("volume"):
 381                     for volume_params in vdu_params["volume"]:
 382                         if volume_params.get("vim-volume-id"):
 383                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 384                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 385                                           volume_params["vim-volume-id"])
 386                 if vdu_params.get("interface"):
 387                     for interface_params in vdu_params["interface"]:
 388                         if interface_params.get("ip-address"):
 389                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 390                                                          vdu_params["id"], "interfaces", interface_params["name"],
 391                                                          "ip_address"),
 392                                           interface_params["ip-address"])
 393                         if interface_params.get("mac-address"):
 394                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 395                                                          vdu_params["id"], "interfaces", interface_params["name"],
 396                                                          "mac_address"),
 397                                           interface_params["mac-address"])
 398                         if interface_params.get("floating-ip-required"):
 399                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 400                                                          vdu_params["id"], "interfaces", interface_params["name"],
 401                                                          "floating-ip"),
 402                                           interface_params["floating-ip-required"])
 403
 404             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 405                 if internal_vld_params.get("vim-network-name"):
 406                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 407                                                  internal_vld_params["name"], "vim-network-name"),
 408                                   internal_vld_params["vim-network-name"])
 409                 if internal_vld_params.get("vim-network-id"):
 410                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 411                                                  internal_vld_params["name"], "vim-network-id"),
 412                                   internal_vld_params["vim-network-id"])
 413                 if internal_vld_params.get("ip-profile"):
 414                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 415                                                  internal_vld_params["name"], "ip-profile"),
 416                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 417                 if internal_vld_params.get("provider-network"):
 418
 419                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 420                                                  internal_vld_params["name"], "provider-network"),
 421                                   internal_vld_params["provider-network"].copy())
 422
 423                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 424                     # look for interface
 425                     iface_found = False
 426                     for vdu_descriptor in vnf_descriptor["vdu"]:
 427                         for vdu_interface in vdu_descriptor["interface"]:
 428                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 429                                 if icp_params.get("ip-address"):
 430                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 431                                                                  vdu_descriptor["id"], "interfaces",
 432                                                                  vdu_interface["name"], "ip_address"),
 433                                                   icp_params["ip-address"])
 434
 435                                 if icp_params.get("mac-address"):
 436                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 437                                                                  vdu_descriptor["id"], "interfaces",
 438                                                                  vdu_interface["name"], "mac_address"),
 439                                                   icp_params["mac-address"])
 440                                 iface_found = True
 441                                 break
 442                         if iface_found:
 443                             break
 444                     else:
 445                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 446                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 447                                            "connection-point".format(vnf_params["member-vnf-index"],
 448                                                                      icp_params["id-ref"]))
 449
 450         for vld_params in get_iterable(ns_params, "vld"):
 451             if "ip-profile" in vld_params:
 452                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 453                               ip_profile_2_RO(vld_params["ip-profile"]))
 454
 455             if vld_params.get("provider-network"):
 456
 457                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "provider-network"),
 458                               vld_params["provider-network"].copy())
 459
 460             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 461                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 462                               wim_account_2_RO(vld_params["wimAccountId"])),
 463             if vld_params.get("vim-network-name"):
 464                 RO_vld_sites = []
 465                 if isinstance(vld_params["vim-network-name"], dict):
 466                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 467                         RO_vld_sites.append({
 468                             "netmap-use": vim_net,
 469                             "datacenter": vim_account_2_RO(vim_account)
 470                         })
 471                 else:  # isinstance str
 472                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 473                 if RO_vld_sites:
 474                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 475
 476             if vld_params.get("vim-network-id"):
 477                 RO_vld_sites = []
 478                 if isinstance(vld_params["vim-network-id"], dict):
 479                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 480                         RO_vld_sites.append({
 481                             "netmap-use": vim_net,
 482                             "datacenter": vim_account_2_RO(vim_account)
 483                         })
 484                 else:  # isinstance str
 485                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 486                 if RO_vld_sites:
 487                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 488             if vld_params.get("ns-net"):
 489                 if isinstance(vld_params["ns-net"], dict):
 490                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 491                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 492                 if RO_vld_ns_net:
 493                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 494             if "vnfd-connection-point-ref" in vld_params:
 495                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 496                     # look for interface
 497                     for constituent_vnfd in nsd["constituent-vnfd"]:
 498                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 499                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 500                             break
 501                     else:
 502                         raise LcmException(
 503                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 504                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 505                     match_cp = False
 506                     for vdu_descriptor in vnf_descriptor["vdu"]:
 507                         for interface_descriptor in vdu_descriptor["interface"]:
 508                             if interface_descriptor.get("external-connection-point-ref") == \
 509                                     cp_params["vnfd-connection-point-ref"]:
 510                                 match_cp = True
 511                                 break
 512                         if match_cp:
 513                             break
 514                     else:
 515                         raise LcmException(
 516                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 517                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 518                                 cp_params["member-vnf-index-ref"],
 519                                 cp_params["vnfd-connection-point-ref"],
 520                                 vnf_descriptor["id"]))
 521                     if cp_params.get("ip-address"):
 522                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 523                                                      vdu_descriptor["id"], "interfaces",
 524                                                      interface_descriptor["name"], "ip_address"),
 525                                       cp_params["ip-address"])
 526                     if cp_params.get("mac-address"):
 527                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 528                                                      vdu_descriptor["id"], "interfaces",
 529                                                      interface_descriptor["name"], "mac_address"),
 530                                       cp_params["mac-address"])
 531         return RO_ns_params
 532
 533     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 534         # make a copy to do not change
 535         vdu_create = copy(vdu_create)
 536         vdu_delete = copy(vdu_delete)
 537
 538         vdurs = db_vnfr.get("vdur")
 539         if vdurs is None:
 540             vdurs = []
 541         vdu_index = len(vdurs)
 542         while vdu_index:
 543             vdu_index -= 1
 544             vdur = vdurs[vdu_index]
 545             if vdur.get("pdu-type"):
 546                 continue
 547             vdu_id_ref = vdur["vdu-id-ref"]
 548             if vdu_create and vdu_create.get(vdu_id_ref):
 549                 for index in range(0, vdu_create[vdu_id_ref]):
 550                     vdur = deepcopy(vdur)
 551                     vdur["_id"] = str(uuid4())
 552                     vdur["count-index"] += 1
 553                     vdurs.insert(vdu_index+1+index, vdur)
 554                 del vdu_create[vdu_id_ref]
 555             if vdu_delete and vdu_delete.get(vdu_id_ref):
 556                 del vdurs[vdu_index]
 557                 vdu_delete[vdu_id_ref] -= 1
 558                 if not vdu_delete[vdu_id_ref]:
 559                     del vdu_delete[vdu_id_ref]
 560         # check all operations are done
 561         if vdu_create or vdu_delete:
 562             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 563                 vdu_create))
 564         if vdu_delete:
 565             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 566                 vdu_delete))
 567
 568         vnfr_update = {"vdur": vdurs}
 569         db_vnfr["vdur"] = vdurs
 570         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 571
 572     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 573         """
 574         Updates database nsr with the RO info for the created vld
 575         :param ns_update_nsr: dictionary to be filled with the updated info
 576         :param db_nsr: content of db_nsr. This is also modified
 577         :param nsr_desc_RO: nsr descriptor from RO
 578         :return: Nothing, LcmException is raised on errors
 579         """
 580
 581         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 582             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 583                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 584                     continue
 585                 vld["vim-id"] = net_RO.get("vim_net_id")
 586                 vld["name"] = net_RO.get("vim_name")
 587                 vld["status"] = net_RO.get("status")
 588                 vld["status-detailed"] = net_RO.get("error_msg")
 589                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 590                 break
 591             else:
 592                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 593
 594     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 595         """
 596         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 597         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 598         :param nsr_desc_RO: nsr descriptor from RO
 599         :return: Nothing, LcmException is raised on errors
 600         """
 601         for vnf_index, db_vnfr in db_vnfrs.items():
 602             for vnf_RO in nsr_desc_RO["vnfs"]:
 603                 if vnf_RO["member_vnf_index"] != vnf_index:
 604                     continue
 605                 vnfr_update = {}
 606                 if vnf_RO.get("ip_address"):
 607                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 608                 elif not db_vnfr.get("ip-address"):
 609                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 610
 611                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 612                     vdur_RO_count_index = 0
 613                     if vdur.get("pdu-type"):
 614                         continue
 615                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 616                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 617                             continue
 618                         if vdur["count-index"] != vdur_RO_count_index:
 619                             vdur_RO_count_index += 1
 620                             continue
 621                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 622                         if vdur_RO.get("ip_address"):
 623                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 624                         else:
 625                             vdur["ip-address"] = None
 626                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 627                         vdur["name"] = vdur_RO.get("vim_name")
 628                         vdur["status"] = vdur_RO.get("status")
 629                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 630                         for ifacer in get_iterable(vdur, "interfaces"):
 631                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 632                                 if ifacer["name"] == interface_RO.get("internal_name"):
 633                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 634                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 635                                     break
 636                             else:
 637                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 638                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 639                                                                           ifacer["name"]))
 640                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 641                         break
 642                     else:
 643                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 644                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 645
 646                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 647                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 648                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 649                             continue
 650                         vld["vim-id"] = net_RO.get("vim_net_id")
 651                         vld["name"] = net_RO.get("vim_name")
 652                         vld["status"] = net_RO.get("status")
 653                         vld["status-detailed"] = net_RO.get("error_msg")
 654                         vnfr_update["vld.{}".format(vld_index)] = vld
 655                         break
 656                     else:
 657                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 658                             vnf_index, vld["id"]))
 659
 660                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 661                 break
 662
 663             else:
 664                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 665
 666     async def instantiate(self, nsr_id, nslcmop_id):
 667
 668         # Try to lock HA task here
 669         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 670         if not task_is_locked_by_me:
 671             return
 672
 673         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 674         self.logger.debug(logging_text + "Enter")
 675         # get all needed from database
 676         start_deploy = time()
 677         db_nsr = None
 678         db_nslcmop = None
 679         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 680         db_nslcmop_update = {}
 681         nslcmop_operation_state = None
 682         db_vnfrs = {}
 683         RO_descriptor_number = 0   # number of descriptors created at RO
 684         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 685         n2vc_info = {}
 686         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 687         exc = None
 688         try:
 689             # wait for any previous tasks in process
 690             step = "Waiting for previous tasks"
 691             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 692
 693             step = "Getting nslcmop={} from db".format(nslcmop_id)
 694             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 695             step = "Getting nsr={} from db".format(nsr_id)
 696             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 697             ns_params = db_nslcmop.get("operationParams")
 698             nsd = db_nsr["nsd"]
 699             nsr_name = db_nsr["name"]   # TODO short-name??
 700
 701             step = "Getting vnfrs from db"
 702             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 703             db_vnfds_ref = {}
 704             db_vnfds = {}
 705             db_vnfds_index = {}
 706             for vnfr in db_vnfrs_list:
 707                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 708                 vnfd_id = vnfr["vnfd-id"]
 709                 vnfd_ref = vnfr["vnfd-ref"]
 710                 if vnfd_id not in db_vnfds:
 711                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 712                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 713                     db_vnfds_ref[vnfd_ref] = vnfd
 714                     db_vnfds[vnfd_id] = vnfd
 715                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 716
 717             # Get or generates the _admin.deployed,VCA list
 718             vca_deployed_list = None
 719             vca_model_name = None
 720             if db_nsr["_admin"].get("deployed"):
 721                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 722                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 723             if vca_deployed_list is None:
 724                 vca_deployed_list = []
 725                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 726                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 727             elif isinstance(vca_deployed_list, dict):
 728                 # maintain backward compatibility. Change a dict to list at database
 729                 vca_deployed_list = list(vca_deployed_list.values())
 730                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 731                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 732
 733             db_nsr_update["detailed-status"] = "creating"
 734             db_nsr_update["operational-status"] = "init"
 735             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
 736                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 737                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 738
 739             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 740             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 741             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 742
 743             # Deploy charms
 744             # The parameters we'll need to deploy a charm
 745             number_to_configure = 0
 746
 747             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 748                 """An inner function to deploy the charm from either ns, vnf or vdu
 749                 For ns both vnf_index and vdu_id are None.
 750                 For vnf only vdu_id is None
 751                 For vdu both vnf_index and vdu_id contain a value
 752                 """
 753                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 754                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 755
 756                 machine_spec = {}
 757                 if native_charm:
 758                     machine_spec["username"] = charm_params.get("username")
 759                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 760
 761                 # Note: The charm needs to exist on disk at the location
 762                 # specified by charm_path.
 763                 descriptor = vnfd if vnf_index else nsd
 764                 base_folder = descriptor["_admin"]["storage"]
 765                 storage_params = self.fs.get_params()
 766                 charm_path = "{}{}/{}/charms/{}".format(
 767                     storage_params["path"],
 768                     base_folder["folder"],
 769                     base_folder["pkg-dir"],
 770                     proxy_charm
 771                 )
 772
 773                 # ns_name will be ignored in the current version of N2VC
 774                 # but will be implemented for the next point release.
 775                 model_name = nsr_id
 776                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 777                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 778                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 779
 780                 vca_index = len(vca_deployed_list)
 781                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 782                 # 26*26 charm in the same NS
 783                 application_name = application_name[0:48]
 784                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 785                 vca_deployed_ = {
 786                     "member-vnf-index": vnf_index,
 787                     "vdu_id": vdu_id,
 788                     "model": model_name,
 789                     "application": application_name,
 790                     "operational-status": "init",
 791                     "detailed-status": "",
 792                     "step": "initial-deploy",
 793                     "vnfd_id": vnfd_id,
 794                     "vdu_name": vdu_name,
 795                     "vdu_count_index": vdu_count_index,
 796                 }
 797                 vca_deployed_list.append(vca_deployed_)
 798                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 799                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 800
 801                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 802                                                                                                 proxy_charm))
 803                 if not n2vc_info:
 804                     n2vc_info["nsr_id"] = nsr_id
 805                     n2vc_info["nslcmop_id"] = nslcmop_id
 806                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 807                     n2vc_info["lcmOperationType"] = "instantiate"
 808                     n2vc_info["deployed"] = vca_deployed_list
 809                     n2vc_info["db_update"] = db_nsr_update
 810                 task = asyncio.ensure_future(
 811                     self.n2vc.DeployCharms(
 812                         model_name,          # The network service name
 813                         application_name,    # The application name
 814                         descriptor,          # The vnf/nsd descriptor
 815                         charm_path,          # Path to charm
 816                         charm_params,        # Runtime params, like mgmt ip
 817                         machine_spec,        # for native charms only
 818                         self.n2vc_callback,  # Callback for status changes
 819                         n2vc_info,           # Callback parameter
 820                         None,                # Callback parameter (task)
 821                     )
 822                 )
 823                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 824                                                          n2vc_info))
 825                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 826
 827             step = "Looking for needed vnfd to configure with proxy charm"
 828             self.logger.debug(logging_text + step)
 829
 830             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 831                 vnfd_id = c_vnf["vnfd-id-ref"]
 832                 vnf_index = str(c_vnf["member-vnf-index"])
 833                 vnfd = db_vnfds_ref[vnfd_id]
 834
 835                 # Get additional parameters
 836                 vnfr_params = {}
 837                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 838                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 839                 for k, v in vnfr_params.items():
 840                     if isinstance(v, str) and v.startswith("!!yaml "):
 841                         vnfr_params[k] = yaml.safe_load(v[7:])
 842
 843                 step = "deploying proxy charms for configuration"
 844                 # Check if this VNF has a charm configuration
 845                 vnf_config = vnfd.get("vnf-configuration")
 846                 if vnf_config and vnf_config.get("juju"):
 847                     proxy_charm = vnf_config["juju"]["charm"]
 848                     if vnf_config["juju"].get("proxy") is False:
 849                         # native_charm, will be deployed after VM. Skip
 850                         proxy_charm = None
 851
 852                     if proxy_charm:
 853                         if not vca_model_name:
 854                             step = "creating VCA model name '{}'".format(nsr_id)
 855                             self.logger.debug(logging_text + step)
 856                             await self.n2vc.CreateNetworkService(nsr_id)
 857                             vca_model_name = nsr_id
 858                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 859                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 860                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 861                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 862                         charm_params = {
 863                             "user_values": vnfr_params,
 864                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 865                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 866                         }
 867
 868                         # Login to the VCA. If there are multiple calls to login(),
 869                         # subsequent calls will be a nop and return immediately.
 870                         await self.n2vc.login()
 871
 872                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 873                         number_to_configure += 1
 874
 875                 # Deploy charms for each VDU that supports one.
 876                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 877                     vdu_config = vdu.get('vdu-configuration')
 878                     proxy_charm = None
 879
 880                     if vdu_config and vdu_config.get("juju"):
 881                         proxy_charm = vdu_config["juju"]["charm"]
 882                         if vdu_config["juju"].get("proxy") is False:
 883                             # native_charm, will be deployed after VM. Skip
 884                             proxy_charm = None
 885                         if proxy_charm:
 886                             if not vca_model_name:
 887                                 step = "creating VCA model name"
 888                                 await self.n2vc.CreateNetworkService(nsr_id)
 889                                 vca_model_name = nsr_id
 890                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 891                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 892                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 893                                                                                                           vdu["id"])
 894                             await self.n2vc.login()
 895                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 896                             # TODO for the moment only first vdu_id contains a charm deployed
 897                             if vdur["vdu-id-ref"] != vdu["id"]:
 898                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 899                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 900                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 901                             charm_params = {
 902                                 "user_values": vnfr_params,
 903                                 "rw_mgmt_ip": vdur["ip-address"],
 904                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 905                             }
 906                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 907                                          charm_params, n2vc_info)
 908                             number_to_configure += 1
 909
 910             # Check if this NS has a charm configuration
 911
 912             ns_config = nsd.get("ns-configuration")
 913             if ns_config and ns_config.get("juju"):
 914                 proxy_charm = ns_config["juju"]["charm"]
 915                 if ns_config["juju"].get("proxy") is False:
 916                     # native_charm, will be deployed after VM. Skip
 917                     proxy_charm = None
 918                 if proxy_charm:
 919                     step = "deploying proxy charm to configure ns"
 920                     # TODO is NS magmt IP address needed?
 921
 922                     # Get additional parameters
 923                     additional_params = {}
 924                     if db_nsr.get("additionalParamsForNs"):
 925                         additional_params = db_nsr["additionalParamsForNs"].copy()
 926                     for k, v in additional_params.items():
 927                         if isinstance(v, str) and v.startswith("!!yaml "):
 928                             additional_params[k] = yaml.safe_load(v[7:])
 929
 930                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 931                     charm_params = {
 932                         "user_values": additional_params,
 933                         # "rw_mgmt_ip": db_nsr["ip-address"],
 934                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 935                     }
 936
 937                     # Login to the VCA. If there are multiple calls to login(),
 938                     # subsequent calls will be a nop and return immediately.
 939                     await self.n2vc.login()
 940                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 941                     number_to_configure += 1
 942
 943             db_nsr_update["operational-status"] = "running"
 944
 945             # Wait until all charms has reached blocked or active status
 946             step = "waiting proxy charms to be ready"
 947             if number_to_configure:
 948                 # wait until all charms are configured.
 949                 # steps are:
 950                 #       initial-deploy
 951                 #       get-ssh-public-key
 952                 #       generate-ssh-key
 953                 #       retry-get-ssh-public-key
 954                 #       ssh-public-key-obtained
 955                 while time() <= start_deploy + self.total_deploy_timeout:
 956                     if db_nsr_update:
 957                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 958                     if db_nslcmop_update:
 959                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 960
 961                     all_active = True
 962                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 963                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 964                         if vca_deployed["step"] == "initial-deploy":
 965                             if vca_deployed["operational-status"] in ("active", "blocked"):
 966                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 967                                        .format(vca_deployed["member-vnf-index"],
 968                                                vca_deployed["vdu_id"])
 969                                 self.logger.debug(logging_text + step)
 970                                 try:
 971                                     primitive_id = await self.n2vc.ExecutePrimitive(
 972                                         vca_deployed["model"],
 973                                         vca_deployed["application"],
 974                                         "get-ssh-public-key",
 975                                         None,
 976                                     )
 977                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 978                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 979                                         primitive_id
 980                                     db_nsr_update[database_entry + "operational-status"] =\
 981                                         vca_deployed["operational-status"]
 982                                 except PrimitiveDoesNotExist:
 983                                     ssh_public_key = None
 984                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 985                                         "ssh-public-key-obtained"
 986                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 987                                         ssh_public_key
 988                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 989                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 990                                     self.logger.debug(logging_text + step)
 991
 992                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 993                             primitive_id = vca_deployed["primitive_id"]
 994                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 995                                                                                   primitive_id)
 996                             if primitive_status in ("completed", "failed"):
 997                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 998                                                                                       primitive_id)
 999                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
1000                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
1001                                         primitive_result.get("pubkey"):
1002                                     ssh_public_key = primitive_result.get("pubkey")
1003                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1004                                         "ssh-public-key-obtained"
1005                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
1006                                         ssh_public_key
1007                                     n2vc_key_list.append(ssh_public_key)
1008                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
1009                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
1010                                     self.logger.debug(logging_text + step)
1011                                 else:  # primitive_status == "failed":
1012                                     if vca_deployed["step"] == "get-ssh-public-key":
1013                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
1014                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
1015                                                                      vca_deployed["vdu_id"])
1016                                         self.logger.debug(logging_text + step)
1017                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1018                                             "generate-ssh-key"
1019                                         primitive_id = await self.n2vc.ExecutePrimitive(
1020                                             vca_deployed["model"],
1021                                             vca_deployed["application"],
1022                                             "generate-ssh-key",
1023                                             None,
1024                                         )
1025                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1026                                             primitive_id
1027                                     else:  # failed for second time
1028                                         raise LcmException(
1029                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
1030
1031                         elif vca_deployed["step"] == "generate-ssh-key":
1032                             primitive_id = vca_deployed["primitive_id"]
1033                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1034                                                                                   primitive_id)
1035                             if primitive_status in ("completed", "failed"):
1036                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1037                                                                                       primitive_id)
1038                                 vca_deployed["primitive_id"] = db_nsr_update[
1039                                     database_entry + "primitive_id"] = None
1040                                 if primitive_status == "completed":
1041                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1042                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1043                                                               vca_deployed["vdu_id"])
1044                                     self.logger.debug(logging_text + step)
1045                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1046                                         "retry-get-ssh-public-key"
1047                                     primitive_id = await self.n2vc.ExecutePrimitive(
1048                                         vca_deployed["model"],
1049                                         vca_deployed["application"],
1050                                         "get-ssh-public-key",
1051                                         None,
1052                                     )
1053                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1054                                         primitive_id
1055
1056                                 else:  # primitive_status == "failed":
1057                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1058                                                        .format(primitive_result))
1059
1060                         if vca_deployed["step"] != "ssh-public-key-obtained":
1061                             all_active = False
1062
1063                     if all_active:
1064                         break
1065                     await asyncio.sleep(5)
1066                 else:   # total_deploy_timeout
1067                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1068                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1069
1070             # deploy RO
1071             # get vnfds, instantiate at RO
1072             for c_vnf in nsd.get("constituent-vnfd", ()):
1073                 member_vnf_index = c_vnf["member-vnf-index"]
1074                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1075                 vnfd_ref = vnfd["id"]
1076                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1077                     vnfd_ref, member_vnf_index)
1078                 # self.logger.debug(logging_text + step)
1079                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1080                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1081                 RO_descriptor_number += 1
1082
1083                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1084                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1085                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1086                         break
1087                 else:
1088                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1089                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1090
1091                 # look if present
1092                 RO_update = {"member-vnf-index": member_vnf_index}
1093                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1094                 if vnfd_list:
1095                     RO_update["id"] = vnfd_list[0]["uuid"]
1096                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1097                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1098                 else:
1099                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1100                                            get("additionalParamsForVnf"), nsr_id)
1101                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1102                     RO_update["id"] = desc["uuid"]
1103                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1104                         vnfd_ref, member_vnf_index, desc["uuid"]))
1105                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1106                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1107                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1108
1109             # create nsd at RO
1110             nsd_ref = nsd["id"]
1111             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1112             # self.logger.debug(logging_text + step)
1113
1114             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1115             RO_descriptor_number += 1
1116             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1117             if nsd_list:
1118                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1119                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1120                     nsd_ref, RO_nsd_uuid))
1121             else:
1122                 nsd_RO = deepcopy(nsd)
1123                 nsd_RO["id"] = RO_osm_nsd_id
1124                 nsd_RO.pop("_id", None)
1125                 nsd_RO.pop("_admin", None)
1126                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1127                     member_vnf_index = c_vnf["member-vnf-index"]
1128                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1129                 for c_vld in nsd_RO.get("vld", ()):
1130                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1131                         member_vnf_index = cp["member-vnf-index-ref"]
1132                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1133
1134                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1135                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1136                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1137                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1138             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1139
1140             # Crate ns at RO
1141             # if present use it unless in error status
1142             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1143             if RO_nsr_id:
1144                 try:
1145                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1146                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1147                     desc = await self.RO.show("ns", RO_nsr_id)
1148                 except ROclient.ROClientException as e:
1149                     if e.http_code != HTTPStatus.NOT_FOUND:
1150                         raise
1151                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1152                 if RO_nsr_id:
1153                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1154                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1155                     if ns_status == "ERROR":
1156                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1157                         self.logger.debug(logging_text + step)
1158                         await self.RO.delete("ns", RO_nsr_id)
1159                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1160             if not RO_nsr_id:
1161                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1162                 # self.logger.debug(logging_text + step)
1163
1164                 # check if VIM is creating and wait  look if previous tasks in process
1165                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1166                 if task_dependency:
1167                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1168                     self.logger.debug(logging_text + step)
1169                     await asyncio.wait(task_dependency, timeout=3600)
1170                 if ns_params.get("vnf"):
1171                     for vnf in ns_params["vnf"]:
1172                         if "vimAccountId" in vnf:
1173                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1174                                                                                         vnf["vimAccountId"])
1175                         if task_dependency:
1176                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1177                             self.logger.debug(logging_text + step)
1178                             await asyncio.wait(task_dependency, timeout=3600)
1179
1180                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1181
1182                 # feature 1429. Add n2vc public key to needed VMs
1183                 n2vc_key = await self.n2vc.GetPublicKey()
1184                 n2vc_key_list.append(n2vc_key)
1185                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1186
1187                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1188                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1189                                             name=db_nsr["name"],
1190                                             scenario=RO_nsd_uuid)
1191                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1192                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1193                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1194                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1195             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1196
1197             # wait until NS is ready
1198             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1199             detailed_status_old = None
1200             self.logger.debug(logging_text + step)
1201
1202             while time() <= start_deploy + self.total_deploy_timeout:
1203                 desc = await self.RO.show("ns", RO_nsr_id)
1204                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1205                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1206                 if ns_status == "ERROR":
1207                     raise ROclient.ROClientException(ns_status_info)
1208                 elif ns_status == "BUILD":
1209                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1210                 elif ns_status == "ACTIVE":
1211                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1212                     try:
1213                         self.ns_update_vnfr(db_vnfrs, desc)
1214                         break
1215                     except LcmExceptionNoMgmtIP:
1216                         pass
1217                 else:
1218                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1219                 if detailed_status != detailed_status_old:
1220                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1221                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1222                 await asyncio.sleep(5, loop=self.loop)
1223             else:   # total_deploy_timeout
1224                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1225
1226             step = "Updating NSR"
1227             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1228
1229             db_nsr_update["operational-status"] = "running"
1230             db_nsr["detailed-status"] = "Configuring vnfr"
1231             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1232
1233             # Configure proxy charms once VMs are up
1234             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1235                 vnf_index = vca_deployed.get("member-vnf-index")
1236                 vdu_id = vca_deployed.get("vdu_id")
1237                 vdu_name = None
1238                 vdu_count_index = None
1239
1240                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1241                                                                                                            vdu_id)
1242                 add_params = {}
1243                 initial_config_primitive_list = []
1244                 if vnf_index:
1245                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1246                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1247                     vnfd = db_vnfds_index[vnf_index]
1248
1249                     if vdu_id:
1250                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1251                             if vdu["id"] == vdu_id:
1252                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1253                                     'initial-config-primitive', [])
1254                                 break
1255                         else:
1256                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1257                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1258                         # TODO for the moment only first vdu_id contains a charm deployed
1259                         if vdur["vdu-id-ref"] != vdu["id"]:
1260                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1261                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1262                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1263                     else:
1264                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1265                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1266                 else:
1267                     if db_nsr.get("additionalParamsForNs"):
1268                         add_params = db_nsr["additionalParamsForNs"].copy()
1269                     for k, v in add_params.items():
1270                         if isinstance(v, str) and v.startswith("!!yaml "):
1271                             add_params[k] = yaml.safe_load(v[7:])
1272                     add_params["rw_mgmt_ip"] = None
1273                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1274
1275                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1276                 initial_config_primitive_list = initial_config_primitive_list.copy()
1277                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1278                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1279
1280                 for initial_config_primitive in initial_config_primitive_list:
1281                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1282                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1283                                       .format(initial_config_primitive["name"], primitive_params_))
1284                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1285                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1286                         initial_config_primitive["name"],
1287                         primitive_params_,
1288                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1289                         retries_interval=30)
1290                     if primitive_result != "COMPLETED":
1291                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1292                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1293                                                    vca_deployed["vdu_id"], primitive_detail))
1294
1295             # Deploy native charms
1296             step = "Looking for needed vnfd to configure with native charm"
1297             self.logger.debug(logging_text + step)
1298
1299             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1300                 vnfd_id = c_vnf["vnfd-id-ref"]
1301                 vnf_index = str(c_vnf["member-vnf-index"])
1302                 vnfd = db_vnfds_ref[vnfd_id]
1303
1304                 # Get additional parameters
1305                 vnfr_params = {}
1306                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1307                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1308                 for k, v in vnfr_params.items():
1309                     if isinstance(v, str) and v.startswith("!!yaml "):
1310                         vnfr_params[k] = yaml.safe_load(v[7:])
1311
1312                 # Check if this VNF has a charm configuration
1313                 vnf_config = vnfd.get("vnf-configuration")
1314                 if vnf_config and vnf_config.get("juju"):
1315                     native_charm = vnf_config["juju"].get("proxy") is False
1316                     proxy_charm = vnf_config["juju"]["charm"]
1317                     if native_charm and proxy_charm:
1318                         if not vca_model_name:
1319                             step = "creating VCA model name '{}'".format(nsr_id)
1320                             self.logger.debug(logging_text + step)
1321                             await self.n2vc.CreateNetworkService(nsr_id)
1322                             vca_model_name = nsr_id
1323                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1324                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1325                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1326                         self.logger.debug(logging_text + step)
1327
1328                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1329                         charm_params = {
1330                             "user_values": vnfr_params,
1331                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1332                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1333                         }
1334
1335                         # get username
1336                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1337                         #  merged. Meanwhile let's get username from initial-config-primitive
1338                         if vnf_config.get("initial-config-primitive"):
1339                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1340                                 if param["name"] == "ssh-username":
1341                                     charm_params["username"] = param["value"]
1342                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1343                             if vnf_config["config-access"]["ssh-access"].get("required"):
1344                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1345
1346                         # Login to the VCA. If there are multiple calls to login(),
1347                         # subsequent calls will be a nop and return immediately.
1348                         await self.n2vc.login()
1349
1350                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1351                         number_to_configure += 1
1352
1353                 # Deploy charms for each VDU that supports one.
1354                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1355                     vdu_config = vdu.get('vdu-configuration')
1356                     native_charm = False
1357
1358                     if vdu_config and vdu_config.get("juju"):
1359                         native_charm = vdu_config["juju"].get("proxy") is False
1360                         proxy_charm = vdu_config["juju"]["charm"]
1361                         if native_charm and proxy_charm:
1362                             if not vca_model_name:
1363                                 step = "creating VCA model name"
1364                                 await self.n2vc.CreateNetworkService(nsr_id)
1365                                 vca_model_name = nsr_id
1366                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1367                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1368                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1369                                                                                                      vdu["id"])
1370
1371                             self.logger.debug(logging_text + step)
1372
1373                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1374
1375                             # TODO for the moment only first vdu_id contains a charm deployed
1376                             if vdur["vdu-id-ref"] != vdu["id"]:
1377                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1378                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1379                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1380                             charm_params = {
1381                                 "user_values": vnfr_params,
1382                                 "rw_mgmt_ip": vdur["ip-address"],
1383                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1384                             }
1385
1386                             # get username
1387                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1388                             #  merged. Meanwhile let's get username from initial-config-primitive
1389                             if vdu_config.get("initial-config-primitive"):
1390                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1391                                     if param["name"] == "ssh-username":
1392                                         charm_params["username"] = param["value"]
1393                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1394                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1395                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1396                                         "default-user")
1397
1398                             await self.n2vc.login()
1399
1400                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1401                                          charm_params, n2vc_info, native_charm)
1402                             number_to_configure += 1
1403
1404             # Check if this NS has a charm configuration
1405
1406             ns_config = nsd.get("ns-configuration")
1407             if ns_config and ns_config.get("juju"):
1408                 native_charm = ns_config["juju"].get("proxy") is False
1409                 proxy_charm = ns_config["juju"]["charm"]
1410                 if native_charm and proxy_charm:
1411                     step = "deploying native charm to configure ns"
1412                     # TODO is NS magmt IP address needed?
1413
1414                     # Get additional parameters
1415                     additional_params = {}
1416                     if db_nsr.get("additionalParamsForNs"):
1417                         additional_params = db_nsr["additionalParamsForNs"].copy()
1418                     for k, v in additional_params.items():
1419                         if isinstance(v, str) and v.startswith("!!yaml "):
1420                             additional_params[k] = yaml.safe_load(v[7:])
1421
1422                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1423                     charm_params = {
1424                         "user_values": additional_params,
1425                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1426                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1427                     }
1428
1429                     # get username
1430                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1431                     #  merged. Meanwhile let's get username from initial-config-primitive
1432                     if ns_config.get("initial-config-primitive"):
1433                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1434                             if param["name"] == "ssh-username":
1435                                 charm_params["username"] = param["value"]
1436                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1437                         if ns_config["config-access"]["ssh-access"].get("required"):
1438                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1439
1440                     # Login to the VCA. If there are multiple calls to login(),
1441                     # subsequent calls will be a nop and return immediately.
1442                     await self.n2vc.login()
1443                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1444                     number_to_configure += 1
1445
1446             # waiting all charms are ok
1447             configuration_failed = False
1448             if number_to_configure:
1449                 step = "Waiting all charms are active"
1450                 old_status = "configuring: init: {}".format(number_to_configure)
1451                 db_nsr_update["config-status"] = old_status
1452                 db_nsr_update["detailed-status"] = old_status
1453                 db_nslcmop_update["detailed-status"] = old_status
1454
1455                 # wait until all are configured.
1456                 while time() <= start_deploy + self.total_deploy_timeout:
1457                     if db_nsr_update:
1458                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1459                     if db_nslcmop_update:
1460                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1461                     # TODO add a fake task that set n2vc_event after some time
1462                     await n2vc_info["n2vc_event"].wait()
1463                     n2vc_info["n2vc_event"].clear()
1464                     all_active = True
1465                     status_map = {}
1466                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1467                     now = time()
1468                     for vca_deployed in vca_deployed_list:
1469                         vca_status = vca_deployed["operational-status"]
1470                         if vca_status not in status_map:
1471                             # Initialize it
1472                             status_map[vca_status] = 0
1473                         status_map[vca_status] += 1
1474
1475                         if vca_status == "active":
1476                             vca_deployed.pop("time_first_error", None)
1477                             vca_deployed.pop("status_first_error", None)
1478                             continue
1479
1480                         all_active = False
1481                         if vca_status in ("error", "blocked"):
1482                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1483                             # if not first time in this status error
1484                             if not vca_deployed.get("time_first_error"):
1485                                 vca_deployed["time_first_error"] = now
1486                                 continue
1487                         if vca_deployed.get("time_first_error") and \
1488                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1489                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1490                                                    .format(vca_deployed["member-vnf-index"],
1491                                                            vca_deployed["vdu_id"], vca_status,
1492                                                            vca_deployed["detailed-status-error"]))
1493
1494                     if all_active:
1495                         break
1496                     elif n2vc_error_text:
1497                         db_nsr_update["config-status"] = "failed"
1498                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1499                         db_nsr_update["detailed-status"] = error_text
1500                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1501                         db_nslcmop_update["detailed-status"] = error_text
1502                         db_nslcmop_update["statusEnteredTime"] = time()
1503                         configuration_failed = True
1504                         break
1505                     else:
1506                         cs = "configuring: "
1507                         separator = ""
1508                         for status, num in status_map.items():
1509                             cs += separator + "{}: {}".format(status, num)
1510                             separator = ", "
1511                         if old_status != cs:
1512                             db_nsr_update["config-status"] = cs
1513                             db_nsr_update["detailed-status"] = cs
1514                             db_nslcmop_update["detailed-status"] = cs
1515                             old_status = cs
1516                 else:   # total_deploy_timeout
1517                     raise LcmException("Timeout waiting ns to be configured")
1518
1519             if not configuration_failed:
1520                 # all is done
1521                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1522                 db_nslcmop_update["statusEnteredTime"] = time()
1523                 db_nslcmop_update["detailed-status"] = "done"
1524                 db_nsr_update["config-status"] = "configured"
1525                 db_nsr_update["detailed-status"] = "done"
1526
1527             return
1528
1529         except (ROclient.ROClientException, DbException, LcmException) as e:
1530             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1531             exc = e
1532         except asyncio.CancelledError:
1533             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1534             exc = "Operation was cancelled"
1535         except Exception as e:
1536             exc = traceback.format_exc()
1537             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1538                                  exc_info=True)
1539         finally:
1540             if exc:
1541                 if db_nsr:
1542                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1543                     db_nsr_update["operational-status"] = "failed"
1544                 if db_nslcmop:
1545                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1546                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1547                     db_nslcmop_update["statusEnteredTime"] = time()
1548             try:
1549                 if db_nsr:
1550                     db_nsr_update["_admin.nslcmop"] = None
1551                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1552                 if db_nslcmop_update:
1553                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1554             except DbException as e:
1555                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1556             if nslcmop_operation_state:
1557                 try:
1558                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1559                                                                    "operationState": nslcmop_operation_state},
1560                                             loop=self.loop)
1561                 except Exception as e:
1562                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1563
1564             self.logger.debug(logging_text + "Exit")
1565             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1566
1567     async def _destroy_charm(self, model, application):
1568         """
1569         Order N2VC destroy a charm
1570         :param model:
1571         :param application:
1572         :return: True if charm does not exist. False if it exist
1573         """
1574         if not await self.n2vc.HasApplication(model, application):
1575             return True  # Already removed
1576         await self.n2vc.RemoveCharms(model, application)
1577         return False
1578
1579     async def _wait_charm_destroyed(self, model, application, timeout):
1580         """
1581         Wait until charm does not exist
1582         :param model:
1583         :param application:
1584         :param timeout:
1585         :return: True if not exist, False if timeout
1586         """
1587         while True:
1588             if not await self.n2vc.HasApplication(model, application):
1589                 return True
1590             if timeout < 0:
1591                 return False
1592             await asyncio.sleep(10)
1593             timeout -= 10
1594
1595     # Check if this VNFD has a configured terminate action
1596     def _has_terminate_config_primitive(self, vnfd):
1597         vnf_config = vnfd.get("vnf-configuration")
1598         if vnf_config and vnf_config.get("terminate-config-primitive"):
1599             return True
1600         else:
1601             return False
1602
1603     @staticmethod
1604     def _get_terminate_config_primitive_seq_list(vnfd):
1605         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1606         # No need to check for existing primitive twice, already done before
1607         vnf_config = vnfd.get("vnf-configuration")
1608         seq_list = vnf_config.get("terminate-config-primitive")
1609         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1610         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1611         return seq_list_sorted
1612
1613     @staticmethod
1614     def _create_nslcmop(nsr_id, operation, params):
1615         """
1616         Creates a ns-lcm-opp content to be stored at database.
1617         :param nsr_id: internal id of the instance
1618         :param operation: instantiate, terminate, scale, action, ...
1619         :param params: user parameters for the operation
1620         :return: dictionary following SOL005 format
1621         """
1622         # Raise exception if invalid arguments
1623         if not (nsr_id and operation and params):
1624             raise LcmException(
1625                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1626         now = time()
1627         _id = str(uuid4())
1628         nslcmop = {
1629             "id": _id,
1630             "_id": _id,
1631             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1632             "operationState": "PROCESSING",
1633             "statusEnteredTime": now,
1634             "nsInstanceId": nsr_id,
1635             "lcmOperationType": operation,
1636             "startTime": now,
1637             "isAutomaticInvocation": False,
1638             "operationParams": params,
1639             "isCancelPending": False,
1640             "links": {
1641                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1642                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1643             }
1644         }
1645         return nslcmop
1646
1647     def _get_terminate_primitive_params(self, seq, vnf_index):
1648         primitive = seq.get('name')
1649         primitive_params = {}
1650         params = {
1651             "member_vnf_index": vnf_index,
1652             "primitive": primitive,
1653             "primitive_params": primitive_params,
1654         }
1655         desc_params = {}
1656         return self._map_primitive_params(seq, params, desc_params)
1657
1658     # sub-operations
1659
1660     def _reintent_or_skip_suboperation(self, db_nslcmop, op_index):
1661         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
1662         if (op.get('operationState') == 'COMPLETED'):
1663             # b. Skip sub-operation
1664             # _ns_execute_primitive() or RO.create_action() will NOT be executed
1665             return self.SUBOPERATION_STATUS_SKIP
1666         else:
1667             # c. Reintent executing sub-operation
1668             # The sub-operation exists, and operationState != 'COMPLETED'
1669             # Update operationState = 'PROCESSING' to indicate a reintent.
1670             operationState = 'PROCESSING'
1671             detailed_status = 'In progress'
1672             self._update_suboperation_status(
1673                 db_nslcmop, op_index, operationState, detailed_status)
1674             # Return the sub-operation index
1675             # _ns_execute_primitive() or RO.create_action() will be called from scale()
1676             # with arguments extracted from the sub-operation
1677             return op_index
1678
1679     # Find a sub-operation where all keys in a matching dictionary must match
1680     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
1681     def _find_suboperation(self, db_nslcmop, match):
1682         if (db_nslcmop and match):
1683             op_list = db_nslcmop.get('_admin', {}).get('operations', [])
1684             for i, op in enumerate(op_list):
1685                 if all(op.get(k) == match[k] for k in match):
1686                     return i
1687         return self.SUBOPERATION_STATUS_NOT_FOUND
1688
1689     # Update status for a sub-operation given its index
1690     def _update_suboperation_status(self, db_nslcmop, op_index, operationState, detailed_status):
1691         # Update DB for HA tasks
1692         q_filter = {'_id': db_nslcmop['_id']}
1693         update_dict = {'_admin.operations.{}.operationState'.format(op_index): operationState,
1694                        '_admin.operations.{}.detailed-status'.format(op_index): detailed_status}
1695         self.db.set_one("nslcmops",
1696                         q_filter=q_filter,
1697                         update_dict=update_dict,
1698                         fail_on_empty=False)
1699
1700     # Add sub-operation, return the index of the added sub-operation
1701     # Optionally, set operationState, detailed-status, and operationType
1702     # Status and type are currently set for 'scale' sub-operations:
1703     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
1704     # 'detailed-status' : status message
1705     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
1706     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
1707     def _add_suboperation(self, db_nslcmop, vnf_index, vdu_id, vdu_count_index,
1708                           vdu_name, primitive, mapped_primitive_params,
1709                           operationState=None, detailed_status=None, operationType=None,
1710                           RO_nsr_id=None, RO_scaling_info=None):
1711         if not (db_nslcmop):
1712             return self.SUBOPERATION_STATUS_NOT_FOUND
1713         # Get the "_admin.operations" list, if it exists
1714         db_nslcmop_admin = db_nslcmop.get('_admin', {})
1715         op_list = db_nslcmop_admin.get('operations')
1716         # Create or append to the "_admin.operations" list
1717         new_op = {'member_vnf_index': vnf_index,
1718                   'vdu_id': vdu_id,
1719                   'vdu_count_index': vdu_count_index,
1720                   'primitive': primitive,
1721                   'primitive_params': mapped_primitive_params}
1722         if operationState:
1723             new_op['operationState'] = operationState
1724         if detailed_status:
1725             new_op['detailed-status'] = detailed_status
1726         if operationType:
1727             new_op['lcmOperationType'] = operationType
1728         if RO_nsr_id:
1729             new_op['RO_nsr_id'] = RO_nsr_id
1730         if RO_scaling_info:
1731             new_op['RO_scaling_info'] = RO_scaling_info
1732         if not op_list:
1733             # No existing operations, create key 'operations' with current operation as first list element
1734             db_nslcmop_admin.update({'operations': [new_op]})
1735             op_list = db_nslcmop_admin.get('operations')
1736         else:
1737             # Existing operations, append operation to list
1738             op_list.append(new_op)
1739
1740         db_nslcmop_update = {'_admin.operations': op_list}
1741         self.update_db_2("nslcmops", db_nslcmop['_id'], db_nslcmop_update)
1742         op_index = len(op_list) - 1
1743         return op_index
1744
1745     # Helper methods for scale() sub-operations
1746
1747     # pre-scale/post-scale:
1748     # Check for 3 different cases:
1749     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
1750     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
1751     # c. Reintent: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
1752     def _check_or_add_scale_suboperation(self, db_nslcmop, vnf_index,
1753                                          vnf_config_primitive, primitive_params, operationType,
1754                                          RO_nsr_id=None, RO_scaling_info=None):
1755         # Find this sub-operation
1756         if (RO_nsr_id and RO_scaling_info):
1757             operationType = 'SCALE-RO'
1758             match = {
1759                 'member_vnf_index': vnf_index,
1760                 'RO_nsr_id': RO_nsr_id,
1761                 'RO_scaling_info': RO_scaling_info,
1762             }
1763         else:
1764             match = {
1765                 'member_vnf_index': vnf_index,
1766                 'primitive': vnf_config_primitive,
1767                 'primitive_params': primitive_params,
1768                 'lcmOperationType': operationType
1769             }
1770         op_index = self._find_suboperation(db_nslcmop, match)
1771         if (op_index == self.SUBOPERATION_STATUS_NOT_FOUND):
1772             # a. New sub-operation
1773             # The sub-operation does not exist, add it.
1774             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
1775             # The following parameters are set to None for all kind of scaling:
1776             vdu_id = None
1777             vdu_count_index = None
1778             vdu_name = None
1779             if (RO_nsr_id and RO_scaling_info):
1780                 vnf_config_primitive = None
1781                 primitive_params = None
1782             else:
1783                 RO_nsr_id = None
1784                 RO_scaling_info = None
1785             # Initial status for sub-operation
1786             operationState = 'PROCESSING'
1787             detailed_status = 'In progress'
1788             # Add sub-operation for pre/post-scaling (zero or more operations)
1789             self._add_suboperation(db_nslcmop,
1790                                    vnf_index,
1791                                    vdu_id,
1792                                    vdu_count_index,
1793                                    vdu_name,
1794                                    vnf_config_primitive,
1795                                    primitive_params,
1796                                    operationState,
1797                                    detailed_status,
1798                                    operationType,
1799                                    RO_nsr_id,
1800                                    RO_scaling_info)
1801             return self.SUBOPERATION_STATUS_NEW
1802         else:
1803             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
1804             # or op_index (operationState != 'COMPLETED')
1805             return self._reintent_or_skip_suboperation(db_nslcmop, op_index)
1806
1807     # Helper methods for terminate()
1808
1809     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1810         """ Create a primitive with params from VNFD
1811             Called from terminate() before deleting instance
1812             Calls action() to execute the primitive """
1813         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1814         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1815         db_vnfds = {}
1816         # Loop over VNFRs
1817         for vnfr in db_vnfrs_list:
1818             vnfd_id = vnfr["vnfd-id"]
1819             vnf_index = vnfr["member-vnf-index-ref"]
1820             if vnfd_id not in db_vnfds:
1821                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1822                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1823                 db_vnfds[vnfd_id] = vnfd
1824             vnfd = db_vnfds[vnfd_id]
1825             if not self._has_terminate_config_primitive(vnfd):
1826                 continue
1827             # Get the primitive's sorted sequence list
1828             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1829             for seq in seq_list:
1830                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1831                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1832                     vnf_index, seq.get("name"))
1833                 self.logger.debug(logging_text + step)
1834                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1835                 primitive = seq.get('name')
1836                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1837                 # The following 3 parameters are currently set to None for 'terminate':
1838                 # vdu_id, vdu_count_index, vdu_name
1839                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1840                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1841                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1842                 # Add sub-operation
1843                 self._add_suboperation(db_nslcmop,
1844                                        nslcmop_id,
1845                                        vnf_index,
1846                                        vdu_id,
1847                                        vdu_count_index,
1848                                        vdu_name,
1849                                        primitive,
1850                                        mapped_primitive_params)
1851                 # Sub-operations: Call _ns_execute_primitive() instead of action()
1852                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1853                 nsr_deployed = db_nsr["_admin"]["deployed"]
1854                 result, result_detail = await self._ns_execute_primitive(
1855                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1856                     mapped_primitive_params)
1857
1858                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1859                 #    nsr_id, nslcmop_terminate_action_id)
1860                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1861                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1862                 if result not in result_ok:
1863                     raise LcmException(
1864                         "terminate_primitive_action for vnf_member_index={}",
1865                         " primitive={} fails with error {}".format(
1866                             vnf_index, seq.get("name"), result_detail))
1867
1868     async def terminate(self, nsr_id, nslcmop_id):
1869
1870         # Try to lock HA task here
1871         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1872         if not task_is_locked_by_me:
1873             return
1874
1875         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1876         self.logger.debug(logging_text + "Enter")
1877         db_nsr = None
1878         db_nslcmop = None
1879         exc = None
1880         failed_detail = []   # annotates all failed error messages
1881         vca_time_destroy = None   # time of where destroy charm order
1882         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1883         db_nslcmop_update = {}
1884         nslcmop_operation_state = None
1885         autoremove = False  # autoremove after terminated
1886         try:
1887             # wait for any previous tasks in process
1888             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1889
1890             step = "Getting nslcmop={} from db".format(nslcmop_id)
1891             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1892             step = "Getting nsr={} from db".format(nsr_id)
1893             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1894             # nsd = db_nsr["nsd"]
1895             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1896             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1897                 return
1898             # #TODO check if VIM is creating and wait
1899             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1900             # Call internal terminate action
1901             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1902
1903             db_nsr_update["operational-status"] = "terminating"
1904             db_nsr_update["config-status"] = "terminating"
1905
1906             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1907                 vca_model_name = nsr_deployed["VCA-model-name"]
1908                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1909                 self.logger.debug(logging_text + step)
1910                 try:
1911                     await self.n2vc.DestroyNetworkService(vca_model_name)
1912                 except NetworkServiceDoesNotExist:
1913                     pass
1914                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1915                 if nsr_deployed.get("VCA"):
1916                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1917                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1918                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1919             # for backward compatibility if charm have been created with "default"  model name delete one by one
1920             elif nsr_deployed and nsr_deployed.get("VCA"):
1921                 try:
1922                     step = "Scheduling configuration charms removing"
1923                     db_nsr_update["detailed-status"] = "Deleting charms"
1924                     self.logger.debug(logging_text + step)
1925                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1926                     # for backward compatibility
1927                     if isinstance(nsr_deployed["VCA"], dict):
1928                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1929                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1930                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1931
1932                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1933                         if vca_deployed:
1934                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1935                                 vca_deployed.clear()
1936                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1937                             else:
1938                                 vca_time_destroy = time()
1939                 except Exception as e:
1940                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1941
1942             # remove from RO
1943             RO_fail = False
1944
1945             # Delete ns
1946             RO_nsr_id = RO_delete_action = None
1947             if nsr_deployed and nsr_deployed.get("RO"):
1948                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1949                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1950             try:
1951                 if RO_nsr_id:
1952                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1953                         "Deleting ns from VIM"
1954                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1955                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1956                     self.logger.debug(logging_text + step)
1957                     desc = await self.RO.delete("ns", RO_nsr_id)
1958                     RO_delete_action = desc["action_id"]
1959                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1960                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1961                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1962                 if RO_delete_action:
1963                     # wait until NS is deleted from VIM
1964                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1965                         format(RO_nsr_id, RO_delete_action)
1966                     detailed_status_old = None
1967                     self.logger.debug(logging_text + step)
1968
1969                     delete_timeout = 20 * 60   # 20 minutes
1970                     while delete_timeout > 0:
1971                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1972                                                   extra_item_id=RO_delete_action)
1973                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1974                         if ns_status == "ERROR":
1975                             raise ROclient.ROClientException(ns_status_info)
1976                         elif ns_status == "BUILD":
1977                             detailed_status = step + "; {}".format(ns_status_info)
1978                         elif ns_status == "ACTIVE":
1979                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1980                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1981                             break
1982                         else:
1983                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1984                         if detailed_status != detailed_status_old:
1985                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1986                                 db_nsr_update["detailed-status"] = detailed_status
1987                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1988                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1989                         await asyncio.sleep(5, loop=self.loop)
1990                         delete_timeout -= 5
1991                     else:  # delete_timeout <= 0:
1992                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1993
1994             except ROclient.ROClientException as e:
1995                 if e.http_code == 404:  # not found
1996                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1997                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1998                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1999                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
2000                 elif e.http_code == 409:   # conflict
2001                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
2002                     self.logger.debug(logging_text + failed_detail[-1])
2003                     RO_fail = True
2004                 else:
2005                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
2006                     self.logger.error(logging_text + failed_detail[-1])
2007                     RO_fail = True
2008
2009             # Delete nsd
2010             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
2011                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
2012                 try:
2013                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
2014                         "Deleting nsd from RO"
2015                     await self.RO.delete("nsd", RO_nsd_id)
2016                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
2017                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2018                 except ROclient.ROClientException as e:
2019                     if e.http_code == 404:  # not found
2020                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
2021                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
2022                     elif e.http_code == 409:   # conflict
2023                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
2024                         self.logger.debug(logging_text + failed_detail[-1])
2025                         RO_fail = True
2026                     else:
2027                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
2028                         self.logger.error(logging_text + failed_detail[-1])
2029                         RO_fail = True
2030
2031             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
2032                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
2033                     if not vnf_deployed or not vnf_deployed["id"]:
2034                         continue
2035                     try:
2036                         RO_vnfd_id = vnf_deployed["id"]
2037                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
2038                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
2039                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
2040                         await self.RO.delete("vnfd", RO_vnfd_id)
2041                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
2042                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2043                     except ROclient.ROClientException as e:
2044                         if e.http_code == 404:  # not found
2045                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
2046                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
2047                         elif e.http_code == 409:   # conflict
2048                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
2049                             self.logger.debug(logging_text + failed_detail[-1])
2050                         else:
2051                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
2052                             self.logger.error(logging_text + failed_detail[-1])
2053
2054             # wait until charm deleted
2055             if vca_time_destroy:
2056                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
2057                     "Waiting for deletion of configuration charms"
2058                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2059                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2060                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
2061                     if not vca_deployed:
2062                         continue
2063                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
2064                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
2065                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
2066                                                             timeout):
2067                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
2068                             vca_deployed["application"]))
2069                     else:
2070                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
2071
2072             if failed_detail:
2073                 self.logger.error(logging_text + " ;".join(failed_detail))
2074                 db_nsr_update["operational-status"] = "failed"
2075                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
2076                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
2077                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2078                 db_nslcmop_update["statusEnteredTime"] = time()
2079             else:
2080                 db_nsr_update["operational-status"] = "terminated"
2081                 db_nsr_update["detailed-status"] = "Done"
2082                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
2083                 db_nslcmop_update["detailed-status"] = "Done"
2084                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2085                 db_nslcmop_update["statusEnteredTime"] = time()
2086                 if db_nslcmop["operationParams"].get("autoremove"):
2087                     autoremove = True
2088
2089         except (ROclient.ROClientException, DbException, LcmException) as e:
2090             self.logger.error(logging_text + "Exit Exception {}".format(e))
2091             exc = e
2092         except asyncio.CancelledError:
2093             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2094             exc = "Operation was cancelled"
2095         except Exception as e:
2096             exc = traceback.format_exc()
2097             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
2098         finally:
2099             if exc and db_nslcmop:
2100                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2101                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2102                 db_nslcmop_update["statusEnteredTime"] = time()
2103             try:
2104                 if db_nslcmop and db_nslcmop_update:
2105                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2106                 if db_nsr:
2107                     db_nsr_update["_admin.nslcmop"] = None
2108                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2109             except DbException as e:
2110                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2111             if nslcmop_operation_state:
2112                 try:
2113                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2114                                                                  "operationState": nslcmop_operation_state,
2115                                                                  "autoremove": autoremove},
2116                                             loop=self.loop)
2117                 except Exception as e:
2118                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2119             self.logger.debug(logging_text + "Exit")
2120             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
2121
2122     @staticmethod
2123     def _map_primitive_params(primitive_desc, params, instantiation_params):
2124         """
2125         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
2126         The default-value is used. If it is between < > it look for a value at instantiation_params
2127         :param primitive_desc: portion of VNFD/NSD that describes primitive
2128         :param params: Params provided by user
2129         :param instantiation_params: Instantiation params provided by user
2130         :return: a dictionary with the calculated params
2131         """
2132         calculated_params = {}
2133         for parameter in primitive_desc.get("parameter", ()):
2134             param_name = parameter["name"]
2135             if param_name in params:
2136                 calculated_params[param_name] = params[param_name]
2137             elif "default-value" in parameter or "value" in parameter:
2138                 if "value" in parameter:
2139                     calculated_params[param_name] = parameter["value"]
2140                 else:
2141                     calculated_params[param_name] = parameter["default-value"]
2142                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
2143                         and calculated_params[param_name].endswith(">"):
2144                     if calculated_params[param_name][1:-1] in instantiation_params:
2145                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
2146                     else:
2147                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
2148                                            format(parameter["default-value"], primitive_desc["name"]))
2149             else:
2150                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
2151                                    format(param_name, primitive_desc["name"]))
2152
2153             if isinstance(calculated_params[param_name], (dict, list, tuple)):
2154                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
2155                                                                width=256)
2156             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
2157                 calculated_params[param_name] = calculated_params[param_name][7:]
2158         return calculated_params
2159
2160     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
2161                                     primitive, primitive_params, retries=0, retries_interval=30):
2162         start_primitive_time = time()
2163         try:
2164             for vca_deployed in db_deployed["VCA"]:
2165                 if not vca_deployed:
2166                     continue
2167                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2168                     continue
2169                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2170                     continue
2171                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2172                     continue
2173                 break
2174             else:
2175                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2176                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2177             model_name = vca_deployed.get("model")
2178             application_name = vca_deployed.get("application")
2179             if not model_name or not application_name:
2180                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2181                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2182                                                                        vdu_count_index))
2183             # if vca_deployed["operational-status"] != "active":
2184             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2185             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2186             callback = None  # self.n2vc_callback
2187             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2188             await self.n2vc.login()
2189             if primitive == "config":
2190                 primitive_params = {"params": primitive_params}
2191             while retries >= 0:
2192                 primitive_id = await self.n2vc.ExecutePrimitive(
2193                     model_name,
2194                     application_name,
2195                     primitive,
2196                     callback,
2197                     *callback_args,
2198                     **primitive_params
2199                 )
2200                 while time() - start_primitive_time < self.timeout_primitive:
2201                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2202                     if primitive_result_ in ("completed", "failed"):
2203                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2204                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2205                         break
2206                     elif primitive_result_ is None and primitive == "config":
2207                         primitive_result = "COMPLETED"
2208                         detailed_result = None
2209                         break
2210                     else:  # ("running", "pending", None):
2211                         pass
2212                     await asyncio.sleep(5)
2213                 else:
2214                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2215                 if primitive_result == "COMPLETED":
2216                     break
2217                 retries -= 1
2218                 if retries >= 0:
2219                     await asyncio.sleep(retries_interval)
2220
2221             return primitive_result, detailed_result
2222         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2223             return "FAILED", str(e)
2224
2225     async def action(self, nsr_id, nslcmop_id):
2226
2227         # Try to lock HA task here
2228         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2229         if not task_is_locked_by_me:
2230             return
2231
2232         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2233         self.logger.debug(logging_text + "Enter")
2234         # get all needed from database
2235         db_nsr = None
2236         db_nslcmop = None
2237         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2238         db_nslcmop_update = {}
2239         nslcmop_operation_state = None
2240         nslcmop_operation_state_detail = None
2241         exc = None
2242         try:
2243             # wait for any previous tasks in process
2244             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2245
2246             step = "Getting information from database"
2247             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2248             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2249
2250             nsr_deployed = db_nsr["_admin"].get("deployed")
2251             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2252             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2253             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2254             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2255
2256             if vnf_index:
2257                 step = "Getting vnfr from database"
2258                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2259                 step = "Getting vnfd from database"
2260                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2261             else:
2262                 if db_nsr.get("nsd"):
2263                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2264                 else:
2265                     step = "Getting nsd from database"
2266                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2267
2268             # for backward compatibility
2269             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2270                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2271                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2272                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2273
2274             primitive = db_nslcmop["operationParams"]["primitive"]
2275             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2276
2277             # look for primitive
2278             config_primitive_desc = None
2279             if vdu_id:
2280                 for vdu in get_iterable(db_vnfd, "vdu"):
2281                     if vdu_id == vdu["id"]:
2282                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2283                             if config_primitive["name"] == primitive:
2284                                 config_primitive_desc = config_primitive
2285                                 break
2286             elif vnf_index:
2287                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2288                     if config_primitive["name"] == primitive:
2289                         config_primitive_desc = config_primitive
2290                         break
2291             else:
2292                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2293                     if config_primitive["name"] == primitive:
2294                         config_primitive_desc = config_primitive
2295                         break
2296
2297             if not config_primitive_desc:
2298                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2299                                    format(primitive))
2300
2301             desc_params = {}
2302             if vnf_index:
2303                 if db_vnfr.get("additionalParamsForVnf"):
2304                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2305             else:
2306                 if db_nsr.get("additionalParamsForVnf"):
2307                     desc_params.update(db_nsr["additionalParamsForNs"])
2308
2309             # TODO check if ns is in a proper status
2310             result, result_detail = await self._ns_execute_primitive(
2311                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2312                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2313             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2314             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2315             db_nslcmop_update["statusEnteredTime"] = time()
2316             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2317             return  # database update is called inside finally
2318
2319         except (DbException, LcmException) as e:
2320             self.logger.error(logging_text + "Exit Exception {}".format(e))
2321             exc = e
2322         except asyncio.CancelledError:
2323             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2324             exc = "Operation was cancelled"
2325         except Exception as e:
2326             exc = traceback.format_exc()
2327             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2328         finally:
2329             if exc and db_nslcmop:
2330                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2331                     "FAILED {}: {}".format(step, exc)
2332                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2333                 db_nslcmop_update["statusEnteredTime"] = time()
2334             try:
2335                 if db_nslcmop_update:
2336                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2337                 if db_nsr:
2338                     db_nsr_update["_admin.nslcmop"] = None
2339                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2340             except DbException as e:
2341                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2342             self.logger.debug(logging_text + "Exit")
2343             if nslcmop_operation_state:
2344                 try:
2345                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2346                                                                "operationState": nslcmop_operation_state},
2347                                             loop=self.loop)
2348                 except Exception as e:
2349                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2350             self.logger.debug(logging_text + "Exit")
2351             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2352             return nslcmop_operation_state, nslcmop_operation_state_detail
2353
2354     async def scale(self, nsr_id, nslcmop_id):
2355
2356         # Try to lock HA task here
2357         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2358         if not task_is_locked_by_me:
2359             return
2360
2361         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2362         self.logger.debug(logging_text + "Enter")
2363         # get all needed from database
2364         db_nsr = None
2365         db_nslcmop = None
2366         db_nslcmop_update = {}
2367         nslcmop_operation_state = None
2368         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2369         exc = None
2370         # in case of error, indicates what part of scale was failed to put nsr at error status
2371         scale_process = None
2372         old_operational_status = ""
2373         old_config_status = ""
2374         vnfr_scaled = False
2375         try:
2376             # wait for any previous tasks in process
2377             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2378
2379             step = "Getting nslcmop from database"
2380             self.logger.debug(step + " after having waited for previous tasks to be completed")
2381             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2382             step = "Getting nsr from database"
2383             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2384
2385             old_operational_status = db_nsr["operational-status"]
2386             old_config_status = db_nsr["config-status"]
2387             step = "Parsing scaling parameters"
2388             # self.logger.debug(step)
2389             db_nsr_update["operational-status"] = "scaling"
2390             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2391             nsr_deployed = db_nsr["_admin"].get("deployed")
2392             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2393             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2394             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2395             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2396             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2397
2398             # for backward compatibility
2399             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2400                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2401                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2402                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2403
2404             step = "Getting vnfr from database"
2405             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2406             step = "Getting vnfd from database"
2407             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2408
2409             step = "Getting scaling-group-descriptor"
2410             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2411                 if scaling_descriptor["name"] == scaling_group:
2412                     break
2413             else:
2414                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2415                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2416
2417             # cooldown_time = 0
2418             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2419             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2420             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2421             #         break
2422
2423             # TODO check if ns is in a proper status
2424             step = "Sending scale order to VIM"
2425             nb_scale_op = 0
2426             if not db_nsr["_admin"].get("scaling-group"):
2427                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2428                 admin_scale_index = 0
2429             else:
2430                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2431                     if admin_scale_info["name"] == scaling_group:
2432                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2433                         break
2434                 else:  # not found, set index one plus last element and add new entry with the name
2435                     admin_scale_index += 1
2436                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2437             RO_scaling_info = []
2438             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2439             if scaling_type == "SCALE_OUT":
2440                 # count if max-instance-count is reached
2441                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2442                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2443                 if nb_scale_op >= max_instance_count:
2444                     raise LcmException("reached the limit of {} (max-instance-count) "
2445                                        "scaling-out operations for the "
2446                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2447
2448                 nb_scale_op += 1
2449                 vdu_scaling_info["scaling_direction"] = "OUT"
2450                 vdu_scaling_info["vdu-create"] = {}
2451                 for vdu_scale_info in scaling_descriptor["vdu"]:
2452                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2453                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2454                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2455
2456             elif scaling_type == "SCALE_IN":
2457                 # count if min-instance-count is reached
2458                 min_instance_count = 0
2459                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2460                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2461                 if nb_scale_op <= min_instance_count:
2462                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2463                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2464                 nb_scale_op -= 1
2465                 vdu_scaling_info["scaling_direction"] = "IN"
2466                 vdu_scaling_info["vdu-delete"] = {}
2467                 for vdu_scale_info in scaling_descriptor["vdu"]:
2468                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2469                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2470                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2471
2472             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2473             vdu_create = vdu_scaling_info.get("vdu-create")
2474             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2475             if vdu_scaling_info["scaling_direction"] == "IN":
2476                 for vdur in reversed(db_vnfr["vdur"]):
2477                     if vdu_delete.get(vdur["vdu-id-ref"]):
2478                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2479                         vdu_scaling_info["vdu"].append({
2480                             "name": vdur["name"],
2481                             "vdu_id": vdur["vdu-id-ref"],
2482                             "interface": []
2483                         })
2484                         for interface in vdur["interfaces"]:
2485                             vdu_scaling_info["vdu"][-1]["interface"].append({
2486                                 "name": interface["name"],
2487                                 "ip_address": interface["ip-address"],
2488                                 "mac_address": interface.get("mac-address"),
2489                             })
2490                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2491
2492             # PRE-SCALE BEGIN
2493             step = "Executing pre-scale vnf-config-primitive"
2494             if scaling_descriptor.get("scaling-config-action"):
2495                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2496                     if (scaling_config_action.get("trigger") == "pre-scale-in" and scaling_type == "SCALE_IN") \
2497                        or (scaling_config_action.get("trigger") == "pre-scale-out" and scaling_type == "SCALE_OUT"):
2498                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2499                         step = db_nslcmop_update["detailed-status"] = \
2500                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2501
2502                         # look for primitive
2503                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2504                             if config_primitive["name"] == vnf_config_primitive:
2505                                 break
2506                         else:
2507                             raise LcmException(
2508                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2509                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2510                                 "primitive".format(scaling_group, config_primitive))
2511
2512                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2513                         if db_vnfr.get("additionalParamsForVnf"):
2514                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2515                         scale_process = "VCA"
2516                         db_nsr_update["config-status"] = "configuring pre-scaling"
2517                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
2518
2519                         # Pre-scale reintent check: Check if this sub-operation has been executed before
2520                         op_index = self._check_or_add_scale_suboperation(
2521                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'PRE-SCALE')
2522                         if (op_index == self.SUBOPERATION_STATUS_SKIP):
2523                             # Skip sub-operation
2524                             result = 'COMPLETED'
2525                             result_detail = 'Done'
2526                             self.logger.debug(logging_text +
2527                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
2528                                                   vnf_config_primitive, result, result_detail))
2529                         else:
2530                             if (op_index == self.SUBOPERATION_STATUS_NEW):
2531                                 # New sub-operation: Get index of this sub-operation
2532                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2533                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
2534                                                   format(vnf_config_primitive))
2535                             else:
2536                                 # Reintent:  Get registered params for this existing sub-operation
2537                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2538                                 vnf_index = op.get('member_vnf_index')
2539                                 vnf_config_primitive = op.get('primitive')
2540                                 primitive_params = op.get('primitive_params')
2541                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
2542                                                   format(vnf_config_primitive))
2543                             # Execute the primitive, either with new (first-time) or registered (reintent) args
2544                             result, result_detail = await self._ns_execute_primitive(
2545                                 nsr_deployed, vnf_index, None, None, None, vnf_config_primitive, primitive_params)
2546                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2547                                 vnf_config_primitive, result, result_detail))
2548                             # Update operationState = COMPLETED | FAILED
2549                             self._update_suboperation_status(
2550                                 db_nslcmop, op_index, result, result_detail)
2551
2552                         if result == "FAILED":
2553                             raise LcmException(result_detail)
2554                         db_nsr_update["config-status"] = old_config_status
2555                         scale_process = None
2556             # PRE-SCALE END
2557
2558             # SCALE RO - BEGIN
2559             # Should this block be skipped if 'RO_nsr_id' == None ?
2560             # if (RO_nsr_id and RO_scaling_info):
2561             if RO_scaling_info:
2562                 scale_process = "RO"
2563                 # Scale RO reintent check: Check if this sub-operation has been executed before
2564                 op_index = self._check_or_add_scale_suboperation(
2565                     db_nslcmop, vnf_index, None, None, 'SCALE-RO', RO_nsr_id, RO_scaling_info)
2566                 if (op_index == self.SUBOPERATION_STATUS_SKIP):
2567                     # Skip sub-operation
2568                     result = 'COMPLETED'
2569                     result_detail = 'Done'
2570                     self.logger.debug(logging_text + "Skipped sub-operation RO, result {} {}".format(
2571                         result, result_detail))
2572                 else:
2573                     if (op_index == self.SUBOPERATION_STATUS_NEW):
2574                         # New sub-operation: Get index of this sub-operation
2575                         op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2576                         self.logger.debug(logging_text + "New sub-operation RO")
2577                     else:
2578                         # Reintent:  Get registered params for this existing sub-operation
2579                         op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2580                         RO_nsr_id = op.get('RO_nsr_id')
2581                         RO_scaling_info = op.get('RO_scaling_info')
2582                         self.logger.debug(logging_text + "Sub-operation RO reintent".format(
2583                             vnf_config_primitive))
2584
2585                     RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2586                     db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2587                     db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2588                     # wait until ready
2589                     RO_nslcmop_id = RO_desc["instance_action_id"]
2590                     db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2591
2592                     RO_task_done = False
2593                     step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2594                     detailed_status_old = None
2595                     self.logger.debug(logging_text + step)
2596
2597                     deployment_timeout = 1 * 3600   # One hour
2598                     while deployment_timeout > 0:
2599                         if not RO_task_done:
2600                             desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2601                                                       extra_item_id=RO_nslcmop_id)
2602                             ns_status, ns_status_info = self.RO.check_action_status(desc)
2603                             if ns_status == "ERROR":
2604                                 raise ROclient.ROClientException(ns_status_info)
2605                             elif ns_status == "BUILD":
2606                                 detailed_status = step + "; {}".format(ns_status_info)
2607                             elif ns_status == "ACTIVE":
2608                                 RO_task_done = True
2609                                 step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2610                                 self.logger.debug(logging_text + step)
2611                             else:
2612                                 assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2613                         else:
2614                             if ns_status == "ERROR":
2615                                 raise ROclient.ROClientException(ns_status_info)
2616                             elif ns_status == "BUILD":
2617                                 detailed_status = step + "; {}".format(ns_status_info)
2618                             elif ns_status == "ACTIVE":
2619                                 step = detailed_status = \
2620                                     "Waiting for management IP address reported by the VIM. Updating VNFRs"
2621                                 if not vnfr_scaled:
2622                                     self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2623                                     vnfr_scaled = True
2624                                 try:
2625                                     desc = await self.RO.show("ns", RO_nsr_id)
2626                                     # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2627                                     self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2628                                     break
2629                                 except LcmExceptionNoMgmtIP:
2630                                     pass
2631                             else:
2632                                 assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2633                         if detailed_status != detailed_status_old:
2634                             self._update_suboperation_status(
2635                                 db_nslcmop, op_index, 'COMPLETED', detailed_status)
2636                             detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2637                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2638
2639                         await asyncio.sleep(5, loop=self.loop)
2640                         deployment_timeout -= 5
2641                     if deployment_timeout <= 0:
2642                         self._update_suboperation_status(
2643                             db_nslcmop, nslcmop_id, op_index, 'FAILED', "Timeout when waiting for ns to get ready")
2644                         raise ROclient.ROClientException("Timeout waiting ns to be ready")
2645
2646                     # update VDU_SCALING_INFO with the obtained ip_addresses
2647                     if vdu_scaling_info["scaling_direction"] == "OUT":
2648                         for vdur in reversed(db_vnfr["vdur"]):
2649                             if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2650                                 vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2651                                 vdu_scaling_info["vdu"].append({
2652                                     "name": vdur["name"],
2653                                     "vdu_id": vdur["vdu-id-ref"],
2654                                     "interface": []
2655                                 })
2656                                 for interface in vdur["interfaces"]:
2657                                     vdu_scaling_info["vdu"][-1]["interface"].append({
2658                                         "name": interface["name"],
2659                                         "ip_address": interface["ip-address"],
2660                                         "mac_address": interface.get("mac-address"),
2661                                     })
2662                         del vdu_scaling_info["vdu-create"]
2663
2664                     self._update_suboperation_status(db_nslcmop, op_index, 'COMPLETED', 'Done')
2665             # SCALE RO - END
2666
2667             scale_process = None
2668             if db_nsr_update:
2669                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2670
2671             # POST-SCALE BEGIN
2672             # execute primitive service POST-SCALING
2673             step = "Executing post-scale vnf-config-primitive"
2674             if scaling_descriptor.get("scaling-config-action"):
2675                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2676                     if (scaling_config_action.get("trigger") == "post-scale-in" and scaling_type == "SCALE_IN") \
2677                        or (scaling_config_action.get("trigger") == "post-scale-out" and scaling_type == "SCALE_OUT"):
2678                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2679                         step = db_nslcmop_update["detailed-status"] = \
2680                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2681
2682                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2683                         if db_vnfr.get("additionalParamsForVnf"):
2684                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2685
2686                         # look for primitive
2687                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2688                             if config_primitive["name"] == vnf_config_primitive:
2689                                 break
2690                         else:
2691                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2692                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2693                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2694                                                                                                      config_primitive))
2695                         scale_process = "VCA"
2696                         db_nsr_update["config-status"] = "configuring post-scaling"
2697                         primitive_params = self._map_primitive_params(config_primitive, {}, vnfr_params)
2698
2699                         # Post-scale reintent check: Check if this sub-operation has been executed before
2700                         op_index = self._check_or_add_scale_suboperation(
2701                             db_nslcmop, nslcmop_id, vnf_index, vnf_config_primitive, primitive_params, 'POST-SCALE')
2702                         if (op_index == self.SUBOPERATION_STATUS_SKIP):
2703                             # Skip sub-operation
2704                             result = 'COMPLETED'
2705                             result_detail = 'Done'
2706                             self.logger.debug(logging_text +
2707                                               "vnf_config_primitive={} Skipped sub-operation, result {} {}".
2708                                               format(vnf_config_primitive, result, result_detail))
2709                         else:
2710                             if (op_index == self.SUBOPERATION_STATUS_NEW):
2711                                 # New sub-operation: Get index of this sub-operation
2712                                 op_index = len(db_nslcmop.get('_admin', {}).get('operations')) - 1
2713                                 self.logger.debug(logging_text + "vnf_config_primitive={} New sub-operation".
2714                                                   format(vnf_config_primitive))
2715                             else:
2716                                 # Reintent:  Get registered params for this existing sub-operation
2717                                 op = db_nslcmop.get('_admin', {}).get('operations', [])[op_index]
2718                                 vnf_index = op.get('member_vnf_index')
2719                                 vnf_config_primitive = op.get('primitive')
2720                                 primitive_params = op.get('primitive_params')
2721                                 self.logger.debug(logging_text + "vnf_config_primitive={} Sub-operation reintent".
2722                                                   format(vnf_config_primitive))
2723                             # Execute the primitive, either with new (first-time) or registered (reintent) args
2724                             result, result_detail = await self._ns_execute_primitive(
2725                                 nsr_deployed, vnf_index, None, None, None, vnf_config_primitive, primitive_params)
2726                             self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2727                                 vnf_config_primitive, result, result_detail))
2728                             # Update operationState = COMPLETED | FAILED
2729                             self._update_suboperation_status(
2730                                 db_nslcmop, op_index, result, result_detail)
2731
2732                         if result == "FAILED":
2733                             raise LcmException(result_detail)
2734                         db_nsr_update["config-status"] = old_config_status
2735                         scale_process = None
2736             # POST-SCALE END
2737
2738             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2739             db_nslcmop_update["statusEnteredTime"] = time()
2740             db_nslcmop_update["detailed-status"] = "done"
2741             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2742             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2743                 else old_operational_status
2744             db_nsr_update["config-status"] = old_config_status
2745             return
2746         except (ROclient.ROClientException, DbException, LcmException) as e:
2747             self.logger.error(logging_text + "Exit Exception {}".format(e))
2748             exc = e
2749         except asyncio.CancelledError:
2750             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2751             exc = "Operation was cancelled"
2752         except Exception as e:
2753             exc = traceback.format_exc()
2754             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2755         finally:
2756             if exc:
2757                 if db_nslcmop:
2758                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2759                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2760                     db_nslcmop_update["statusEnteredTime"] = time()
2761                 if db_nsr:
2762                     db_nsr_update["operational-status"] = old_operational_status
2763                     db_nsr_update["config-status"] = old_config_status
2764                     db_nsr_update["detailed-status"] = ""
2765                     db_nsr_update["_admin.nslcmop"] = None
2766                     if scale_process:
2767                         if "VCA" in scale_process:
2768                             db_nsr_update["config-status"] = "failed"
2769                         if "RO" in scale_process:
2770                             db_nsr_update["operational-status"] = "failed"
2771                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2772                                                                                                      exc)
2773             try:
2774                 if db_nslcmop and db_nslcmop_update:
2775                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2776                 if db_nsr:
2777                     db_nsr_update["_admin.nslcmop"] = None
2778                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2779             except DbException as e:
2780                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2781             if nslcmop_operation_state:
2782                 try:
2783                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2784                                                              "operationState": nslcmop_operation_state},
2785                                             loop=self.loop)
2786                     # if cooldown_time:
2787                     #     await asyncio.sleep(cooldown_time)
2788                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2789                 except Exception as e:
2790                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2791             self.logger.debug(logging_text + "Exit")
2792             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")