osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102         )
 103
 104     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 105         """
 106         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 107         :param vnfd: input vnfd
 108         :param new_id: overrides vnf id if provided
 109         :param additionalParams: Instantiation params for VNFs provided
 110         :param nsrId: Id of the NSR
 111         :return: copy of vnfd
 112         """
 113         try:
 114             vnfd_RO = deepcopy(vnfd)
 115             # remove unused by RO configuration, monitoring, scaling and internal keys
 116             vnfd_RO.pop("_id", None)
 117             vnfd_RO.pop("_admin", None)
 118             vnfd_RO.pop("vnf-configuration", None)
 119             vnfd_RO.pop("monitoring-param", None)
 120             vnfd_RO.pop("scaling-group-descriptor", None)
 121             if new_id:
 122                 vnfd_RO["id"] = new_id
 123
 124             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 125             for vdu in get_iterable(vnfd_RO, "vdu"):
 126                 cloud_init_file = None
 127                 if vdu.get("cloud-init-file"):
 128                     base_folder = vnfd["_admin"]["storage"]
 129                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 130                                                                    vdu["cloud-init-file"])
 131                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 132                         cloud_init_content = ci_file.read()
 133                     vdu.pop("cloud-init-file", None)
 134                 elif vdu.get("cloud-init"):
 135                     cloud_init_content = vdu["cloud-init"]
 136                 else:
 137                     continue
 138
 139                 env = Environment()
 140                 ast = env.parse(cloud_init_content)
 141                 mandatory_vars = meta.find_undeclared_variables(ast)
 142                 if mandatory_vars:
 143                     for var in mandatory_vars:
 144                         if not additionalParams or var not in additionalParams.keys():
 145                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 146                                                "file, must be provided in the instantiation parameters inside the "
 147                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 148                 template = Template(cloud_init_content)
 149                 cloud_init_content = template.render(additionalParams or {})
 150                 vdu["cloud-init"] = cloud_init_content
 151
 152             return vnfd_RO
 153         except FsException as e:
 154             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 155                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 156         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 157             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 158                                format(vnfd["id"], vdu["id"], e))
 159
 160     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 161         """
 162         Callback both for charm status change and task completion
 163         :param model_name: Charm model name
 164         :param application_name: Charm application name
 165         :param status: Can be
 166             - blocked: The unit needs manual intervention
 167             - maintenance: The unit is actively deploying/configuring
 168             - waiting: The unit is waiting for another charm to be ready
 169             - active: The unit is deployed, configured, and ready
 170             - error: The charm has failed and needs attention.
 171             - terminated: The charm has been destroyed
 172             - removing,
 173             - removed
 174         :param message: detailed message error
 175         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 176             nsr_id:
 177             nslcmop_id:
 178             lcmOperationType: currently "instantiate"
 179             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 180             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 181             n2vc_event: event used to notify instantiation task that some change has been produced
 182         :param task: None for charm status change, or task for completion task callback
 183         :return:
 184         """
 185         try:
 186             nsr_id = n2vc_info["nsr_id"]
 187             deployed = n2vc_info["deployed"]
 188             db_nsr_update = n2vc_info["db_update"]
 189             nslcmop_id = n2vc_info["nslcmop_id"]
 190             ns_operation = n2vc_info["lcmOperationType"]
 191             n2vc_event = n2vc_info["n2vc_event"]
 192             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 193                                                                                     application_name)
 194             for vca_index, vca_deployed in enumerate(deployed):
 195                 if not vca_deployed:
 196                     continue
 197                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 198                     break
 199             else:
 200                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA")
 201                 return
 202             if task:
 203                 if task.cancelled():
 204                     self.logger.debug(logging_text + " task Cancelled")
 205                     vca_deployed['operational-status'] = "error"
 206                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 207                     vca_deployed['detailed-status'] = "Task Cancelled"
 208                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 209
 210                 elif task.done():
 211                     exc = task.exception()
 212                     if exc:
 213                         self.logger.error(logging_text + " task Exception={}".format(exc))
 214                         vca_deployed['operational-status'] = "error"
 215                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 216                         vca_deployed['detailed-status'] = str(exc)
 217                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 218                     else:
 219                         self.logger.debug(logging_text + " task Done")
 220                         # task is Done, but callback is still ongoing. So ignore
 221                         return
 222             elif status:
 223                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 224                 if vca_deployed['operational-status'] == status:
 225                     return  # same status, ignore
 226                 vca_deployed['operational-status'] = status
 227                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 228                 vca_deployed['detailed-status'] = str(message)
 229                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 230             else:
 231                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 232                 return
 233             # wake up instantiate task
 234             n2vc_event.set()
 235         except Exception as e:
 236             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 237
 238     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 239         """
 240         Creates a RO ns descriptor from OSM ns_instantiate params
 241         :param ns_params: OSM instantiate params
 242         :return: The RO ns descriptor
 243         """
 244         vim_2_RO = {}
 245         wim_2_RO = {}
 246         # TODO feature 1417: Check that no instantiation is set over PDU
 247         # check if PDU forces a concrete vim-network-id and add it
 248         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 249
 250         def vim_account_2_RO(vim_account):
 251             if vim_account in vim_2_RO:
 252                 return vim_2_RO[vim_account]
 253
 254             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 255             if db_vim["_admin"]["operationalState"] != "ENABLED":
 256                 raise LcmException("VIM={} is not available. operationalState={}".format(
 257                     vim_account, db_vim["_admin"]["operationalState"]))
 258             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 259             vim_2_RO[vim_account] = RO_vim_id
 260             return RO_vim_id
 261
 262         def wim_account_2_RO(wim_account):
 263             if isinstance(wim_account, str):
 264                 if wim_account in wim_2_RO:
 265                     return wim_2_RO[wim_account]
 266
 267                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 268                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 269                     raise LcmException("WIM={} is not available. operationalState={}".format(
 270                         wim_account, db_wim["_admin"]["operationalState"]))
 271                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 272                 wim_2_RO[wim_account] = RO_wim_id
 273                 return RO_wim_id
 274             else:
 275                 return wim_account
 276
 277         def ip_profile_2_RO(ip_profile):
 278             RO_ip_profile = deepcopy((ip_profile))
 279             if "dns-server" in RO_ip_profile:
 280                 if isinstance(RO_ip_profile["dns-server"], list):
 281                     RO_ip_profile["dns-address"] = []
 282                     for ds in RO_ip_profile.pop("dns-server"):
 283                         RO_ip_profile["dns-address"].append(ds['address'])
 284                 else:
 285                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 286             if RO_ip_profile.get("ip-version") == "ipv4":
 287                 RO_ip_profile["ip-version"] = "IPv4"
 288             if RO_ip_profile.get("ip-version") == "ipv6":
 289                 RO_ip_profile["ip-version"] = "IPv6"
 290             if "dhcp-params" in RO_ip_profile:
 291                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 292             return RO_ip_profile
 293
 294         if not ns_params:
 295             return None
 296         RO_ns_params = {
 297             # "name": ns_params["nsName"],
 298             # "description": ns_params.get("nsDescription"),
 299             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 300             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 301             # "scenario": ns_params["nsdId"],
 302         }
 303         if n2vc_key_list:
 304             for vnfd_ref, vnfd in vnfd_dict.items():
 305                 vdu_needed_access = []
 306                 mgmt_cp = None
 307                 if vnfd.get("vnf-configuration"):
 308                     if vnfd.get("mgmt-interface"):
 309                         if vnfd["mgmt-interface"].get("vdu-id"):
 310                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 311                         elif vnfd["mgmt-interface"].get("cp"):
 312                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 313
 314                 for vdu in vnfd.get("vdu", ()):
 315                     if vdu.get("vdu-configuration"):
 316                         vdu_needed_access.append(vdu["id"])
 317                     elif mgmt_cp:
 318                         for vdu_interface in vdu.get("interface"):
 319                             if vdu_interface.get("external-connection-point-ref") and \
 320                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 321                                 vdu_needed_access.append(vdu["id"])
 322                                 mgmt_cp = None
 323                                 break
 324
 325                 if vdu_needed_access:
 326                     for vnf_member in nsd.get("constituent-vnfd"):
 327                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 328                             continue
 329                         for vdu in vdu_needed_access:
 330                             populate_dict(RO_ns_params,
 331                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 332                                           n2vc_key_list)
 333
 334         if ns_params.get("vduImage"):
 335             RO_ns_params["vduImage"] = ns_params["vduImage"]
 336
 337         if ns_params.get("ssh_keys"):
 338             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 339         for vnf_params in get_iterable(ns_params, "vnf"):
 340             for constituent_vnfd in nsd["constituent-vnfd"]:
 341                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 342                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 343                     break
 344             else:
 345                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 346                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 347             if vnf_params.get("vimAccountId"):
 348                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 349                               vim_account_2_RO(vnf_params["vimAccountId"]))
 350
 351             for vdu_params in get_iterable(vnf_params, "vdu"):
 352                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 353                 if vdu_params.get("volume"):
 354                     for volume_params in vdu_params["volume"]:
 355                         if volume_params.get("vim-volume-id"):
 356                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 357                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 358                                           volume_params["vim-volume-id"])
 359                 if vdu_params.get("interface"):
 360                     for interface_params in vdu_params["interface"]:
 361                         if interface_params.get("ip-address"):
 362                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 363                                                          vdu_params["id"], "interfaces", interface_params["name"],
 364                                                          "ip_address"),
 365                                           interface_params["ip-address"])
 366                         if interface_params.get("mac-address"):
 367                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 368                                                          vdu_params["id"], "interfaces", interface_params["name"],
 369                                                          "mac_address"),
 370                                           interface_params["mac-address"])
 371                         if interface_params.get("floating-ip-required"):
 372                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 373                                                          vdu_params["id"], "interfaces", interface_params["name"],
 374                                                          "floating-ip"),
 375                                           interface_params["floating-ip-required"])
 376
 377             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 378                 if internal_vld_params.get("vim-network-name"):
 379                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 380                                                  internal_vld_params["name"], "vim-network-name"),
 381                                   internal_vld_params["vim-network-name"])
 382                 if internal_vld_params.get("vim-network-id"):
 383                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 384                                                  internal_vld_params["name"], "vim-network-id"),
 385                                   internal_vld_params["vim-network-id"])
 386                 if internal_vld_params.get("ip-profile"):
 387                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 388                                                  internal_vld_params["name"], "ip-profile"),
 389                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 390
 391                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 392                     # look for interface
 393                     iface_found = False
 394                     for vdu_descriptor in vnf_descriptor["vdu"]:
 395                         for vdu_interface in vdu_descriptor["interface"]:
 396                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 397                                 if icp_params.get("ip-address"):
 398                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 399                                                                  vdu_descriptor["id"], "interfaces",
 400                                                                  vdu_interface["name"], "ip_address"),
 401                                                   icp_params["ip-address"])
 402
 403                                 if icp_params.get("mac-address"):
 404                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 405                                                                  vdu_descriptor["id"], "interfaces",
 406                                                                  vdu_interface["name"], "mac_address"),
 407                                                   icp_params["mac-address"])
 408                                 iface_found = True
 409                                 break
 410                         if iface_found:
 411                             break
 412                     else:
 413                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 414                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 415                                            "connection-point".format(vnf_params["member-vnf-index"],
 416                                                                      icp_params["id-ref"]))
 417
 418         for vld_params in get_iterable(ns_params, "vld"):
 419             if "ip-profile" in vld_params:
 420                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 421                               ip_profile_2_RO(vld_params["ip-profile"]))
 422
 423             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 424                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 425                               wim_account_2_RO(vld_params["wimAccountId"])),
 426             if vld_params.get("vim-network-name"):
 427                 RO_vld_sites = []
 428                 if isinstance(vld_params["vim-network-name"], dict):
 429                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 430                         RO_vld_sites.append({
 431                             "netmap-use": vim_net,
 432                             "datacenter": vim_account_2_RO(vim_account)
 433                         })
 434                 else:  # isinstance str
 435                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 436                 if RO_vld_sites:
 437                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 438             if vld_params.get("vim-network-id"):
 439                 RO_vld_sites = []
 440                 if isinstance(vld_params["vim-network-id"], dict):
 441                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 442                         RO_vld_sites.append({
 443                             "netmap-use": vim_net,
 444                             "datacenter": vim_account_2_RO(vim_account)
 445                         })
 446                 else:  # isinstance str
 447                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 448                 if RO_vld_sites:
 449                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 450             if "vnfd-connection-point-ref" in vld_params:
 451                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 452                     # look for interface
 453                     for constituent_vnfd in nsd["constituent-vnfd"]:
 454                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 455                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 456                             break
 457                     else:
 458                         raise LcmException(
 459                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 460                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 461                     match_cp = False
 462                     for vdu_descriptor in vnf_descriptor["vdu"]:
 463                         for interface_descriptor in vdu_descriptor["interface"]:
 464                             if interface_descriptor.get("external-connection-point-ref") == \
 465                                     cp_params["vnfd-connection-point-ref"]:
 466                                 match_cp = True
 467                                 break
 468                         if match_cp:
 469                             break
 470                     else:
 471                         raise LcmException(
 472                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 473                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 474                                 cp_params["member-vnf-index-ref"],
 475                                 cp_params["vnfd-connection-point-ref"],
 476                                 vnf_descriptor["id"]))
 477                     if cp_params.get("ip-address"):
 478                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 479                                                      vdu_descriptor["id"], "interfaces",
 480                                                      interface_descriptor["name"], "ip_address"),
 481                                       cp_params["ip-address"])
 482                     if cp_params.get("mac-address"):
 483                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 484                                                      vdu_descriptor["id"], "interfaces",
 485                                                      interface_descriptor["name"], "mac_address"),
 486                                       cp_params["mac-address"])
 487         return RO_ns_params
 488
 489     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 490         # make a copy to do not change
 491         vdu_create = copy(vdu_create)
 492         vdu_delete = copy(vdu_delete)
 493
 494         vdurs = db_vnfr.get("vdur")
 495         if vdurs is None:
 496             vdurs = []
 497         vdu_index = len(vdurs)
 498         while vdu_index:
 499             vdu_index -= 1
 500             vdur = vdurs[vdu_index]
 501             if vdur.get("pdu-type"):
 502                 continue
 503             vdu_id_ref = vdur["vdu-id-ref"]
 504             if vdu_create and vdu_create.get(vdu_id_ref):
 505                 for index in range(0, vdu_create[vdu_id_ref]):
 506                     vdur = deepcopy(vdur)
 507                     vdur["_id"] = str(uuid4())
 508                     vdur["count-index"] += 1
 509                     vdurs.insert(vdu_index+1+index, vdur)
 510                 del vdu_create[vdu_id_ref]
 511             if vdu_delete and vdu_delete.get(vdu_id_ref):
 512                 del vdurs[vdu_index]
 513                 vdu_delete[vdu_id_ref] -= 1
 514                 if not vdu_delete[vdu_id_ref]:
 515                     del vdu_delete[vdu_id_ref]
 516         # check all operations are done
 517         if vdu_create or vdu_delete:
 518             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 519                 vdu_create))
 520         if vdu_delete:
 521             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 522                 vdu_delete))
 523
 524         vnfr_update = {"vdur": vdurs}
 525         db_vnfr["vdur"] = vdurs
 526         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 527
 528     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 529         """
 530         Updates database nsr with the RO info for the created vld
 531         :param ns_update_nsr: dictionary to be filled with the updated info
 532         :param db_nsr: content of db_nsr. This is also modified
 533         :param nsr_desc_RO: nsr descriptor from RO
 534         :return: Nothing, LcmException is raised on errors
 535         """
 536
 537         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 538             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 539                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 540                     continue
 541                 vld["vim-id"] = net_RO.get("vim_net_id")
 542                 vld["name"] = net_RO.get("vim_name")
 543                 vld["status"] = net_RO.get("status")
 544                 vld["status-detailed"] = net_RO.get("error_msg")
 545                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 546                 break
 547             else:
 548                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 549
 550     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 551         """
 552         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 553         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 554         :param nsr_desc_RO: nsr descriptor from RO
 555         :return: Nothing, LcmException is raised on errors
 556         """
 557         for vnf_index, db_vnfr in db_vnfrs.items():
 558             for vnf_RO in nsr_desc_RO["vnfs"]:
 559                 if vnf_RO["member_vnf_index"] != vnf_index:
 560                     continue
 561                 vnfr_update = {}
 562                 if vnf_RO.get("ip_address"):
 563                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 564                 elif not db_vnfr.get("ip-address"):
 565                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 566
 567                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 568                     vdur_RO_count_index = 0
 569                     if vdur.get("pdu-type"):
 570                         continue
 571                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 572                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 573                             continue
 574                         if vdur["count-index"] != vdur_RO_count_index:
 575                             vdur_RO_count_index += 1
 576                             continue
 577                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 578                         if vdur_RO.get("ip_address"):
 579                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 580                         else:
 581                             vdur["ip-address"] = None
 582                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 583                         vdur["name"] = vdur_RO.get("vim_name")
 584                         vdur["status"] = vdur_RO.get("status")
 585                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 586                         for ifacer in get_iterable(vdur, "interfaces"):
 587                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 588                                 if ifacer["name"] == interface_RO.get("internal_name"):
 589                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 590                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 591                                     break
 592                             else:
 593                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 594                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 595                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 596                         break
 597                     else:
 598                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 599                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 600
 601                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 602                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 603                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 604                             continue
 605                         vld["vim-id"] = net_RO.get("vim_net_id")
 606                         vld["name"] = net_RO.get("vim_name")
 607                         vld["status"] = net_RO.get("status")
 608                         vld["status-detailed"] = net_RO.get("error_msg")
 609                         vnfr_update["vld.{}".format(vld_index)] = vld
 610                         break
 611                     else:
 612                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 613                             vnf_index, vld["id"]))
 614
 615                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 616                 break
 617
 618             else:
 619                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 620
 621     async def instantiate(self, nsr_id, nslcmop_id):
 622         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 623         self.logger.debug(logging_text + "Enter")
 624         # get all needed from database
 625         start_deploy = time()
 626         db_nsr = None
 627         db_nslcmop = None
 628         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 629         db_nslcmop_update = {}
 630         nslcmop_operation_state = None
 631         db_vnfrs = {}
 632         RO_descriptor_number = 0   # number of descriptors created at RO
 633         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 634         n2vc_info = {}
 635         exc = None
 636         try:
 637             step = "Getting nslcmop={} from db".format(nslcmop_id)
 638             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 639             step = "Getting nsr={} from db".format(nsr_id)
 640             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 641             ns_params = db_nslcmop.get("operationParams")
 642             nsd = db_nsr["nsd"]
 643             nsr_name = db_nsr["name"]   # TODO short-name??
 644
 645             # look if previous tasks in process
 646             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
 647             if task_dependency:
 648                 step = db_nslcmop_update["detailed-status"] = \
 649                     "Waiting for related tasks to be completed: {}".format(task_name)
 650                 self.logger.debug(logging_text + step)
 651                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 652                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 653                 if pending:
 654                     raise LcmException("Timeout waiting related tasks to be completed")
 655
 656             step = "Getting vnfrs from db"
 657             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 658             db_vnfds_ref = {}
 659             db_vnfds = {}
 660             for vnfr in db_vnfrs_list:
 661                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 662                 vnfd_id = vnfr["vnfd-id"]
 663                 vnfd_ref = vnfr["vnfd-ref"]
 664                 if vnfd_id not in db_vnfds:
 665                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 666                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 667                     db_vnfds_ref[vnfd_ref] = vnfd
 668                     db_vnfds[vnfd_id] = vnfd
 669
 670             # Get or generates the _admin.deployed,VCA list
 671             vca_deployed_list = None
 672             if db_nsr["_admin"].get("deployed"):
 673                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 674             if vca_deployed_list is None:
 675                 vca_deployed_list = []
 676                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 677             elif isinstance(vca_deployed_list, dict):
 678                 # maintain backward compatibility. Change a dict to list at database
 679                 vca_deployed_list = list(vca_deployed_list.values())
 680                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 681
 682             db_nsr_update["detailed-status"] = "creating"
 683             db_nsr_update["operational-status"] = "init"
 684             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 685                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 686                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 687                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 688
 689             RO = ROclient.ROClient(self.loop, **self.ro_config)
 690
 691             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 692             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 693             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 694
 695             # get vnfds, instantiate at RO
 696             for c_vnf in nsd.get("constituent-vnfd", ()):
 697                 member_vnf_index = c_vnf["member-vnf-index"]
 698                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 699                 vnfd_ref = vnfd["id"]
 700                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member-vnf-index='{}' at RO".format(
 701                     vnfd_ref, member_vnf_index)
 702                 # self.logger.debug(logging_text + step)
 703                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 704                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 705                 RO_descriptor_number += 1
 706
 707                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 708                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 709                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 710                         break
 711                 else:
 712                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
 713                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
 714
 715                 # look if present
 716                 RO_update = {"member-vnf-index": member_vnf_index}
 717                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
 718                 if vnfd_list:
 719                     RO_update["id"] = vnfd_list[0]["uuid"]
 720                     self.logger.debug(logging_text + "vnfd='{}'  member-vnf-index='{}' exists at RO. Using RO_id={}".
 721                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
 722                 else:
 723                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
 724                                            get("additionalParamsForVnf"), nsr_id)
 725                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
 726                     RO_update["id"] = desc["uuid"]
 727                     self.logger.debug(logging_text + "vnfd='{}' member-vnf-index='{}' created at RO. RO_id={}".format(
 728                         vnfd_ref, member_vnf_index, desc["uuid"]))
 729                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
 730                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
 731                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 732
 733             # create nsd at RO
 734             nsd_ref = nsd["id"]
 735             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
 736             # self.logger.debug(logging_text + step)
 737
 738             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
 739             RO_descriptor_number += 1
 740             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
 741             if nsd_list:
 742                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
 743                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
 744                     nsd_ref, RO_nsd_uuid))
 745             else:
 746                 nsd_RO = deepcopy(nsd)
 747                 nsd_RO["id"] = RO_osm_nsd_id
 748                 nsd_RO.pop("_id", None)
 749                 nsd_RO.pop("_admin", None)
 750                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
 751                     member_vnf_index = c_vnf["member-vnf-index"]
 752                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 753                 for c_vld in nsd_RO.get("vld", ()):
 754                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
 755                         member_vnf_index = cp["member-vnf-index-ref"]
 756                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 757
 758                 desc = await RO.create("nsd", descriptor=nsd_RO)
 759                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 760                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
 761                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
 762             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 763
 764             # Crate ns at RO
 765             # if present use it unless in error status
 766             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
 767             if RO_nsr_id:
 768                 try:
 769                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
 770                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
 771                     desc = await RO.show("ns", RO_nsr_id)
 772                 except ROclient.ROClientException as e:
 773                     if e.http_code != HTTPStatus.NOT_FOUND:
 774                         raise
 775                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 776                 if RO_nsr_id:
 777                     ns_status, ns_status_info = RO.check_ns_status(desc)
 778                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 779                     if ns_status == "ERROR":
 780                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
 781                         self.logger.debug(logging_text + step)
 782                         await RO.delete("ns", RO_nsr_id)
 783                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 784             if not RO_nsr_id:
 785                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
 786                 # self.logger.debug(logging_text + step)
 787
 788                 # check if VIM is creating and wait  look if previous tasks in process
 789                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
 790                 if task_dependency:
 791                     step = "Waiting for related tasks to be completed: {}".format(task_name)
 792                     self.logger.debug(logging_text + step)
 793                     await asyncio.wait(task_dependency, timeout=3600)
 794                 if ns_params.get("vnf"):
 795                     for vnf in ns_params["vnf"]:
 796                         if "vimAccountId" in vnf:
 797                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 798                                                                                         vnf["vimAccountId"])
 799                         if task_dependency:
 800                             step = "Waiting for related tasks to be completed: {}".format(task_name)
 801                             self.logger.debug(logging_text + step)
 802                             await asyncio.wait(task_dependency, timeout=3600)
 803
 804                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
 805
 806                 # feature 1429. Add n2vc public key to needed VMs
 807                 n2vc_key = await self.n2vc.GetPublicKey()
 808                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, [n2vc_key])
 809
 810                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
 811                 desc = await RO.create("ns", descriptor=RO_ns_params,
 812                                        name=db_nsr["name"],
 813                                        scenario=RO_nsd_uuid)
 814                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
 815                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 816                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
 817                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 818             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 819
 820             # wait until NS is ready
 821             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
 822             detailed_status_old = None
 823             self.logger.debug(logging_text + step)
 824
 825             while time() <= start_deploy + self.total_deploy_timeout:
 826                 desc = await RO.show("ns", RO_nsr_id)
 827                 ns_status, ns_status_info = RO.check_ns_status(desc)
 828                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 829                 if ns_status == "ERROR":
 830                     raise ROclient.ROClientException(ns_status_info)
 831                 elif ns_status == "BUILD":
 832                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 833                 elif ns_status == "ACTIVE":
 834                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
 835                     try:
 836                         self.ns_update_vnfr(db_vnfrs, desc)
 837                         break
 838                     except LcmExceptionNoMgmtIP:
 839                         pass
 840                 else:
 841                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 842                 if detailed_status != detailed_status_old:
 843                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
 844                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 845                 await asyncio.sleep(5, loop=self.loop)
 846             else:   # total_deploy_timeout
 847                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 848
 849             step = "Updating NSR"
 850             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 851
 852             db_nsr["detailed-status"] = "Configuring vnfr"
 853             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 854
 855             # The parameters we'll need to deploy a charm
 856             number_to_configure = 0
 857
 858             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info):
 859                 """An inner function to deploy the charm from either vnf or vdu
 860                 vnf_index is mandatory. vdu_id can be None for a vnf configuration or the id for vdu configuration
 861                 """
 862                 if not charm_params["rw_mgmt_ip"]:
 863                     raise LcmException("vnfd/vdu has not management ip address to configure it")
 864                 # Login to the VCA.
 865                 # if number_to_configure == 0:
 866                 #     self.logger.debug("Logging into N2VC...")
 867                 #     task = asyncio.ensure_future(self.n2vc.login())
 868                 #     yield from asyncio.wait_for(task, 30.0)
 869                 #     self.logger.debug("Logged into N2VC!")
 870
 871                 # # await self.n2vc.login()
 872
 873                 # Note: The charm needs to exist on disk at the location
 874                 # specified by charm_path.
 875                 base_folder = vnfd["_admin"]["storage"]
 876                 storage_params = self.fs.get_params()
 877                 charm_path = "{}{}/{}/charms/{}".format(
 878                     storage_params["path"],
 879                     base_folder["folder"],
 880                     base_folder["pkg-dir"],
 881                     proxy_charm
 882                 )
 883
 884                 # ns_name will be ignored in the current version of N2VC
 885                 # but will be implemented for the next point release.
 886                 model_name = "default"    # TODO bug 585  nsr_id
 887                 if vdu_id:
 888                     vdu_id_text = vdu_id + "-"
 889                 else:
 890                     vdu_id_text = "-"
 891                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index, vdu_id_text)
 892
 893                 vca_index = len(vca_deployed_list)
 894                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 895                 # 26*26 charm in the same NS
 896                 application_name = application_name[0:48]
 897                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 898                 vca_deployed_ = {
 899                     "member-vnf-index": vnf_index,
 900                     "vdu_id": vdu_id,
 901                     "model": model_name,
 902                     "application": application_name,
 903                     "operational-status": "init",
 904                     "detailed-status": "",
 905                     "vnfd_id": vnfd_id,
 906                     "vdu_name": vdu_name,
 907                     "vdu_count_index": vdu_count_index,
 908                 }
 909                 vca_deployed_list.append(vca_deployed_)
 910                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 911                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 912
 913                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 914                                                                                                 proxy_charm))
 915                 if not n2vc_info:
 916                     n2vc_info["nsr_id"] = nsr_id
 917                     n2vc_info["nslcmop_id"] = nslcmop_id
 918                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 919                     n2vc_info["lcmOperationType"] = "instantiate"
 920                     n2vc_info["deployed"] = vca_deployed_list
 921                     n2vc_info["db_update"] = db_nsr_update
 922                 task = asyncio.ensure_future(
 923                     self.n2vc.DeployCharms(
 924                         model_name,          # The network service name
 925                         application_name,    # The application name
 926                         vnfd,                # The vnf descriptor
 927                         charm_path,          # Path to charm
 928                         charm_params,        # Runtime params, like mgmt ip
 929                         {},                  # for native charms only
 930                         self.n2vc_callback,  # Callback for status changes
 931                         n2vc_info,           # Callback parameter
 932                         None,                # Callback parameter (task)
 933                     )
 934                 )
 935                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 936                                                          n2vc_info))
 937                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 938
 939             step = "Looking for needed vnfd to configure"
 940             self.logger.debug(logging_text + step)
 941
 942             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 943                 vnfd_id = c_vnf["vnfd-id-ref"]
 944                 vnf_index = str(c_vnf["member-vnf-index"])
 945                 vnfd = db_vnfds_ref[vnfd_id]
 946
 947                 # Get additional parameters
 948                 vnfr_params = {}
 949                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 950                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 951                 for k, v in vnfr_params.items():
 952                     if isinstance(v, str) and v.startswith("!!yaml "):
 953                         vnfr_params[k] = yaml.safe_load(v[7:])
 954
 955                 # Check if this VNF has a charm configuration
 956                 vnf_config = vnfd.get("vnf-configuration")
 957                 if vnf_config and vnf_config.get("juju"):
 958                     proxy_charm = vnf_config["juju"]["charm"]
 959
 960                     if proxy_charm:
 961                         step = "connecting to N2VC to configure vnf {}".format(vnf_index)
 962                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 963                         charm_params = {
 964                             "user_values": vnfr_params,
 965                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 966                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {}
 967                         }
 968
 969                         # Login to the VCA. If there are multiple calls to login(),
 970                         # subsequent calls will be a nop and return immediately.
 971                         await self.n2vc.login()
 972                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 973                         number_to_configure += 1
 974
 975                 # Deploy charms for each VDU that supports one.
 976                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 977                     vdu_config = vdu.get('vdu-configuration')
 978                     proxy_charm = None
 979
 980                     if vdu_config and vdu_config.get("juju"):
 981                         proxy_charm = vdu_config["juju"]["charm"]
 982
 983                         if proxy_charm:
 984                             step = "connecting to N2VC to configure vdu {} from vnf {}".format(vdu["id"], vnf_index)
 985                             await self.n2vc.login()
 986                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 987                             # TODO for the moment only first vdu_id contains a charm deployed
 988                             if vdur["vdu-id-ref"] != vdu["id"]:
 989                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
 990                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 991                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 992                             charm_params = {
 993                                 "user_values": vnfr_params,
 994                                 "rw_mgmt_ip": vdur["ip-address"],
 995                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
 996                             }
 997                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 998                                          charm_params, n2vc_info)
 999                             number_to_configure += 1
1000
1001             db_nsr_update["operational-status"] = "running"
1002             configuration_failed = False
1003             if number_to_configure:
1004                 old_status = "configuring: init: {}".format(number_to_configure)
1005                 db_nsr_update["config-status"] = old_status
1006                 db_nsr_update["detailed-status"] = old_status
1007                 db_nslcmop_update["detailed-status"] = old_status
1008
1009                 # wait until all are configured.
1010                 while time() <= start_deploy + self.total_deploy_timeout:
1011                     if db_nsr_update:
1012                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1013                     if db_nslcmop_update:
1014                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1015                     # TODO add a fake task that set n2vc_event after some time
1016                     await n2vc_info["n2vc_event"].wait()
1017                     n2vc_info["n2vc_event"].clear()
1018                     all_active = True
1019                     status_map = {}
1020                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1021                     now = time()
1022                     for vca_deployed in vca_deployed_list:
1023                         vca_status = vca_deployed["operational-status"]
1024                         if vca_status not in status_map:
1025                             # Initialize it
1026                             status_map[vca_status] = 0
1027                         status_map[vca_status] += 1
1028
1029                         if vca_status == "active":
1030                             vca_deployed.pop("time_first_error", None)
1031                             vca_deployed.pop("status_first_error", None)
1032                             continue
1033
1034                         all_active = False
1035                         if vca_status in ("error", "blocked"):
1036                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1037                             # if not first time in this status error
1038                             if not vca_deployed.get("time_first_error"):
1039                                 vca_deployed["time_first_error"] = now
1040                                 continue
1041                         if vca_deployed.get("time_first_error") and \
1042                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1043                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1044                                                    .format(vca_deployed["member-vnf-index"],
1045                                                            vca_deployed["vdu_id"], vca_status,
1046                                                            vca_deployed["detailed-status-error"]))
1047
1048                     if all_active:
1049                         break
1050                     elif n2vc_error_text:
1051                         db_nsr_update["config-status"] = "failed"
1052                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1053                         db_nsr_update["detailed-status"] = error_text
1054                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1055                         db_nslcmop_update["detailed-status"] = error_text
1056                         db_nslcmop_update["statusEnteredTime"] = time()
1057                         configuration_failed = True
1058                         break
1059                     else:
1060                         cs = "configuring: "
1061                         separator = ""
1062                         for status, num in status_map.items():
1063                             cs += separator + "{}: {}".format(status, num)
1064                             separator = ", "
1065                         if old_status != cs:
1066                             db_nsr_update["config-status"] = cs
1067                             db_nsr_update["detailed-status"] = cs
1068                             db_nslcmop_update["detailed-status"] = cs
1069                             old_status = cs
1070                 else:   # total_deploy_timeout
1071                     raise LcmException("Timeout waiting ns to be configured")
1072
1073             if not configuration_failed:
1074                 # all is done
1075                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1076                 db_nslcmop_update["statusEnteredTime"] = time()
1077                 db_nslcmop_update["detailed-status"] = "done"
1078                 db_nsr_update["config-status"] = "configured"
1079                 db_nsr_update["detailed-status"] = "done"
1080
1081             return
1082
1083         except (ROclient.ROClientException, DbException, LcmException) as e:
1084             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1085             exc = e
1086         except asyncio.CancelledError:
1087             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1088             exc = "Operation was cancelled"
1089         except Exception as e:
1090             exc = traceback.format_exc()
1091             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1092                                  exc_info=True)
1093         finally:
1094             if exc:
1095                 if db_nsr:
1096                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1097                     db_nsr_update["operational-status"] = "failed"
1098                 if db_nslcmop:
1099                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1100                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1101                     db_nslcmop_update["statusEnteredTime"] = time()
1102             try:
1103                 if db_nsr:
1104                     db_nsr_update["_admin.nslcmop"] = None
1105                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1106                 if db_nslcmop_update:
1107                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1108             except DbException as e:
1109                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1110             if nslcmop_operation_state:
1111                 try:
1112                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1113                                                                    "operationState": nslcmop_operation_state},
1114                                             loop=self.loop)
1115                 except Exception as e:
1116                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1117
1118             self.logger.debug(logging_text + "Exit")
1119             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1120
1121     async def _destroy_charm(self, model, application):
1122         """
1123         Order N2VC destroy a charm
1124         :param model:
1125         :param application:
1126         :return: True if charm does not exist. False if it exist
1127         """
1128         if not await self.n2vc.HasApplication(model, application):
1129             return True  # Already removed
1130         await self.n2vc.RemoveCharms(model, application)
1131         return False
1132
1133     async def _wait_charm_destroyed(self, model, application, timeout):
1134         """
1135         Wait until charm does not exist
1136         :param model:
1137         :param application:
1138         :param timeout:
1139         :return: True if not exist, False if timeout
1140         """
1141         while True:
1142             if not await self.n2vc.HasApplication(model, application):
1143                 return True
1144             if timeout < 0:
1145                 return False
1146             await asyncio.sleep(10)
1147             timeout -= 10
1148
1149     async def terminate(self, nsr_id, nslcmop_id):
1150         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1151         self.logger.debug(logging_text + "Enter")
1152         db_nsr = None
1153         db_nslcmop = None
1154         exc = None
1155         failed_detail = []   # annotates all failed error messages
1156         vca_time_destroy = None   # time of where destroy charm order
1157         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1158         db_nslcmop_update = {}
1159         nslcmop_operation_state = None
1160         autoremove = False  # autoremove after terminated
1161         try:
1162             step = "Getting nslcmop={} from db".format(nslcmop_id)
1163             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1164             step = "Getting nsr={} from db".format(nsr_id)
1165             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1166             # nsd = db_nsr["nsd"]
1167             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1168             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1169                 return
1170             # #TODO check if VIM is creating and wait
1171             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1172
1173             db_nsr_update["operational-status"] = "terminating"
1174             db_nsr_update["config-status"] = "terminating"
1175
1176             if nsr_deployed and nsr_deployed.get("VCA"):
1177                 try:
1178                     step = "Scheduling configuration charms removing"
1179                     db_nsr_update["detailed-status"] = "Deleting charms"
1180                     self.logger.debug(logging_text + step)
1181                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1182                     # for backward compatibility
1183                     if isinstance(nsr_deployed["VCA"], dict):
1184                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1185                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1186                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1187
1188                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1189                         if vca_deployed:
1190                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1191                                 vca_deployed.clear()
1192                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1193                             else:
1194                                 vca_time_destroy = time()
1195                 except Exception as e:
1196                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1197
1198             # remove from RO
1199             RO_fail = False
1200             RO = ROclient.ROClient(self.loop, **self.ro_config)
1201
1202             # Delete ns
1203             RO_nsr_id = RO_delete_action = None
1204             if nsr_deployed and nsr_deployed.get("RO"):
1205                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1206                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1207             try:
1208                 if RO_nsr_id:
1209                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1210                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1211                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1212                     self.logger.debug(logging_text + step)
1213                     desc = await RO.delete("ns", RO_nsr_id)
1214                     RO_delete_action = desc["action_id"]
1215                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1216                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1217                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1218                 if RO_delete_action:
1219                     # wait until NS is deleted from VIM
1220                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1221                         format(RO_nsr_id, RO_delete_action)
1222                     detailed_status_old = None
1223                     self.logger.debug(logging_text + step)
1224
1225                     delete_timeout = 20 * 60   # 20 minutes
1226                     while delete_timeout > 0:
1227                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1228                                              extra_item_id=RO_delete_action)
1229                         ns_status, ns_status_info = RO.check_action_status(desc)
1230                         if ns_status == "ERROR":
1231                             raise ROclient.ROClientException(ns_status_info)
1232                         elif ns_status == "BUILD":
1233                             detailed_status = step + "; {}".format(ns_status_info)
1234                         elif ns_status == "ACTIVE":
1235                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1236                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1237                             break
1238                         else:
1239                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1240                         if detailed_status != detailed_status_old:
1241                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1242                                 db_nsr_update["detailed-status"] = detailed_status
1243                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1244                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1245                         await asyncio.sleep(5, loop=self.loop)
1246                         delete_timeout -= 5
1247                     else:  # delete_timeout <= 0:
1248                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1249
1250             except ROclient.ROClientException as e:
1251                 if e.http_code == 404:  # not found
1252                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1253                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1254                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1255                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1256                 elif e.http_code == 409:   # conflict
1257                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1258                     self.logger.debug(logging_text + failed_detail[-1])
1259                     RO_fail = True
1260                 else:
1261                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1262                     self.logger.error(logging_text + failed_detail[-1])
1263                     RO_fail = True
1264
1265             # Delete nsd
1266             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1267                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1268                 try:
1269                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1270                         "Deleting nsd at RO"
1271                     await RO.delete("nsd", RO_nsd_id)
1272                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1273                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1274                 except ROclient.ROClientException as e:
1275                     if e.http_code == 404:  # not found
1276                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1277                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1278                     elif e.http_code == 409:   # conflict
1279                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1280                         self.logger.debug(logging_text + failed_detail[-1])
1281                         RO_fail = True
1282                     else:
1283                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1284                         self.logger.error(logging_text + failed_detail[-1])
1285                         RO_fail = True
1286
1287             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1288                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1289                     if not vnf_deployed or not vnf_deployed["id"]:
1290                         continue
1291                     try:
1292                         RO_vnfd_id = vnf_deployed["id"]
1293                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1294                             "Deleting member-vnf-index={} RO_vnfd_id={} from RO".format(
1295                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1296                         await RO.delete("vnfd", RO_vnfd_id)
1297                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1298                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1299                     except ROclient.ROClientException as e:
1300                         if e.http_code == 404:  # not found
1301                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1302                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1303                         elif e.http_code == 409:   # conflict
1304                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1305                             self.logger.debug(logging_text + failed_detail[-1])
1306                         else:
1307                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1308                             self.logger.error(logging_text + failed_detail[-1])
1309
1310             # wait until charm deleted
1311             if vca_time_destroy:
1312                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1313                     "Waiting for deletion of configuration charms"
1314                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1315                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1316                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1317                     if not vca_deployed:
1318                         continue
1319                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1320                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1321                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1322                                                             timeout):
1323                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1324                             vca_deployed["application"]))
1325                     else:
1326                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1327
1328             if failed_detail:
1329                 self.logger.error(logging_text + " ;".join(failed_detail))
1330                 db_nsr_update["operational-status"] = "failed"
1331                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1332                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1333                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1334                 db_nslcmop_update["statusEnteredTime"] = time()
1335             else:
1336                 db_nsr_update["operational-status"] = "terminated"
1337                 db_nsr_update["detailed-status"] = "Done"
1338                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1339                 db_nslcmop_update["detailed-status"] = "Done"
1340                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1341                 db_nslcmop_update["statusEnteredTime"] = time()
1342                 if db_nslcmop["operationParams"].get("autoremove"):
1343                     autoremove = True
1344
1345         except (ROclient.ROClientException, DbException) as e:
1346             self.logger.error(logging_text + "Exit Exception {}".format(e))
1347             exc = e
1348         except asyncio.CancelledError:
1349             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1350             exc = "Operation was cancelled"
1351         except Exception as e:
1352             exc = traceback.format_exc()
1353             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1354         finally:
1355             if exc and db_nslcmop:
1356                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1357                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1358                 db_nslcmop_update["statusEnteredTime"] = time()
1359             try:
1360                 if db_nslcmop and db_nslcmop_update:
1361                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1362                 if db_nsr:
1363                     db_nsr_update["_admin.nslcmop"] = None
1364                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1365             except DbException as e:
1366                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1367             if nslcmop_operation_state:
1368                 try:
1369                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1370                                                                  "operationState": nslcmop_operation_state,
1371                                                                  "autoremove": autoremove},
1372                                             loop=self.loop)
1373                 except Exception as e:
1374                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1375             self.logger.debug(logging_text + "Exit")
1376             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1377
1378     @staticmethod
1379     def _map_primitive_params(primitive_desc, params, instantiation_params):
1380         """
1381         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1382         The default-value is used. If it is between < > it look for a value at instantiation_params
1383         :param primitive_desc: portion of VNFD/NSD that describes primitive
1384         :param params: Params provided by user
1385         :param instantiation_params: Instantiation params provided by user
1386         :return: a dictionary with the calculated params
1387         """
1388         calculated_params = {}
1389         for parameter in primitive_desc.get("parameter", ()):
1390             param_name = parameter["name"]
1391             if param_name in params:
1392                 calculated_params[param_name] = params[param_name]
1393             elif "default-value" in parameter:
1394                 calculated_params[param_name] = parameter["default-value"]
1395                 if isinstance(parameter["default-value"], str) and parameter["default-value"].startswith("<") and \
1396                         parameter["default-value"].endswith(">"):
1397                     if parameter["default-value"][1:-1] in instantiation_params:
1398                         calculated_params[param_name] = instantiation_params[parameter["default-value"][1:-1]]
1399                     else:
1400                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1401                                            format(parameter["default-value"], primitive_desc["name"]))
1402             else:
1403                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1404                                    format(param_name, primitive_desc["name"]))
1405
1406             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1407                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1408                                                                width=256)
1409             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1410                 calculated_params[param_name] = calculated_params[param_name][7:]
1411         return calculated_params
1412
1413     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1414                                     primitive, primitive_params):
1415         start_primitive_time = time()
1416         try:
1417             for vca_deployed in db_deployed["VCA"]:
1418                 if not vca_deployed:
1419                     continue
1420                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1421                     continue
1422                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1423                     continue
1424                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1425                     continue
1426                 break
1427             else:
1428                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1429                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1430             model_name = vca_deployed.get("model")
1431             application_name = vca_deployed.get("application")
1432             if not model_name or not application_name:
1433                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1434                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1435                                                                        vdu_count_index))
1436             if vca_deployed["operational-status"] != "active":
1437                 raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1438                     member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1439             callback = None  # self.n2vc_callback
1440             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1441             await self.n2vc.login()
1442             primitive_id = await self.n2vc.ExecutePrimitive(
1443                 model_name,
1444                 application_name,
1445                 primitive,
1446                 callback,
1447                 *callback_args,
1448                 **primitive_params
1449             )
1450             while time() - start_primitive_time < self.timeout_primitive:
1451                 primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1452                 if primitive_result_ in ("running", "pending"):
1453                     pass
1454                 elif primitive_result_ in ("completed", "failed"):
1455                     primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1456                     detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1457                     break
1458                 else:
1459                     detailed_result = "Invalid N2VC.GetPrimitiveStatus = {} obtained".format(primitive_result_)
1460                     primitive_result = "FAILED"
1461                     break
1462                 await asyncio.sleep(5)
1463             else:
1464                 raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1465             return primitive_result, detailed_result
1466         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
1467             return "FAILED", str(e)
1468
1469     async def action(self, nsr_id, nslcmop_id):
1470         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
1471         self.logger.debug(logging_text + "Enter")
1472         # get all needed from database
1473         db_nsr = None
1474         db_nslcmop = None
1475         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1476         db_nslcmop_update = {}
1477         nslcmop_operation_state = None
1478         exc = None
1479         try:
1480             step = "Getting information from database"
1481             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1482             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1483
1484             nsr_deployed = db_nsr["_admin"].get("deployed")
1485             vnf_index = db_nslcmop["operationParams"]["member_vnf_index"]
1486             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1487             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1488             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1489
1490             step = "Getting vnfr from database"
1491             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1492             step = "Getting vnfd from database"
1493             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1494
1495             # look if previous tasks in process
1496             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1497             if task_dependency:
1498                 step = db_nslcmop_update["detailed-status"] = \
1499                     "Waiting for related tasks to be completed: {}".format(task_name)
1500                 self.logger.debug(logging_text + step)
1501                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1502                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1503                 if pending:
1504                     raise LcmException("Timeout waiting related tasks to be completed")
1505
1506             # for backward compatibility
1507             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1508                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1509                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1510                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1511
1512             primitive = db_nslcmop["operationParams"]["primitive"]
1513             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
1514
1515             # look for primitive
1516             config_primitive_desc = None
1517             if vdu_id:
1518                 for vdu in get_iterable(db_vnfd, "vdu"):
1519                     if vdu_id == vdu["id"]:
1520                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
1521                             if config_primitive["name"] == primitive:
1522                                 config_primitive_desc = config_primitive
1523                                 break
1524             for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1525                 if config_primitive["name"] == primitive:
1526                     config_primitive_desc = config_primitive
1527                     break
1528             if not config_primitive_desc:
1529                 raise LcmException("Primitive {} not found at vnf-configuration:config-primitive or vdu:"
1530                                    "vdu-configuration:config-primitive".format(primitive))
1531
1532             vnfr_params = {}
1533             if db_vnfr.get("additionalParamsForVnf"):
1534                 vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1535
1536             # TODO check if ns is in a proper status
1537             result, result_detail = await self._ns_execute_primitive(
1538                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1539                 self._map_primitive_params(config_primitive_desc, primitive_params, vnfr_params))
1540             db_nslcmop_update["detailed-status"] = result_detail
1541             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
1542             db_nslcmop_update["statusEnteredTime"] = time()
1543             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
1544             return  # database update is called inside finally
1545
1546         except (DbException, LcmException) as e:
1547             self.logger.error(logging_text + "Exit Exception {}".format(e))
1548             exc = e
1549         except asyncio.CancelledError:
1550             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1551             exc = "Operation was cancelled"
1552         except Exception as e:
1553             exc = traceback.format_exc()
1554             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1555         finally:
1556             if exc and db_nslcmop:
1557                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1558                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1559                 db_nslcmop_update["statusEnteredTime"] = time()
1560             try:
1561                 if db_nslcmop_update:
1562                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1563                 if db_nsr:
1564                     db_nsr_update["_admin.nslcmop"] = None
1565                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1566             except DbException as e:
1567                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1568             self.logger.debug(logging_text + "Exit")
1569             if nslcmop_operation_state:
1570                 try:
1571                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1572                                                                "operationState": nslcmop_operation_state},
1573                                             loop=self.loop)
1574                 except Exception as e:
1575                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1576             self.logger.debug(logging_text + "Exit")
1577             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
1578
1579     async def scale(self, nsr_id, nslcmop_id):
1580         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
1581         self.logger.debug(logging_text + "Enter")
1582         # get all needed from database
1583         db_nsr = None
1584         db_nslcmop = None
1585         db_nslcmop_update = {}
1586         nslcmop_operation_state = None
1587         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1588         exc = None
1589         # in case of error, indicates what part of scale was failed to put nsr at error status
1590         scale_process = None
1591         old_operational_status = ""
1592         old_config_status = ""
1593         vnfr_scaled = False
1594         try:
1595             step = "Getting nslcmop from database"
1596             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1597             step = "Getting nsr from database"
1598             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1599
1600             old_operational_status = db_nsr["operational-status"]
1601             old_config_status = db_nsr["config-status"]
1602
1603             # look if previous tasks in process
1604             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1605             if task_dependency:
1606                 step = db_nslcmop_update["detailed-status"] = \
1607                     "Waiting for related tasks to be completed: {}".format(task_name)
1608                 self.logger.debug(logging_text + step)
1609                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1610                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1611                 if pending:
1612                     raise LcmException("Timeout waiting related tasks to be completed")
1613
1614             step = "Parsing scaling parameters"
1615             db_nsr_update["operational-status"] = "scaling"
1616             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1617             nsr_deployed = db_nsr["_admin"].get("deployed")
1618             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
1619             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
1620             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
1621             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
1622             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
1623
1624             # for backward compatibility
1625             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1626                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1627                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1628                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1629
1630             step = "Getting vnfr from database"
1631             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1632             step = "Getting vnfd from database"
1633             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1634             step = "Getting scaling-group-descriptor"
1635             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
1636                 if scaling_descriptor["name"] == scaling_group:
1637                     break
1638             else:
1639                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
1640                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
1641             # cooldown_time = 0
1642             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
1643             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
1644             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
1645             #         break
1646
1647             # TODO check if ns is in a proper status
1648             step = "Sending scale order to RO"
1649             nb_scale_op = 0
1650             if not db_nsr["_admin"].get("scaling-group"):
1651                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
1652                 admin_scale_index = 0
1653             else:
1654                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
1655                     if admin_scale_info["name"] == scaling_group:
1656                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
1657                         break
1658                 else:  # not found, set index one plus last element and add new entry with the name
1659                     admin_scale_index += 1
1660                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
1661             RO_scaling_info = []
1662             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
1663             if scaling_type == "SCALE_OUT":
1664                 # count if max-instance-count is reached
1665                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
1666                     max_instance_count = int(scaling_descriptor["max-instance-count"])
1667                     if nb_scale_op >= max_instance_count:
1668                         raise LcmException("reached the limit of {} (max-instance-count) scaling-out operations for the"
1669                                            " scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1670                 nb_scale_op = nb_scale_op + 1
1671                 vdu_scaling_info["scaling_direction"] = "OUT"
1672                 vdu_scaling_info["vdu-create"] = {}
1673                 for vdu_scale_info in scaling_descriptor["vdu"]:
1674                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1675                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
1676                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1677             elif scaling_type == "SCALE_IN":
1678                 # count if min-instance-count is reached
1679                 min_instance_count = 0
1680                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
1681                     min_instance_count = int(scaling_descriptor["min-instance-count"])
1682                 if nb_scale_op <= min_instance_count:
1683                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
1684                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1685                 nb_scale_op = nb_scale_op - 1
1686                 vdu_scaling_info["scaling_direction"] = "IN"
1687                 vdu_scaling_info["vdu-delete"] = {}
1688                 for vdu_scale_info in scaling_descriptor["vdu"]:
1689                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1690                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
1691                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1692
1693             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
1694             vdu_create = vdu_scaling_info.get("vdu-create")
1695             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
1696             if vdu_scaling_info["scaling_direction"] == "IN":
1697                 for vdur in reversed(db_vnfr["vdur"]):
1698                     if vdu_delete.get(vdur["vdu-id-ref"]):
1699                         vdu_delete[vdur["vdu-id-ref"]] -= 1
1700                         vdu_scaling_info["vdu"].append({
1701                             "name": vdur["name"],
1702                             "vdu_id": vdur["vdu-id-ref"],
1703                             "interface": []
1704                         })
1705                         for interface in vdur["interfaces"]:
1706                             vdu_scaling_info["vdu"][-1]["interface"].append({
1707                                 "name": interface["name"],
1708                                 "ip_address": interface["ip-address"],
1709                                 "mac_address": interface.get("mac-address"),
1710                             })
1711                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
1712
1713             # execute primitive service PRE-SCALING
1714             step = "Executing pre-scale vnf-config-primitive"
1715             if scaling_descriptor.get("scaling-config-action"):
1716                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1717                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
1718                             and scaling_type == "SCALE_IN":
1719                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1720                         step = db_nslcmop_update["detailed-status"] = \
1721                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
1722
1723                         # look for primitive
1724                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1725                             if config_primitive["name"] == vnf_config_primitive:
1726                                 break
1727                         else:
1728                             raise LcmException(
1729                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
1730                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
1731                                 "primitive".format(scaling_group, config_primitive))
1732
1733                         vnfr_params = {"<VDU_SCALE_INFO>": vdu_scaling_info}
1734                         if db_vnfr.get("additionalParamsForVnf"):
1735                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1736
1737                         scale_process = "VCA"
1738                         db_nsr_update["config-status"] = "configuring pre-scaling"
1739                         result, result_detail = await self._ns_execute_primitive(
1740                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1741                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1742                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1743                             vnf_config_primitive, result, result_detail))
1744                         if result == "FAILED":
1745                             raise LcmException(result_detail)
1746                         db_nsr_update["config-status"] = old_config_status
1747                         scale_process = None
1748
1749             if RO_scaling_info:
1750                 scale_process = "RO"
1751                 RO = ROclient.ROClient(self.loop, **self.ro_config)
1752                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
1753                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
1754                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
1755                 # wait until ready
1756                 RO_nslcmop_id = RO_desc["instance_action_id"]
1757                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
1758
1759                 RO_task_done = False
1760                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
1761                 detailed_status_old = None
1762                 self.logger.debug(logging_text + step)
1763
1764                 deployment_timeout = 1 * 3600   # One hour
1765                 while deployment_timeout > 0:
1766                     if not RO_task_done:
1767                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1768                                              extra_item_id=RO_nslcmop_id)
1769                         ns_status, ns_status_info = RO.check_action_status(desc)
1770                         if ns_status == "ERROR":
1771                             raise ROclient.ROClientException(ns_status_info)
1772                         elif ns_status == "BUILD":
1773                             detailed_status = step + "; {}".format(ns_status_info)
1774                         elif ns_status == "ACTIVE":
1775                             RO_task_done = True
1776                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1777                             self.logger.debug(logging_text + step)
1778                         else:
1779                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1780                     else:
1781                         desc = await RO.show("ns", RO_nsr_id)
1782                         ns_status, ns_status_info = RO.check_ns_status(desc)
1783                         if ns_status == "ERROR":
1784                             raise ROclient.ROClientException(ns_status_info)
1785                         elif ns_status == "BUILD":
1786                             detailed_status = step + "; {}".format(ns_status_info)
1787                         elif ns_status == "ACTIVE":
1788                             step = detailed_status = \
1789                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
1790                             if not vnfr_scaled:
1791                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
1792                                 vnfr_scaled = True
1793                             try:
1794                                 desc = await RO.show("ns", RO_nsr_id)
1795                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
1796                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
1797                                 break
1798                             except LcmExceptionNoMgmtIP:
1799                                 pass
1800                         else:
1801                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1802                     if detailed_status != detailed_status_old:
1803                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
1804                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1805
1806                     await asyncio.sleep(5, loop=self.loop)
1807                     deployment_timeout -= 5
1808                 if deployment_timeout <= 0:
1809                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
1810
1811                 # update VDU_SCALING_INFO with the obtained ip_addresses
1812                 if vdu_scaling_info["scaling_direction"] == "OUT":
1813                     for vdur in reversed(db_vnfr["vdur"]):
1814                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
1815                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
1816                             vdu_scaling_info["vdu"].append({
1817                                 "name": vdur["name"],
1818                                 "vdu_id": vdur["vdu-id-ref"],
1819                                 "interface": []
1820                             })
1821                             for interface in vdur["interfaces"]:
1822                                 vdu_scaling_info["vdu"][-1]["interface"].append({
1823                                     "name": interface["name"],
1824                                     "ip_address": interface["ip-address"],
1825                                     "mac_address": interface.get("mac-address"),
1826                                 })
1827                     del vdu_scaling_info["vdu-create"]
1828
1829             scale_process = None
1830             if db_nsr_update:
1831                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1832
1833             # execute primitive service POST-SCALING
1834             step = "Executing post-scale vnf-config-primitive"
1835             if scaling_descriptor.get("scaling-config-action"):
1836                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1837                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
1838                             and scaling_type == "SCALE_OUT":
1839                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1840                         step = db_nslcmop_update["detailed-status"] = \
1841                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
1842
1843                         vnfr_params = {"<VDU_SCALE_INFO>": vdu_scaling_info}
1844                         if db_vnfr.get("additionalParamsForVnf"):
1845                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1846
1847                         # look for primitive
1848                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1849                             if config_primitive["name"] == vnf_config_primitive:
1850                                 break
1851                         else:
1852                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
1853                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
1854                                                "match any vnf-configuration:config-primitive".format(scaling_group,
1855                                                                                                      config_primitive))
1856                         scale_process = "VCA"
1857                         db_nsr_update["config-status"] = "configuring post-scaling"
1858
1859                         result, result_detail = await self._ns_execute_primitive(
1860                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1861                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1862                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1863                             vnf_config_primitive, result, result_detail))
1864                         if result == "FAILED":
1865                             raise LcmException(result_detail)
1866                         db_nsr_update["config-status"] = old_config_status
1867                         scale_process = None
1868
1869             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1870             db_nslcmop_update["statusEnteredTime"] = time()
1871             db_nslcmop_update["detailed-status"] = "done"
1872             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
1873             db_nsr_update["operational-status"] = old_operational_status
1874             db_nsr_update["config-status"] = old_config_status
1875             return
1876         except (ROclient.ROClientException, DbException, LcmException) as e:
1877             self.logger.error(logging_text + "Exit Exception {}".format(e))
1878             exc = e
1879         except asyncio.CancelledError:
1880             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1881             exc = "Operation was cancelled"
1882         except Exception as e:
1883             exc = traceback.format_exc()
1884             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1885         finally:
1886             if exc:
1887                 if db_nslcmop:
1888                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1889                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1890                     db_nslcmop_update["statusEnteredTime"] = time()
1891                 if db_nsr:
1892                     db_nsr_update["operational-status"] = old_operational_status
1893                     db_nsr_update["config-status"] = old_config_status
1894                     db_nsr_update["detailed-status"] = ""
1895                     db_nsr_update["_admin.nslcmop"] = None
1896                     if scale_process:
1897                         if "VCA" in scale_process:
1898                             db_nsr_update["config-status"] = "failed"
1899                         if "RO" in scale_process:
1900                             db_nsr_update["operational-status"] = "failed"
1901                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
1902                                                                                                      exc)
1903             try:
1904                 if db_nslcmop and db_nslcmop_update:
1905                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1906                 if db_nsr:
1907                     db_nsr_update["_admin.nslcmop"] = None
1908                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1909             except DbException as e:
1910                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1911             if nslcmop_operation_state:
1912                 try:
1913                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1914                                                              "operationState": nslcmop_operation_state},
1915                                             loop=self.loop)
1916                     # if cooldown_time:
1917                     #     await asyncio.sleep(cooldown_time)
1918                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
1919                 except Exception as e:
1920                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1921             self.logger.debug(logging_text + "Exit")
1922             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")