osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102         )
 103
 104     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 105         """
 106         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 107         :param vnfd: input vnfd
 108         :param new_id: overrides vnf id if provided
 109         :param additionalParams: Instantiation params for VNFs provided
 110         :param nsrId: Id of the NSR
 111         :return: copy of vnfd
 112         """
 113         try:
 114             vnfd_RO = deepcopy(vnfd)
 115             # remove unused by RO configuration, monitoring, scaling and internal keys
 116             vnfd_RO.pop("_id", None)
 117             vnfd_RO.pop("_admin", None)
 118             vnfd_RO.pop("vnf-configuration", None)
 119             vnfd_RO.pop("monitoring-param", None)
 120             vnfd_RO.pop("scaling-group-descriptor", None)
 121             if new_id:
 122                 vnfd_RO["id"] = new_id
 123
 124             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 125             for vdu in get_iterable(vnfd_RO, "vdu"):
 126                 cloud_init_file = None
 127                 if vdu.get("cloud-init-file"):
 128                     base_folder = vnfd["_admin"]["storage"]
 129                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 130                                                                    vdu["cloud-init-file"])
 131                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 132                         cloud_init_content = ci_file.read()
 133                     vdu.pop("cloud-init-file", None)
 134                 elif vdu.get("cloud-init"):
 135                     cloud_init_content = vdu["cloud-init"]
 136                 else:
 137                     continue
 138
 139                 env = Environment()
 140                 ast = env.parse(cloud_init_content)
 141                 mandatory_vars = meta.find_undeclared_variables(ast)
 142                 if mandatory_vars:
 143                     for var in mandatory_vars:
 144                         if not additionalParams or var not in additionalParams.keys():
 145                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 146                                                "file, must be provided in the instantiation parameters inside the "
 147                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 148                 template = Template(cloud_init_content)
 149                 cloud_init_content = template.render(additionalParams or {})
 150                 vdu["cloud-init"] = cloud_init_content
 151
 152             return vnfd_RO
 153         except FsException as e:
 154             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 155                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 156         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 157             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 158                                format(vnfd["id"], vdu["id"], e))
 159
 160     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 161         """
 162         Callback both for charm status change and task completion
 163         :param model_name: Charm model name
 164         :param application_name: Charm application name
 165         :param status: Can be
 166             - blocked: The unit needs manual intervention
 167             - maintenance: The unit is actively deploying/configuring
 168             - waiting: The unit is waiting for another charm to be ready
 169             - active: The unit is deployed, configured, and ready
 170             - error: The charm has failed and needs attention.
 171             - terminated: The charm has been destroyed
 172             - removing,
 173             - removed
 174         :param message: detailed message error
 175         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 176             nsr_id:
 177             nslcmop_id:
 178             lcmOperationType: currently "instantiate"
 179             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 180             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 181             n2vc_event: event used to notify instantiation task that some change has been produced
 182         :param task: None for charm status change, or task for completion task callback
 183         :return:
 184         """
 185         try:
 186             nsr_id = n2vc_info["nsr_id"]
 187             deployed = n2vc_info["deployed"]
 188             db_nsr_update = n2vc_info["db_update"]
 189             nslcmop_id = n2vc_info["nslcmop_id"]
 190             ns_operation = n2vc_info["lcmOperationType"]
 191             n2vc_event = n2vc_info["n2vc_event"]
 192             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 193                                                                                     application_name)
 194             for vca_index, vca_deployed in enumerate(deployed):
 195                 if not vca_deployed:
 196                     continue
 197                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 198                     break
 199             else:
 200                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA")
 201                 return
 202             if task:
 203                 if task.cancelled():
 204                     self.logger.debug(logging_text + " task Cancelled")
 205                     vca_deployed['operational-status'] = "error"
 206                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 207                     vca_deployed['detailed-status'] = "Task Cancelled"
 208                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 209
 210                 elif task.done():
 211                     exc = task.exception()
 212                     if exc:
 213                         self.logger.error(logging_text + " task Exception={}".format(exc))
 214                         vca_deployed['operational-status'] = "error"
 215                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 216                         vca_deployed['detailed-status'] = str(exc)
 217                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 218                     else:
 219                         self.logger.debug(logging_text + " task Done")
 220                         # task is Done, but callback is still ongoing. So ignore
 221                         return
 222             elif status:
 223                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 224                 if vca_deployed['operational-status'] == status:
 225                     return  # same status, ignore
 226                 vca_deployed['operational-status'] = status
 227                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 228                 vca_deployed['detailed-status'] = str(message)
 229                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 230             else:
 231                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 232                 return
 233             # wake up instantiate task
 234             n2vc_event.set()
 235         except Exception as e:
 236             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 237
 238     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 239         """
 240         Creates a RO ns descriptor from OSM ns_instantiate params
 241         :param ns_params: OSM instantiate params
 242         :return: The RO ns descriptor
 243         """
 244         vim_2_RO = {}
 245         wim_2_RO = {}
 246         # TODO feature 1417: Check that no instantiation is set over PDU
 247         # check if PDU forces a concrete vim-network-id and add it
 248         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 249
 250         def vim_account_2_RO(vim_account):
 251             if vim_account in vim_2_RO:
 252                 return vim_2_RO[vim_account]
 253
 254             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 255             if db_vim["_admin"]["operationalState"] != "ENABLED":
 256                 raise LcmException("VIM={} is not available. operationalState={}".format(
 257                     vim_account, db_vim["_admin"]["operationalState"]))
 258             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 259             vim_2_RO[vim_account] = RO_vim_id
 260             return RO_vim_id
 261
 262         def wim_account_2_RO(wim_account):
 263             if isinstance(wim_account, str):
 264                 if wim_account in wim_2_RO:
 265                     return wim_2_RO[wim_account]
 266
 267                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 268                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 269                     raise LcmException("WIM={} is not available. operationalState={}".format(
 270                         wim_account, db_wim["_admin"]["operationalState"]))
 271                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 272                 wim_2_RO[wim_account] = RO_wim_id
 273                 return RO_wim_id
 274             else:
 275                 return wim_account
 276
 277         def ip_profile_2_RO(ip_profile):
 278             RO_ip_profile = deepcopy((ip_profile))
 279             if "dns-server" in RO_ip_profile:
 280                 if isinstance(RO_ip_profile["dns-server"], list):
 281                     RO_ip_profile["dns-address"] = []
 282                     for ds in RO_ip_profile.pop("dns-server"):
 283                         RO_ip_profile["dns-address"].append(ds['address'])
 284                 else:
 285                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 286             if RO_ip_profile.get("ip-version") == "ipv4":
 287                 RO_ip_profile["ip-version"] = "IPv4"
 288             if RO_ip_profile.get("ip-version") == "ipv6":
 289                 RO_ip_profile["ip-version"] = "IPv6"
 290             if "dhcp-params" in RO_ip_profile:
 291                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 292             return RO_ip_profile
 293
 294         if not ns_params:
 295             return None
 296         RO_ns_params = {
 297             # "name": ns_params["nsName"],
 298             # "description": ns_params.get("nsDescription"),
 299             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 300             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 301             # "scenario": ns_params["nsdId"],
 302         }
 303         if n2vc_key_list:
 304             for vnfd_ref, vnfd in vnfd_dict.items():
 305                 vdu_needed_access = []
 306                 mgmt_cp = None
 307                 if vnfd.get("vnf-configuration"):
 308                     if vnfd.get("mgmt-interface"):
 309                         if vnfd["mgmt-interface"].get("vdu-id"):
 310                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 311                         elif vnfd["mgmt-interface"].get("cp"):
 312                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 313
 314                 for vdu in vnfd.get("vdu", ()):
 315                     if vdu.get("vdu-configuration"):
 316                         vdu_needed_access.append(vdu["id"])
 317                     elif mgmt_cp:
 318                         for vdu_interface in vdu.get("interface"):
 319                             if vdu_interface.get("external-connection-point-ref") and \
 320                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 321                                 vdu_needed_access.append(vdu["id"])
 322                                 mgmt_cp = None
 323                                 break
 324
 325                 if vdu_needed_access:
 326                     for vnf_member in nsd.get("constituent-vnfd"):
 327                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 328                             continue
 329                         for vdu in vdu_needed_access:
 330                             populate_dict(RO_ns_params,
 331                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 332                                           n2vc_key_list)
 333
 334         if ns_params.get("vduImage"):
 335             RO_ns_params["vduImage"] = ns_params["vduImage"]
 336
 337         if ns_params.get("ssh_keys"):
 338             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 339         for vnf_params in get_iterable(ns_params, "vnf"):
 340             for constituent_vnfd in nsd["constituent-vnfd"]:
 341                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 342                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 343                     break
 344             else:
 345                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 346                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 347             if vnf_params.get("vimAccountId"):
 348                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 349                               vim_account_2_RO(vnf_params["vimAccountId"]))
 350
 351             for vdu_params in get_iterable(vnf_params, "vdu"):
 352                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 353                 if vdu_params.get("volume"):
 354                     for volume_params in vdu_params["volume"]:
 355                         if volume_params.get("vim-volume-id"):
 356                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 357                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 358                                           volume_params["vim-volume-id"])
 359                 if vdu_params.get("interface"):
 360                     for interface_params in vdu_params["interface"]:
 361                         if interface_params.get("ip-address"):
 362                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 363                                                          vdu_params["id"], "interfaces", interface_params["name"],
 364                                                          "ip_address"),
 365                                           interface_params["ip-address"])
 366                         if interface_params.get("mac-address"):
 367                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 368                                                          vdu_params["id"], "interfaces", interface_params["name"],
 369                                                          "mac_address"),
 370                                           interface_params["mac-address"])
 371                         if interface_params.get("floating-ip-required"):
 372                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 373                                                          vdu_params["id"], "interfaces", interface_params["name"],
 374                                                          "floating-ip"),
 375                                           interface_params["floating-ip-required"])
 376
 377             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 378                 if internal_vld_params.get("vim-network-name"):
 379                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 380                                                  internal_vld_params["name"], "vim-network-name"),
 381                                   internal_vld_params["vim-network-name"])
 382                 if internal_vld_params.get("vim-network-id"):
 383                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 384                                                  internal_vld_params["name"], "vim-network-id"),
 385                                   internal_vld_params["vim-network-id"])
 386                 if internal_vld_params.get("ip-profile"):
 387                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 388                                                  internal_vld_params["name"], "ip-profile"),
 389                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 390
 391                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 392                     # look for interface
 393                     iface_found = False
 394                     for vdu_descriptor in vnf_descriptor["vdu"]:
 395                         for vdu_interface in vdu_descriptor["interface"]:
 396                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 397                                 if icp_params.get("ip-address"):
 398                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 399                                                                  vdu_descriptor["id"], "interfaces",
 400                                                                  vdu_interface["name"], "ip_address"),
 401                                                   icp_params["ip-address"])
 402
 403                                 if icp_params.get("mac-address"):
 404                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 405                                                                  vdu_descriptor["id"], "interfaces",
 406                                                                  vdu_interface["name"], "mac_address"),
 407                                                   icp_params["mac-address"])
 408                                 iface_found = True
 409                                 break
 410                         if iface_found:
 411                             break
 412                     else:
 413                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 414                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 415                                            "connection-point".format(vnf_params["member-vnf-index"],
 416                                                                      icp_params["id-ref"]))
 417
 418         for vld_params in get_iterable(ns_params, "vld"):
 419             if "ip-profile" in vld_params:
 420                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 421                               ip_profile_2_RO(vld_params["ip-profile"]))
 422
 423             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 424                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 425                               wim_account_2_RO(vld_params["wimAccountId"])),
 426             if vld_params.get("vim-network-name"):
 427                 RO_vld_sites = []
 428                 if isinstance(vld_params["vim-network-name"], dict):
 429                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 430                         RO_vld_sites.append({
 431                             "netmap-use": vim_net,
 432                             "datacenter": vim_account_2_RO(vim_account)
 433                         })
 434                 else:  # isinstance str
 435                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 436                 if RO_vld_sites:
 437                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 438             if vld_params.get("vim-network-id"):
 439                 RO_vld_sites = []
 440                 if isinstance(vld_params["vim-network-id"], dict):
 441                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 442                         RO_vld_sites.append({
 443                             "netmap-use": vim_net,
 444                             "datacenter": vim_account_2_RO(vim_account)
 445                         })
 446                 else:  # isinstance str
 447                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 448                 if RO_vld_sites:
 449                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 450             if "vnfd-connection-point-ref" in vld_params:
 451                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 452                     # look for interface
 453                     for constituent_vnfd in nsd["constituent-vnfd"]:
 454                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 455                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 456                             break
 457                     else:
 458                         raise LcmException(
 459                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 460                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 461                     match_cp = False
 462                     for vdu_descriptor in vnf_descriptor["vdu"]:
 463                         for interface_descriptor in vdu_descriptor["interface"]:
 464                             if interface_descriptor.get("external-connection-point-ref") == \
 465                                     cp_params["vnfd-connection-point-ref"]:
 466                                 match_cp = True
 467                                 break
 468                         if match_cp:
 469                             break
 470                     else:
 471                         raise LcmException(
 472                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 473                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 474                                 cp_params["member-vnf-index-ref"],
 475                                 cp_params["vnfd-connection-point-ref"],
 476                                 vnf_descriptor["id"]))
 477                     if cp_params.get("ip-address"):
 478                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 479                                                      vdu_descriptor["id"], "interfaces",
 480                                                      interface_descriptor["name"], "ip_address"),
 481                                       cp_params["ip-address"])
 482                     if cp_params.get("mac-address"):
 483                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 484                                                      vdu_descriptor["id"], "interfaces",
 485                                                      interface_descriptor["name"], "mac_address"),
 486                                       cp_params["mac-address"])
 487         return RO_ns_params
 488
 489     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 490         # make a copy to do not change
 491         vdu_create = copy(vdu_create)
 492         vdu_delete = copy(vdu_delete)
 493
 494         vdurs = db_vnfr.get("vdur")
 495         if vdurs is None:
 496             vdurs = []
 497         vdu_index = len(vdurs)
 498         while vdu_index:
 499             vdu_index -= 1
 500             vdur = vdurs[vdu_index]
 501             if vdur.get("pdu-type"):
 502                 continue
 503             vdu_id_ref = vdur["vdu-id-ref"]
 504             if vdu_create and vdu_create.get(vdu_id_ref):
 505                 for index in range(0, vdu_create[vdu_id_ref]):
 506                     vdur = deepcopy(vdur)
 507                     vdur["_id"] = str(uuid4())
 508                     vdur["count-index"] += 1
 509                     vdurs.insert(vdu_index+1+index, vdur)
 510                 del vdu_create[vdu_id_ref]
 511             if vdu_delete and vdu_delete.get(vdu_id_ref):
 512                 del vdurs[vdu_index]
 513                 vdu_delete[vdu_id_ref] -= 1
 514                 if not vdu_delete[vdu_id_ref]:
 515                     del vdu_delete[vdu_id_ref]
 516         # check all operations are done
 517         if vdu_create or vdu_delete:
 518             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 519                 vdu_create))
 520         if vdu_delete:
 521             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 522                 vdu_delete))
 523
 524         vnfr_update = {"vdur": vdurs}
 525         db_vnfr["vdur"] = vdurs
 526         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 527
 528     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 529         """
 530         Updates database nsr with the RO info for the created vld
 531         :param ns_update_nsr: dictionary to be filled with the updated info
 532         :param db_nsr: content of db_nsr. This is also modified
 533         :param nsr_desc_RO: nsr descriptor from RO
 534         :return: Nothing, LcmException is raised on errors
 535         """
 536
 537         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 538             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 539                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 540                     continue
 541                 vld["vim-id"] = net_RO.get("vim_net_id")
 542                 vld["name"] = net_RO.get("vim_name")
 543                 vld["status"] = net_RO.get("status")
 544                 vld["status-detailed"] = net_RO.get("error_msg")
 545                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 546                 break
 547             else:
 548                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 549
 550     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 551         """
 552         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 553         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 554         :param nsr_desc_RO: nsr descriptor from RO
 555         :return: Nothing, LcmException is raised on errors
 556         """
 557         for vnf_index, db_vnfr in db_vnfrs.items():
 558             for vnf_RO in nsr_desc_RO["vnfs"]:
 559                 if vnf_RO["member_vnf_index"] != vnf_index:
 560                     continue
 561                 vnfr_update = {}
 562                 if vnf_RO.get("ip_address"):
 563                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 564                 elif not db_vnfr.get("ip-address"):
 565                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 566
 567                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 568                     vdur_RO_count_index = 0
 569                     if vdur.get("pdu-type"):
 570                         continue
 571                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 572                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 573                             continue
 574                         if vdur["count-index"] != vdur_RO_count_index:
 575                             vdur_RO_count_index += 1
 576                             continue
 577                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 578                         if vdur_RO.get("ip_address"):
 579                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 580                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 581                         vdur["name"] = vdur_RO.get("vim_name")
 582                         vdur["status"] = vdur_RO.get("status")
 583                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 584                         for ifacer in get_iterable(vdur, "interfaces"):
 585                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 586                                 if ifacer["name"] == interface_RO.get("internal_name"):
 587                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 588                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 589                                     break
 590                             else:
 591                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 592                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 593                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 594                         break
 595                     else:
 596                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 597                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 598
 599                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 600                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 601                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 602                             continue
 603                         vld["vim-id"] = net_RO.get("vim_net_id")
 604                         vld["name"] = net_RO.get("vim_name")
 605                         vld["status"] = net_RO.get("status")
 606                         vld["status-detailed"] = net_RO.get("error_msg")
 607                         vnfr_update["vld.{}".format(vld_index)] = vld
 608                         break
 609                     else:
 610                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 611                             vnf_index, vld["id"]))
 612
 613                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 614                 break
 615
 616             else:
 617                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 618
 619     async def instantiate(self, nsr_id, nslcmop_id):
 620         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 621         self.logger.debug(logging_text + "Enter")
 622         # get all needed from database
 623         start_deploy = time()
 624         db_nsr = None
 625         db_nslcmop = None
 626         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 627         db_nslcmop_update = {}
 628         nslcmop_operation_state = None
 629         db_vnfrs = {}
 630         RO_descriptor_number = 0   # number of descriptors created at RO
 631         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 632         n2vc_info = {}
 633         exc = None
 634         try:
 635             step = "Getting nslcmop={} from db".format(nslcmop_id)
 636             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 637             step = "Getting nsr={} from db".format(nsr_id)
 638             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 639             ns_params = db_nslcmop.get("operationParams")
 640             nsd = db_nsr["nsd"]
 641             nsr_name = db_nsr["name"]   # TODO short-name??
 642
 643             # look if previous tasks in process
 644             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
 645             if task_dependency:
 646                 step = db_nslcmop_update["detailed-status"] = \
 647                     "Waiting for related tasks to be completed: {}".format(task_name)
 648                 self.logger.debug(logging_text + step)
 649                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 650                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 651                 if pending:
 652                     raise LcmException("Timeout waiting related tasks to be completed")
 653
 654             step = "Getting vnfrs from db"
 655             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 656             db_vnfds_ref = {}
 657             db_vnfds = {}
 658             for vnfr in db_vnfrs_list:
 659                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 660                 vnfd_id = vnfr["vnfd-id"]
 661                 vnfd_ref = vnfr["vnfd-ref"]
 662                 if vnfd_id not in db_vnfds:
 663                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 664                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 665                     db_vnfds_ref[vnfd_ref] = vnfd
 666                     db_vnfds[vnfd_id] = vnfd
 667
 668             # Get or generates the _admin.deployed,VCA list
 669             vca_deployed_list = None
 670             if db_nsr["_admin"].get("deployed"):
 671                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 672             if vca_deployed_list is None:
 673                 vca_deployed_list = []
 674                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 675             elif isinstance(vca_deployed_list, dict):
 676                 # maintain backward compatibility. Change a dict to list at database
 677                 vca_deployed_list = list(vca_deployed_list.values())
 678                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 679
 680             db_nsr_update["detailed-status"] = "creating"
 681             db_nsr_update["operational-status"] = "init"
 682             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 683                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 684                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 685                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 686
 687             RO = ROclient.ROClient(self.loop, **self.ro_config)
 688
 689             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 690             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 691             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 692
 693             # get vnfds, instantiate at RO
 694             for c_vnf in nsd.get("constituent-vnfd", ()):
 695                 member_vnf_index = c_vnf["member-vnf-index"]
 696                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 697                 vnfd_ref = vnfd["id"]
 698                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member-vnf-index='{}' at RO".format(
 699                     vnfd_ref, member_vnf_index)
 700                 # self.logger.debug(logging_text + step)
 701                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 702                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 703                 RO_descriptor_number += 1
 704
 705                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 706                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 707                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 708                         break
 709                 else:
 710                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
 711                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
 712
 713                 # look if present
 714                 RO_update = {"member-vnf-index": member_vnf_index}
 715                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
 716                 if vnfd_list:
 717                     RO_update["id"] = vnfd_list[0]["uuid"]
 718                     self.logger.debug(logging_text + "vnfd='{}'  member-vnf-index='{}' exists at RO. Using RO_id={}".
 719                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
 720                 else:
 721                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
 722                                            get("additionalParamsForVnf"), nsr_id)
 723                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
 724                     RO_update["id"] = desc["uuid"]
 725                     self.logger.debug(logging_text + "vnfd='{}' member-vnf-index='{}' created at RO. RO_id={}".format(
 726                         vnfd_ref, member_vnf_index, desc["uuid"]))
 727                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
 728                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
 729                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 730
 731             # create nsd at RO
 732             nsd_ref = nsd["id"]
 733             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
 734             # self.logger.debug(logging_text + step)
 735
 736             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
 737             RO_descriptor_number += 1
 738             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
 739             if nsd_list:
 740                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
 741                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
 742                     nsd_ref, RO_nsd_uuid))
 743             else:
 744                 nsd_RO = deepcopy(nsd)
 745                 nsd_RO["id"] = RO_osm_nsd_id
 746                 nsd_RO.pop("_id", None)
 747                 nsd_RO.pop("_admin", None)
 748                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
 749                     member_vnf_index = c_vnf["member-vnf-index"]
 750                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 751                 for c_vld in nsd_RO.get("vld", ()):
 752                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
 753                         member_vnf_index = cp["member-vnf-index-ref"]
 754                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 755
 756                 desc = await RO.create("nsd", descriptor=nsd_RO)
 757                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 758                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
 759                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
 760             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 761
 762             # Crate ns at RO
 763             # if present use it unless in error status
 764             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
 765             if RO_nsr_id:
 766                 try:
 767                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
 768                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
 769                     desc = await RO.show("ns", RO_nsr_id)
 770                 except ROclient.ROClientException as e:
 771                     if e.http_code != HTTPStatus.NOT_FOUND:
 772                         raise
 773                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 774                 if RO_nsr_id:
 775                     ns_status, ns_status_info = RO.check_ns_status(desc)
 776                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 777                     if ns_status == "ERROR":
 778                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
 779                         self.logger.debug(logging_text + step)
 780                         await RO.delete("ns", RO_nsr_id)
 781                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 782             if not RO_nsr_id:
 783                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
 784                 # self.logger.debug(logging_text + step)
 785
 786                 # check if VIM is creating and wait  look if previous tasks in process
 787                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
 788                 if task_dependency:
 789                     step = "Waiting for related tasks to be completed: {}".format(task_name)
 790                     self.logger.debug(logging_text + step)
 791                     await asyncio.wait(task_dependency, timeout=3600)
 792                 if ns_params.get("vnf"):
 793                     for vnf in ns_params["vnf"]:
 794                         if "vimAccountId" in vnf:
 795                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 796                                                                                         vnf["vimAccountId"])
 797                         if task_dependency:
 798                             step = "Waiting for related tasks to be completed: {}".format(task_name)
 799                             self.logger.debug(logging_text + step)
 800                             await asyncio.wait(task_dependency, timeout=3600)
 801
 802                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
 803
 804                 # feature 1429. Add n2vc public key to needed VMs
 805                 n2vc_key = await self.n2vc.GetPublicKey()
 806                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, [n2vc_key])
 807
 808                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
 809                 desc = await RO.create("ns", descriptor=RO_ns_params,
 810                                        name=db_nsr["name"],
 811                                        scenario=RO_nsd_uuid)
 812                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
 813                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 814                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
 815                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 816             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 817
 818             # wait until NS is ready
 819             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
 820             detailed_status_old = None
 821             self.logger.debug(logging_text + step)
 822
 823             while time() <= start_deploy + self.total_deploy_timeout:
 824                 desc = await RO.show("ns", RO_nsr_id)
 825                 ns_status, ns_status_info = RO.check_ns_status(desc)
 826                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 827                 if ns_status == "ERROR":
 828                     raise ROclient.ROClientException(ns_status_info)
 829                 elif ns_status == "BUILD":
 830                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 831                 elif ns_status == "ACTIVE":
 832                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
 833                     try:
 834                         self.ns_update_vnfr(db_vnfrs, desc)
 835                         break
 836                     except LcmExceptionNoMgmtIP:
 837                         pass
 838                 else:
 839                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 840                 if detailed_status != detailed_status_old:
 841                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
 842                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 843                 await asyncio.sleep(5, loop=self.loop)
 844             else:   # total_deploy_timeout
 845                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 846
 847             step = "Updating NSR"
 848             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 849
 850             db_nsr["detailed-status"] = "Configuring vnfr"
 851             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 852
 853             # The parameters we'll need to deploy a charm
 854             number_to_configure = 0
 855
 856             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info):
 857                 """An inner function to deploy the charm from either vnf or vdu
 858                 vnf_index is mandatory. vdu_id can be None for a vnf configuration or the id for vdu configuration
 859                 """
 860                 if not charm_params["rw_mgmt_ip"]:
 861                     raise LcmException("vnfd/vdu has not management ip address to configure it")
 862                 # Login to the VCA.
 863                 # if number_to_configure == 0:
 864                 #     self.logger.debug("Logging into N2VC...")
 865                 #     task = asyncio.ensure_future(self.n2vc.login())
 866                 #     yield from asyncio.wait_for(task, 30.0)
 867                 #     self.logger.debug("Logged into N2VC!")
 868
 869                 # # await self.n2vc.login()
 870
 871                 # Note: The charm needs to exist on disk at the location
 872                 # specified by charm_path.
 873                 base_folder = vnfd["_admin"]["storage"]
 874                 storage_params = self.fs.get_params()
 875                 charm_path = "{}{}/{}/charms/{}".format(
 876                     storage_params["path"],
 877                     base_folder["folder"],
 878                     base_folder["pkg-dir"],
 879                     proxy_charm
 880                 )
 881
 882                 # ns_name will be ignored in the current version of N2VC
 883                 # but will be implemented for the next point release.
 884                 model_name = "default"    # TODO bug 585  nsr_id
 885                 if vdu_id:
 886                     vdu_id_text = vdu_id + "-"
 887                 else:
 888                     vdu_id_text = "-"
 889                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index, vdu_id_text)
 890
 891                 vca_index = len(vca_deployed_list)
 892                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 893                 # 26*26 charm in the same NS
 894                 application_name = application_name[0:48]
 895                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 896                 vca_deployed_ = {
 897                     "member-vnf-index": vnf_index,
 898                     "vdu_id": vdu_id,
 899                     "model": model_name,
 900                     "application": application_name,
 901                     "operational-status": "init",
 902                     "detailed-status": "",
 903                     "vnfd_id": vnfd_id,
 904                     "vdu_name": vdu_name,
 905                     "vdu_count_index": vdu_count_index,
 906                 }
 907                 vca_deployed_list.append(vca_deployed_)
 908                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 909                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 910
 911                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 912                                                                                                 proxy_charm))
 913                 if not n2vc_info:
 914                     n2vc_info["nsr_id"] = nsr_id
 915                     n2vc_info["nslcmop_id"] = nslcmop_id
 916                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 917                     n2vc_info["lcmOperationType"] = "instantiate"
 918                     n2vc_info["deployed"] = vca_deployed_list
 919                     n2vc_info["db_update"] = db_nsr_update
 920                 task = asyncio.ensure_future(
 921                     self.n2vc.DeployCharms(
 922                         model_name,          # The network service name
 923                         application_name,    # The application name
 924                         vnfd,                # The vnf descriptor
 925                         charm_path,          # Path to charm
 926                         charm_params,        # Runtime params, like mgmt ip
 927                         {},                  # for native charms only
 928                         self.n2vc_callback,  # Callback for status changes
 929                         n2vc_info,           # Callback parameter
 930                         None,                # Callback parameter (task)
 931                     )
 932                 )
 933                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 934                                                          n2vc_info))
 935                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 936
 937             step = "Looking for needed vnfd to configure"
 938             self.logger.debug(logging_text + step)
 939
 940             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 941                 vnfd_id = c_vnf["vnfd-id-ref"]
 942                 vnf_index = str(c_vnf["member-vnf-index"])
 943                 vnfd = db_vnfds_ref[vnfd_id]
 944
 945                 # Get additional parameters
 946                 vnfr_params = {}
 947                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 948                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 949                 for k, v in vnfr_params.items():
 950                     if isinstance(v, str) and v.startswith("!!yaml "):
 951                         vnfr_params[k] = yaml.safe_load(v[7:])
 952
 953                 # Check if this VNF has a charm configuration
 954                 vnf_config = vnfd.get("vnf-configuration")
 955                 if vnf_config and vnf_config.get("juju"):
 956                     proxy_charm = vnf_config["juju"]["charm"]
 957
 958                     if proxy_charm:
 959                         step = "connecting to N2VC to configure vnf {}".format(vnf_index)
 960                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 961                         charm_params = {
 962                             "user_values": vnfr_params,
 963                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 964                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {}
 965                         }
 966
 967                         # Login to the VCA. If there are multiple calls to login(),
 968                         # subsequent calls will be a nop and return immediately.
 969                         await self.n2vc.login()
 970                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 971                         number_to_configure += 1
 972
 973                 # Deploy charms for each VDU that supports one.
 974                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 975                     vdu_config = vdu.get('vdu-configuration')
 976                     proxy_charm = None
 977
 978                     if vdu_config and vdu_config.get("juju"):
 979                         proxy_charm = vdu_config["juju"]["charm"]
 980
 981                         if proxy_charm:
 982                             step = "connecting to N2VC to configure vdu {} from vnf {}".format(vdu["id"], vnf_index)
 983                             await self.n2vc.login()
 984                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 985                             # TODO for the moment only first vdu_id contains a charm deployed
 986                             if vdur["vdu-id-ref"] != vdu["id"]:
 987                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
 988                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 989                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 990                             charm_params = {
 991                                 "user_values": vnfr_params,
 992                                 "rw_mgmt_ip": vdur["ip-address"],
 993                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
 994                             }
 995                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 996                                          charm_params, n2vc_info)
 997                             number_to_configure += 1
 998
 999             db_nsr_update["operational-status"] = "running"
1000             configuration_failed = False
1001             if number_to_configure:
1002                 old_status = "configuring: init: {}".format(number_to_configure)
1003                 db_nsr_update["config-status"] = old_status
1004                 db_nsr_update["detailed-status"] = old_status
1005                 db_nslcmop_update["detailed-status"] = old_status
1006
1007                 # wait until all are configured.
1008                 while time() <= start_deploy + self.total_deploy_timeout:
1009                     if db_nsr_update:
1010                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1011                     if db_nslcmop_update:
1012                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1013                     # TODO add a fake task that set n2vc_event after some time
1014                     await n2vc_info["n2vc_event"].wait()
1015                     n2vc_info["n2vc_event"].clear()
1016                     all_active = True
1017                     status_map = {}
1018                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1019                     now = time()
1020                     for vca_deployed in vca_deployed_list:
1021                         vca_status = vca_deployed["operational-status"]
1022                         if vca_status not in status_map:
1023                             # Initialize it
1024                             status_map[vca_status] = 0
1025                         status_map[vca_status] += 1
1026
1027                         if vca_status == "active":
1028                             vca_deployed.pop("time_first_error", None)
1029                             vca_deployed.pop("status_first_error", None)
1030                             continue
1031
1032                         all_active = False
1033                         if vca_status in ("error", "blocked"):
1034                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1035                             # if not first time in this status error
1036                             if not vca_deployed.get("time_first_error"):
1037                                 vca_deployed["time_first_error"] = now
1038                                 continue
1039                         if vca_deployed.get("time_first_error") and \
1040                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1041                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1042                                                    .format(vca_deployed["member-vnf-index"],
1043                                                            vca_deployed["vdu_id"], vca_status,
1044                                                            vca_deployed["detailed-status-error"]))
1045
1046                     if all_active:
1047                         break
1048                     elif n2vc_error_text:
1049                         db_nsr_update["config-status"] = "failed"
1050                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1051                         db_nsr_update["detailed-status"] = error_text
1052                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1053                         db_nslcmop_update["detailed-status"] = error_text
1054                         db_nslcmop_update["statusEnteredTime"] = time()
1055                         configuration_failed = True
1056                         break
1057                     else:
1058                         cs = "configuring: "
1059                         separator = ""
1060                         for status, num in status_map.items():
1061                             cs += separator + "{}: {}".format(status, num)
1062                             separator = ", "
1063                         if old_status != cs:
1064                             db_nsr_update["config-status"] = cs
1065                             db_nsr_update["detailed-status"] = cs
1066                             db_nslcmop_update["detailed-status"] = cs
1067                             old_status = cs
1068                 else:   # total_deploy_timeout
1069                     raise LcmException("Timeout waiting ns to be configured")
1070
1071             if not configuration_failed:
1072                 # all is done
1073                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1074                 db_nslcmop_update["statusEnteredTime"] = time()
1075                 db_nslcmop_update["detailed-status"] = "done"
1076                 db_nsr_update["config-status"] = "configured"
1077                 db_nsr_update["detailed-status"] = "done"
1078
1079             return
1080
1081         except (ROclient.ROClientException, DbException, LcmException) as e:
1082             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1083             exc = e
1084         except asyncio.CancelledError:
1085             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1086             exc = "Operation was cancelled"
1087         except Exception as e:
1088             exc = traceback.format_exc()
1089             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1090                                  exc_info=True)
1091         finally:
1092             if exc:
1093                 if db_nsr:
1094                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1095                     db_nsr_update["operational-status"] = "failed"
1096                 if db_nslcmop:
1097                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1098                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1099                     db_nslcmop_update["statusEnteredTime"] = time()
1100             try:
1101                 if db_nsr:
1102                     db_nsr_update["_admin.nslcmop"] = None
1103                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1104                 if db_nslcmop_update:
1105                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1106             except DbException as e:
1107                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1108             if nslcmop_operation_state:
1109                 try:
1110                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1111                                                                    "operationState": nslcmop_operation_state},
1112                                             loop=self.loop)
1113                 except Exception as e:
1114                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1115
1116             self.logger.debug(logging_text + "Exit")
1117             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1118
1119     async def _destroy_charm(self, model, application):
1120         """
1121         Order N2VC destroy a charm
1122         :param model:
1123         :param application:
1124         :return: True if charm does not exist. False if it exist
1125         """
1126         if not await self.n2vc.HasApplication(model, application):
1127             return True  # Already removed
1128         await self.n2vc.RemoveCharms(model, application)
1129         return False
1130
1131     async def _wait_charm_destroyed(self, model, application, timeout):
1132         """
1133         Wait until charm does not exist
1134         :param model:
1135         :param application:
1136         :param timeout:
1137         :return: True if not exist, False if timeout
1138         """
1139         while True:
1140             if not await self.n2vc.HasApplication(model, application):
1141                 return True
1142             if timeout < 0:
1143                 return False
1144             await asyncio.sleep(10)
1145             timeout -= 10
1146
1147     async def terminate(self, nsr_id, nslcmop_id):
1148         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1149         self.logger.debug(logging_text + "Enter")
1150         db_nsr = None
1151         db_nslcmop = None
1152         exc = None
1153         failed_detail = []   # annotates all failed error messages
1154         vca_time_destroy = None   # time of where destroy charm order
1155         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1156         db_nslcmop_update = {}
1157         nslcmop_operation_state = None
1158         autoremove = False  # autoremove after terminated
1159         try:
1160             step = "Getting nslcmop={} from db".format(nslcmop_id)
1161             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1162             step = "Getting nsr={} from db".format(nsr_id)
1163             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1164             # nsd = db_nsr["nsd"]
1165             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1166             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1167                 return
1168             # #TODO check if VIM is creating and wait
1169             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1170
1171             db_nsr_update["operational-status"] = "terminating"
1172             db_nsr_update["config-status"] = "terminating"
1173
1174             if nsr_deployed and nsr_deployed.get("VCA"):
1175                 try:
1176                     step = "Scheduling configuration charms removing"
1177                     db_nsr_update["detailed-status"] = "Deleting charms"
1178                     self.logger.debug(logging_text + step)
1179                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1180                     # for backward compatibility
1181                     if isinstance(nsr_deployed["VCA"], dict):
1182                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1183                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1184                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1185
1186                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1187                         if vca_deployed:
1188                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1189                                 vca_deployed.clear()
1190                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1191                             else:
1192                                 vca_time_destroy = time()
1193                 except Exception as e:
1194                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1195
1196             # remove from RO
1197             RO_fail = False
1198             RO = ROclient.ROClient(self.loop, **self.ro_config)
1199
1200             # Delete ns
1201             RO_nsr_id = RO_delete_action = None
1202             if nsr_deployed and nsr_deployed.get("RO"):
1203                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1204                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1205             try:
1206                 if RO_nsr_id:
1207                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1208                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1209                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1210                     self.logger.debug(logging_text + step)
1211                     desc = await RO.delete("ns", RO_nsr_id)
1212                     RO_delete_action = desc["action_id"]
1213                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1214                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1215                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1216                 if RO_delete_action:
1217                     # wait until NS is deleted from VIM
1218                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1219                         format(RO_nsr_id, RO_delete_action)
1220                     detailed_status_old = None
1221                     self.logger.debug(logging_text + step)
1222
1223                     delete_timeout = 20 * 60   # 20 minutes
1224                     while delete_timeout > 0:
1225                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1226                                              extra_item_id=RO_delete_action)
1227                         ns_status, ns_status_info = RO.check_action_status(desc)
1228                         if ns_status == "ERROR":
1229                             raise ROclient.ROClientException(ns_status_info)
1230                         elif ns_status == "BUILD":
1231                             detailed_status = step + "; {}".format(ns_status_info)
1232                         elif ns_status == "ACTIVE":
1233                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1234                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1235                             break
1236                         else:
1237                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1238                         if detailed_status != detailed_status_old:
1239                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1240                                 db_nsr_update["detailed-status"] = detailed_status
1241                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1242                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1243                         await asyncio.sleep(5, loop=self.loop)
1244                         delete_timeout -= 5
1245                     else:  # delete_timeout <= 0:
1246                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1247
1248             except ROclient.ROClientException as e:
1249                 if e.http_code == 404:  # not found
1250                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1251                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1252                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1253                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1254                 elif e.http_code == 409:   # conflict
1255                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1256                     self.logger.debug(logging_text + failed_detail[-1])
1257                     RO_fail = True
1258                 else:
1259                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1260                     self.logger.error(logging_text + failed_detail[-1])
1261                     RO_fail = True
1262
1263             # Delete nsd
1264             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1265                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1266                 try:
1267                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1268                         "Deleting nsd at RO"
1269                     await RO.delete("nsd", RO_nsd_id)
1270                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1271                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1272                 except ROclient.ROClientException as e:
1273                     if e.http_code == 404:  # not found
1274                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1275                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1276                     elif e.http_code == 409:   # conflict
1277                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1278                         self.logger.debug(logging_text + failed_detail[-1])
1279                         RO_fail = True
1280                     else:
1281                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1282                         self.logger.error(logging_text + failed_detail[-1])
1283                         RO_fail = True
1284
1285             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1286                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1287                     if not vnf_deployed or not vnf_deployed["id"]:
1288                         continue
1289                     try:
1290                         RO_vnfd_id = vnf_deployed["id"]
1291                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1292                             "Deleting member-vnf-index={} RO_vnfd_id={} from RO".format(
1293                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1294                         await RO.delete("vnfd", RO_vnfd_id)
1295                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1296                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1297                     except ROclient.ROClientException as e:
1298                         if e.http_code == 404:  # not found
1299                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1300                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1301                         elif e.http_code == 409:   # conflict
1302                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1303                             self.logger.debug(logging_text + failed_detail[-1])
1304                         else:
1305                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1306                             self.logger.error(logging_text + failed_detail[-1])
1307
1308             # wait until charm deleted
1309             if vca_time_destroy:
1310                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1311                     "Waiting for deletion of configuration charms"
1312                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1313                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1314                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1315                     if not vca_deployed:
1316                         continue
1317                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1318                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1319                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1320                                                             timeout):
1321                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1322                             vca_deployed["application"]))
1323                     else:
1324                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1325
1326             if failed_detail:
1327                 self.logger.error(logging_text + " ;".join(failed_detail))
1328                 db_nsr_update["operational-status"] = "failed"
1329                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1330                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1331                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1332                 db_nslcmop_update["statusEnteredTime"] = time()
1333             else:
1334                 db_nsr_update["operational-status"] = "terminated"
1335                 db_nsr_update["detailed-status"] = "Done"
1336                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1337                 db_nslcmop_update["detailed-status"] = "Done"
1338                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1339                 db_nslcmop_update["statusEnteredTime"] = time()
1340                 if db_nslcmop["operationParams"].get("autoremove"):
1341                     autoremove = True
1342
1343         except (ROclient.ROClientException, DbException) as e:
1344             self.logger.error(logging_text + "Exit Exception {}".format(e))
1345             exc = e
1346         except asyncio.CancelledError:
1347             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1348             exc = "Operation was cancelled"
1349         except Exception as e:
1350             exc = traceback.format_exc()
1351             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1352         finally:
1353             if exc and db_nslcmop:
1354                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1355                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1356                 db_nslcmop_update["statusEnteredTime"] = time()
1357             try:
1358                 if db_nslcmop and db_nslcmop_update:
1359                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1360                 if db_nsr:
1361                     db_nsr_update["_admin.nslcmop"] = None
1362                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1363             except DbException as e:
1364                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1365             if nslcmop_operation_state:
1366                 try:
1367                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1368                                                                  "operationState": nslcmop_operation_state,
1369                                                                  "autoremove": autoremove},
1370                                             loop=self.loop)
1371                 except Exception as e:
1372                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1373             self.logger.debug(logging_text + "Exit")
1374             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1375
1376     @staticmethod
1377     def _map_primitive_params(primitive_desc, params, instantiation_params):
1378         """
1379         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1380         The default-value is used. If it is between < > it look for a value at instantiation_params
1381         :param primitive_desc: portion of VNFD/NSD that describes primitive
1382         :param params: Params provided by user
1383         :param instantiation_params: Instantiation params provided by user
1384         :return: a dictionary with the calculated params
1385         """
1386         calculated_params = {}
1387         for parameter in primitive_desc.get("parameter", ()):
1388             param_name = parameter["name"]
1389             if param_name in params:
1390                 calculated_params[param_name] = params[param_name]
1391             elif "default-value" in parameter:
1392                 calculated_params[param_name] = parameter["default-value"]
1393                 if isinstance(parameter["default-value"], str) and parameter["default-value"].startswith("<") and \
1394                         parameter["default-value"].endswith(">"):
1395                     if parameter["default-value"][1:-1] in instantiation_params:
1396                         calculated_params[param_name] = instantiation_params[parameter["default-value"][1:-1]]
1397                     else:
1398                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1399                                            format(parameter["default-value"], primitive_desc["name"]))
1400             else:
1401                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1402                                    format(param_name, primitive_desc["name"]))
1403
1404             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1405                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1406                                                                width=256)
1407             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1408                 calculated_params[param_name] = calculated_params[param_name][7:]
1409         return calculated_params
1410
1411     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1412                                     primitive, primitive_params):
1413         start_primitive_time = time()
1414         try:
1415             for vca_deployed in db_deployed["VCA"]:
1416                 if not vca_deployed:
1417                     continue
1418                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1419                     continue
1420                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1421                     continue
1422                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1423                     continue
1424                 break
1425             else:
1426                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1427                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1428             model_name = vca_deployed.get("model")
1429             application_name = vca_deployed.get("application")
1430             if not model_name or not application_name:
1431                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1432                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1433                                                                        vdu_count_index))
1434             if vca_deployed["operational-status"] != "active":
1435                 raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1436                     member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1437             callback = None  # self.n2vc_callback
1438             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1439             await self.n2vc.login()
1440             primitive_id = await self.n2vc.ExecutePrimitive(
1441                 model_name,
1442                 application_name,
1443                 primitive,
1444                 callback,
1445                 *callback_args,
1446                 **primitive_params
1447             )
1448             while time() - start_primitive_time < self.timeout_primitive:
1449                 primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1450                 if primitive_result_ == "running":
1451                     pass
1452                 elif primitive_result_ in ("completed", "failed"):
1453                     primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1454                     detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1455                     break
1456                 else:
1457                     detailed_result = "Invalid N2VC.GetPrimitiveStatus = {} obtained".format(primitive_result_)
1458                     primitive_result = "FAILED"
1459                     break
1460                 await asyncio.sleep(5)
1461             else:
1462                 raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1463             return primitive_result, detailed_result
1464         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
1465             return "FAILED", str(e)
1466
1467     async def action(self, nsr_id, nslcmop_id):
1468         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
1469         self.logger.debug(logging_text + "Enter")
1470         # get all needed from database
1471         db_nsr = None
1472         db_nslcmop = None
1473         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1474         db_nslcmop_update = {}
1475         nslcmop_operation_state = None
1476         exc = None
1477         try:
1478             step = "Getting information from database"
1479             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1480             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1481
1482             nsr_deployed = db_nsr["_admin"].get("deployed")
1483             vnf_index = db_nslcmop["operationParams"]["member_vnf_index"]
1484             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1485             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1486             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1487
1488             step = "Getting vnfr from database"
1489             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1490             step = "Getting vnfd from database"
1491             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1492
1493             # look if previous tasks in process
1494             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1495             if task_dependency:
1496                 step = db_nslcmop_update["detailed-status"] = \
1497                     "Waiting for related tasks to be completed: {}".format(task_name)
1498                 self.logger.debug(logging_text + step)
1499                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1500                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1501                 if pending:
1502                     raise LcmException("Timeout waiting related tasks to be completed")
1503
1504             # for backward compatibility
1505             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1506                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1507                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1508                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1509
1510             primitive = db_nslcmop["operationParams"]["primitive"]
1511             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
1512
1513             # look for primitive
1514             config_primitive_desc = None
1515             if vdu_id:
1516                 for vdu in get_iterable(db_vnfd, "vdu"):
1517                     if vdu_id == vdu["id"]:
1518                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
1519                             if config_primitive["name"] == primitive:
1520                                 config_primitive_desc = config_primitive
1521                                 break
1522             for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1523                 if config_primitive["name"] == primitive:
1524                     config_primitive_desc = config_primitive
1525                     break
1526             if not config_primitive_desc:
1527                 raise LcmException("Primitive {} not found at vnf-configuration:config-primitive or vdu:"
1528                                    "vdu-configuration:config-primitive".format(primitive))
1529
1530             vnfr_params = {}
1531             if db_vnfr.get("additionalParamsForVnf"):
1532                 vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1533
1534             # TODO check if ns is in a proper status
1535             result, result_detail = await self._ns_execute_primitive(
1536                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1537                 self._map_primitive_params(config_primitive_desc, primitive_params, vnfr_params))
1538             db_nslcmop_update["detailed-status"] = result_detail
1539             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
1540             db_nslcmop_update["statusEnteredTime"] = time()
1541             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
1542             return  # database update is called inside finally
1543
1544         except (DbException, LcmException) as e:
1545             self.logger.error(logging_text + "Exit Exception {}".format(e))
1546             exc = e
1547         except asyncio.CancelledError:
1548             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1549             exc = "Operation was cancelled"
1550         except Exception as e:
1551             exc = traceback.format_exc()
1552             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1553         finally:
1554             if exc and db_nslcmop:
1555                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1556                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1557                 db_nslcmop_update["statusEnteredTime"] = time()
1558             try:
1559                 if db_nslcmop_update:
1560                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1561                 if db_nsr:
1562                     db_nsr_update["_admin.nslcmop"] = None
1563                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1564             except DbException as e:
1565                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1566             self.logger.debug(logging_text + "Exit")
1567             if nslcmop_operation_state:
1568                 try:
1569                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1570                                                                "operationState": nslcmop_operation_state},
1571                                             loop=self.loop)
1572                 except Exception as e:
1573                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1574             self.logger.debug(logging_text + "Exit")
1575             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
1576
1577     async def scale(self, nsr_id, nslcmop_id):
1578         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
1579         self.logger.debug(logging_text + "Enter")
1580         # get all needed from database
1581         db_nsr = None
1582         db_nslcmop = None
1583         db_nslcmop_update = {}
1584         nslcmop_operation_state = None
1585         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1586         exc = None
1587         # in case of error, indicates what part of scale was failed to put nsr at error status
1588         scale_process = None
1589         old_operational_status = ""
1590         old_config_status = ""
1591         vnfr_scaled = False
1592         try:
1593             step = "Getting nslcmop from database"
1594             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1595             step = "Getting nsr from database"
1596             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1597
1598             old_operational_status = db_nsr["operational-status"]
1599             old_config_status = db_nsr["config-status"]
1600
1601             # look if previous tasks in process
1602             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1603             if task_dependency:
1604                 step = db_nslcmop_update["detailed-status"] = \
1605                     "Waiting for related tasks to be completed: {}".format(task_name)
1606                 self.logger.debug(logging_text + step)
1607                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1608                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1609                 if pending:
1610                     raise LcmException("Timeout waiting related tasks to be completed")
1611
1612             step = "Parsing scaling parameters"
1613             db_nsr_update["operational-status"] = "scaling"
1614             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1615             nsr_deployed = db_nsr["_admin"].get("deployed")
1616             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
1617             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
1618             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
1619             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
1620             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
1621
1622             # for backward compatibility
1623             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1624                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1625                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1626                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1627
1628             step = "Getting vnfr from database"
1629             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1630             step = "Getting vnfd from database"
1631             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1632             step = "Getting scaling-group-descriptor"
1633             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
1634                 if scaling_descriptor["name"] == scaling_group:
1635                     break
1636             else:
1637                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
1638                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
1639             # cooldown_time = 0
1640             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
1641             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
1642             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
1643             #         break
1644
1645             # TODO check if ns is in a proper status
1646             step = "Sending scale order to RO"
1647             nb_scale_op = 0
1648             if not db_nsr["_admin"].get("scaling-group"):
1649                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
1650                 admin_scale_index = 0
1651             else:
1652                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
1653                     if admin_scale_info["name"] == scaling_group:
1654                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
1655                         break
1656                 else:  # not found, set index one plus last element and add new entry with the name
1657                     admin_scale_index += 1
1658                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
1659             RO_scaling_info = []
1660             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
1661             if scaling_type == "SCALE_OUT":
1662                 # count if max-instance-count is reached
1663                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
1664                     max_instance_count = int(scaling_descriptor["max-instance-count"])
1665                     if nb_scale_op >= max_instance_count:
1666                         raise LcmException("reached the limit of {} (max-instance-count) scaling-out operations for the"
1667                                            " scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1668                 nb_scale_op = nb_scale_op + 1
1669                 vdu_scaling_info["scaling_direction"] = "OUT"
1670                 vdu_scaling_info["vdu-create"] = {}
1671                 for vdu_scale_info in scaling_descriptor["vdu"]:
1672                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1673                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
1674                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1675             elif scaling_type == "SCALE_IN":
1676                 # count if min-instance-count is reached
1677                 min_instance_count = 0
1678                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
1679                     min_instance_count = int(scaling_descriptor["min-instance-count"])
1680                 if nb_scale_op <= min_instance_count:
1681                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
1682                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1683                 nb_scale_op = nb_scale_op - 1
1684                 vdu_scaling_info["scaling_direction"] = "IN"
1685                 vdu_scaling_info["vdu-delete"] = {}
1686                 for vdu_scale_info in scaling_descriptor["vdu"]:
1687                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1688                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
1689                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1690
1691             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
1692             vdu_create = vdu_scaling_info.get("vdu-create")
1693             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
1694             if vdu_scaling_info["scaling_direction"] == "IN":
1695                 for vdur in reversed(db_vnfr["vdur"]):
1696                     if vdu_delete.get(vdur["vdu-id-ref"]):
1697                         vdu_delete[vdur["vdu-id-ref"]] -= 1
1698                         vdu_scaling_info["vdu"].append({
1699                             "name": vdur["name"],
1700                             "vdu_id": vdur["vdu-id-ref"],
1701                             "interface": []
1702                         })
1703                         for interface in vdur["interfaces"]:
1704                             vdu_scaling_info["vdu"][-1]["interface"].append({
1705                                 "name": interface["name"],
1706                                 "ip_address": interface["ip-address"],
1707                                 "mac_address": interface.get("mac-address"),
1708                             })
1709                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
1710
1711             # execute primitive service PRE-SCALING
1712             step = "Executing pre-scale vnf-config-primitive"
1713             if scaling_descriptor.get("scaling-config-action"):
1714                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1715                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
1716                             and scaling_type == "SCALE_IN":
1717                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1718                         step = db_nslcmop_update["detailed-status"] = \
1719                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
1720
1721                         # look for primitive
1722                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1723                             if config_primitive["name"] == vnf_config_primitive:
1724                                 break
1725                         else:
1726                             raise LcmException(
1727                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
1728                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
1729                                 "primitive".format(scaling_group, config_primitive))
1730
1731                         vnfr_params = {"<VDU_SCALE_INFO>": vdu_scaling_info}
1732                         if db_vnfr.get("additionalParamsForVnf"):
1733                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1734
1735                         scale_process = "VCA"
1736                         db_nsr_update["config-status"] = "configuring pre-scaling"
1737                         result, result_detail = await self._ns_execute_primitive(
1738                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1739                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1740                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1741                             vnf_config_primitive, result, result_detail))
1742                         if result == "FAILED":
1743                             raise LcmException(result_detail)
1744                         db_nsr_update["config-status"] = old_config_status
1745                         scale_process = None
1746
1747             if RO_scaling_info:
1748                 scale_process = "RO"
1749                 RO = ROclient.ROClient(self.loop, **self.ro_config)
1750                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
1751                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
1752                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
1753                 # wait until ready
1754                 RO_nslcmop_id = RO_desc["instance_action_id"]
1755                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
1756
1757                 RO_task_done = False
1758                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
1759                 detailed_status_old = None
1760                 self.logger.debug(logging_text + step)
1761
1762                 deployment_timeout = 1 * 3600   # One hour
1763                 while deployment_timeout > 0:
1764                     if not RO_task_done:
1765                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1766                                              extra_item_id=RO_nslcmop_id)
1767                         ns_status, ns_status_info = RO.check_action_status(desc)
1768                         if ns_status == "ERROR":
1769                             raise ROclient.ROClientException(ns_status_info)
1770                         elif ns_status == "BUILD":
1771                             detailed_status = step + "; {}".format(ns_status_info)
1772                         elif ns_status == "ACTIVE":
1773                             RO_task_done = True
1774                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1775                             self.logger.debug(logging_text + step)
1776                         else:
1777                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1778                     else:
1779                         desc = await RO.show("ns", RO_nsr_id)
1780                         ns_status, ns_status_info = RO.check_ns_status(desc)
1781                         if ns_status == "ERROR":
1782                             raise ROclient.ROClientException(ns_status_info)
1783                         elif ns_status == "BUILD":
1784                             detailed_status = step + "; {}".format(ns_status_info)
1785                         elif ns_status == "ACTIVE":
1786                             step = detailed_status = \
1787                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
1788                             if not vnfr_scaled:
1789                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
1790                                 vnfr_scaled = True
1791                             try:
1792                                 desc = await RO.show("ns", RO_nsr_id)
1793                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
1794                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
1795                                 break
1796                             except LcmExceptionNoMgmtIP:
1797                                 pass
1798                         else:
1799                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1800                     if detailed_status != detailed_status_old:
1801                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
1802                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1803
1804                     await asyncio.sleep(5, loop=self.loop)
1805                     deployment_timeout -= 5
1806                 if deployment_timeout <= 0:
1807                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
1808
1809                 # update VDU_SCALING_INFO with the obtained ip_addresses
1810                 if vdu_scaling_info["scaling_direction"] == "OUT":
1811                     for vdur in reversed(db_vnfr["vdur"]):
1812                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
1813                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
1814                             vdu_scaling_info["vdu"].append({
1815                                 "name": vdur["name"],
1816                                 "vdu_id": vdur["vdu-id-ref"],
1817                                 "interface": []
1818                             })
1819                             for interface in vdur["interfaces"]:
1820                                 vdu_scaling_info["vdu"][-1]["interface"].append({
1821                                     "name": interface["name"],
1822                                     "ip_address": interface["ip-address"],
1823                                     "mac_address": interface.get("mac-address"),
1824                                 })
1825                     del vdu_scaling_info["vdu-create"]
1826
1827             scale_process = None
1828             if db_nsr_update:
1829                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1830
1831             # execute primitive service POST-SCALING
1832             step = "Executing post-scale vnf-config-primitive"
1833             if scaling_descriptor.get("scaling-config-action"):
1834                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1835                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
1836                             and scaling_type == "SCALE_OUT":
1837                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1838                         step = db_nslcmop_update["detailed-status"] = \
1839                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
1840
1841                         vnfr_params = {"<VDU_SCALE_INFO>": vdu_scaling_info}
1842                         if db_vnfr.get("additionalParamsForVnf"):
1843                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1844
1845                         # look for primitive
1846                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1847                             if config_primitive["name"] == vnf_config_primitive:
1848                                 break
1849                         else:
1850                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
1851                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
1852                                                "match any vnf-configuration:config-primitive".format(scaling_group,
1853                                                                                                      config_primitive))
1854                         scale_process = "VCA"
1855                         db_nsr_update["config-status"] = "configuring post-scaling"
1856
1857                         result, result_detail = await self._ns_execute_primitive(
1858                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1859                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1860                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1861                             vnf_config_primitive, result, result_detail))
1862                         if result == "FAILED":
1863                             raise LcmException(result_detail)
1864                         db_nsr_update["config-status"] = old_config_status
1865                         scale_process = None
1866
1867             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1868             db_nslcmop_update["statusEnteredTime"] = time()
1869             db_nslcmop_update["detailed-status"] = "done"
1870             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
1871             db_nsr_update["operational-status"] = old_operational_status
1872             db_nsr_update["config-status"] = old_config_status
1873             return
1874         except (ROclient.ROClientException, DbException, LcmException) as e:
1875             self.logger.error(logging_text + "Exit Exception {}".format(e))
1876             exc = e
1877         except asyncio.CancelledError:
1878             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1879             exc = "Operation was cancelled"
1880         except Exception as e:
1881             exc = traceback.format_exc()
1882             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1883         finally:
1884             if exc:
1885                 if db_nslcmop:
1886                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1887                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1888                     db_nslcmop_update["statusEnteredTime"] = time()
1889                 if db_nsr:
1890                     db_nsr_update["operational-status"] = old_operational_status
1891                     db_nsr_update["config-status"] = old_config_status
1892                     db_nsr_update["detailed-status"] = ""
1893                     db_nsr_update["_admin.nslcmop"] = None
1894                     if scale_process:
1895                         if "VCA" in scale_process:
1896                             db_nsr_update["config-status"] = "failed"
1897                         if "RO" in scale_process:
1898                             db_nsr_update["operational-status"] = "failed"
1899                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
1900                                                                                                      exc)
1901             try:
1902                 if db_nslcmop and db_nslcmop_update:
1903                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1904                 if db_nsr:
1905                     db_nsr_update["_admin.nslcmop"] = None
1906                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1907             except DbException as e:
1908                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1909             if nslcmop_operation_state:
1910                 try:
1911                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1912                                                              "operationState": nslcmop_operation_state},
1913                                             loop=self.loop)
1914                     # if cooldown_time:
1915                     #     await asyncio.sleep(cooldown_time)
1916                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
1917                 except Exception as e:
1918                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1919             self.logger.debug(logging_text + "Exit")
1920             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")