osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105
 106     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 107         """
 108         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 109         :param vnfd: input vnfd
 110         :param new_id: overrides vnf id if provided
 111         :param additionalParams: Instantiation params for VNFs provided
 112         :param nsrId: Id of the NSR
 113         :return: copy of vnfd
 114         """
 115         try:
 116             vnfd_RO = deepcopy(vnfd)
 117             # remove unused by RO configuration, monitoring, scaling and internal keys
 118             vnfd_RO.pop("_id", None)
 119             vnfd_RO.pop("_admin", None)
 120             vnfd_RO.pop("vnf-configuration", None)
 121             vnfd_RO.pop("monitoring-param", None)
 122             vnfd_RO.pop("scaling-group-descriptor", None)
 123             if new_id:
 124                 vnfd_RO["id"] = new_id
 125
 126             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 127             for vdu in get_iterable(vnfd_RO, "vdu"):
 128                 cloud_init_file = None
 129                 if vdu.get("cloud-init-file"):
 130                     base_folder = vnfd["_admin"]["storage"]
 131                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 132                                                                    vdu["cloud-init-file"])
 133                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 134                         cloud_init_content = ci_file.read()
 135                     vdu.pop("cloud-init-file", None)
 136                 elif vdu.get("cloud-init"):
 137                     cloud_init_content = vdu["cloud-init"]
 138                 else:
 139                     continue
 140
 141                 env = Environment()
 142                 ast = env.parse(cloud_init_content)
 143                 mandatory_vars = meta.find_undeclared_variables(ast)
 144                 if mandatory_vars:
 145                     for var in mandatory_vars:
 146                         if not additionalParams or var not in additionalParams.keys():
 147                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 148                                                "file, must be provided in the instantiation parameters inside the "
 149                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 150                 template = Template(cloud_init_content)
 151                 cloud_init_content = template.render(additionalParams or {})
 152                 vdu["cloud-init"] = cloud_init_content
 153
 154             return vnfd_RO
 155         except FsException as e:
 156             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 157                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 158         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 159             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 160                                format(vnfd["id"], vdu["id"], e))
 161
 162     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 163         """
 164         Callback both for charm status change and task completion
 165         :param model_name: Charm model name
 166         :param application_name: Charm application name
 167         :param status: Can be
 168             - blocked: The unit needs manual intervention
 169             - maintenance: The unit is actively deploying/configuring
 170             - waiting: The unit is waiting for another charm to be ready
 171             - active: The unit is deployed, configured, and ready
 172             - error: The charm has failed and needs attention.
 173             - terminated: The charm has been destroyed
 174             - removing,
 175             - removed
 176         :param message: detailed message error
 177         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 178             nsr_id:
 179             nslcmop_id:
 180             lcmOperationType: currently "instantiate"
 181             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 182             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 183             n2vc_event: event used to notify instantiation task that some change has been produced
 184         :param task: None for charm status change, or task for completion task callback
 185         :return:
 186         """
 187         try:
 188             nsr_id = n2vc_info["nsr_id"]
 189             deployed = n2vc_info["deployed"]
 190             db_nsr_update = n2vc_info["db_update"]
 191             nslcmop_id = n2vc_info["nslcmop_id"]
 192             ns_operation = n2vc_info["lcmOperationType"]
 193             n2vc_event = n2vc_info["n2vc_event"]
 194             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 195                                                                                     application_name)
 196             for vca_index, vca_deployed in enumerate(deployed):
 197                 if not vca_deployed:
 198                     continue
 199                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 200                     break
 201             else:
 202                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 203                                   format(model_name))
 204                 return
 205             if task:
 206                 if task.cancelled():
 207                     self.logger.debug(logging_text + " task Cancelled")
 208                     vca_deployed['operational-status'] = "error"
 209                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 210                     vca_deployed['detailed-status'] = "Task Cancelled"
 211                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 212
 213                 elif task.done():
 214                     exc = task.exception()
 215                     if exc:
 216                         self.logger.error(logging_text + " task Exception={}".format(exc))
 217                         vca_deployed['operational-status'] = "error"
 218                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 219                         vca_deployed['detailed-status'] = str(exc)
 220                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 221                     else:
 222                         self.logger.debug(logging_text + " task Done")
 223                         # task is Done, but callback is still ongoing. So ignore
 224                         return
 225             elif status:
 226                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 227                 if vca_deployed['operational-status'] == status:
 228                     return  # same status, ignore
 229                 vca_deployed['operational-status'] = status
 230                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 231                 vca_deployed['detailed-status'] = str(message)
 232                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 233             else:
 234                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 235                 return
 236             # wake up instantiate task
 237             n2vc_event.set()
 238         except Exception as e:
 239             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 240
 241     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 242         """
 243         Creates a RO ns descriptor from OSM ns_instantiate params
 244         :param ns_params: OSM instantiate params
 245         :return: The RO ns descriptor
 246         """
 247         vim_2_RO = {}
 248         wim_2_RO = {}
 249         # TODO feature 1417: Check that no instantiation is set over PDU
 250         # check if PDU forces a concrete vim-network-id and add it
 251         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 252
 253         def vim_account_2_RO(vim_account):
 254             if vim_account in vim_2_RO:
 255                 return vim_2_RO[vim_account]
 256
 257             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 258             if db_vim["_admin"]["operationalState"] != "ENABLED":
 259                 raise LcmException("VIM={} is not available. operationalState={}".format(
 260                     vim_account, db_vim["_admin"]["operationalState"]))
 261             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 262             vim_2_RO[vim_account] = RO_vim_id
 263             return RO_vim_id
 264
 265         def wim_account_2_RO(wim_account):
 266             if isinstance(wim_account, str):
 267                 if wim_account in wim_2_RO:
 268                     return wim_2_RO[wim_account]
 269
 270                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 271                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 272                     raise LcmException("WIM={} is not available. operationalState={}".format(
 273                         wim_account, db_wim["_admin"]["operationalState"]))
 274                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 275                 wim_2_RO[wim_account] = RO_wim_id
 276                 return RO_wim_id
 277             else:
 278                 return wim_account
 279
 280         def ip_profile_2_RO(ip_profile):
 281             RO_ip_profile = deepcopy((ip_profile))
 282             if "dns-server" in RO_ip_profile:
 283                 if isinstance(RO_ip_profile["dns-server"], list):
 284                     RO_ip_profile["dns-address"] = []
 285                     for ds in RO_ip_profile.pop("dns-server"):
 286                         RO_ip_profile["dns-address"].append(ds['address'])
 287                 else:
 288                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 289             if RO_ip_profile.get("ip-version") == "ipv4":
 290                 RO_ip_profile["ip-version"] = "IPv4"
 291             if RO_ip_profile.get("ip-version") == "ipv6":
 292                 RO_ip_profile["ip-version"] = "IPv6"
 293             if "dhcp-params" in RO_ip_profile:
 294                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 295             return RO_ip_profile
 296
 297         if not ns_params:
 298             return None
 299         RO_ns_params = {
 300             # "name": ns_params["nsName"],
 301             # "description": ns_params.get("nsDescription"),
 302             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 303             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 304             # "scenario": ns_params["nsdId"],
 305         }
 306         if n2vc_key_list:
 307             for vnfd_ref, vnfd in vnfd_dict.items():
 308                 vdu_needed_access = []
 309                 mgmt_cp = None
 310                 if vnfd.get("vnf-configuration"):
 311                     if vnfd.get("mgmt-interface"):
 312                         if vnfd["mgmt-interface"].get("vdu-id"):
 313                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 314                         elif vnfd["mgmt-interface"].get("cp"):
 315                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 316
 317                 for vdu in vnfd.get("vdu", ()):
 318                     if vdu.get("vdu-configuration"):
 319                         vdu_needed_access.append(vdu["id"])
 320                     elif mgmt_cp:
 321                         for vdu_interface in vdu.get("interface"):
 322                             if vdu_interface.get("external-connection-point-ref") and \
 323                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 324                                 vdu_needed_access.append(vdu["id"])
 325                                 mgmt_cp = None
 326                                 break
 327
 328                 if vdu_needed_access:
 329                     for vnf_member in nsd.get("constituent-vnfd"):
 330                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 331                             continue
 332                         for vdu in vdu_needed_access:
 333                             populate_dict(RO_ns_params,
 334                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 335                                           n2vc_key_list)
 336
 337         if ns_params.get("vduImage"):
 338             RO_ns_params["vduImage"] = ns_params["vduImage"]
 339
 340         if ns_params.get("ssh_keys"):
 341             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 342         for vnf_params in get_iterable(ns_params, "vnf"):
 343             for constituent_vnfd in nsd["constituent-vnfd"]:
 344                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 345                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 346                     break
 347             else:
 348                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 349                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 350             if vnf_params.get("vimAccountId"):
 351                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 352                               vim_account_2_RO(vnf_params["vimAccountId"]))
 353
 354             for vdu_params in get_iterable(vnf_params, "vdu"):
 355                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 356                 if vdu_params.get("volume"):
 357                     for volume_params in vdu_params["volume"]:
 358                         if volume_params.get("vim-volume-id"):
 359                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 360                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 361                                           volume_params["vim-volume-id"])
 362                 if vdu_params.get("interface"):
 363                     for interface_params in vdu_params["interface"]:
 364                         if interface_params.get("ip-address"):
 365                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 366                                                          vdu_params["id"], "interfaces", interface_params["name"],
 367                                                          "ip_address"),
 368                                           interface_params["ip-address"])
 369                         if interface_params.get("mac-address"):
 370                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 371                                                          vdu_params["id"], "interfaces", interface_params["name"],
 372                                                          "mac_address"),
 373                                           interface_params["mac-address"])
 374                         if interface_params.get("floating-ip-required"):
 375                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 376                                                          vdu_params["id"], "interfaces", interface_params["name"],
 377                                                          "floating-ip"),
 378                                           interface_params["floating-ip-required"])
 379
 380             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 381                 if internal_vld_params.get("vim-network-name"):
 382                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 383                                                  internal_vld_params["name"], "vim-network-name"),
 384                                   internal_vld_params["vim-network-name"])
 385                 if internal_vld_params.get("vim-network-id"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-id"),
 388                                   internal_vld_params["vim-network-id"])
 389                 if internal_vld_params.get("ip-profile"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "ip-profile"),
 392                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 393
 394                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 395                     # look for interface
 396                     iface_found = False
 397                     for vdu_descriptor in vnf_descriptor["vdu"]:
 398                         for vdu_interface in vdu_descriptor["interface"]:
 399                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 400                                 if icp_params.get("ip-address"):
 401                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 402                                                                  vdu_descriptor["id"], "interfaces",
 403                                                                  vdu_interface["name"], "ip_address"),
 404                                                   icp_params["ip-address"])
 405
 406                                 if icp_params.get("mac-address"):
 407                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 408                                                                  vdu_descriptor["id"], "interfaces",
 409                                                                  vdu_interface["name"], "mac_address"),
 410                                                   icp_params["mac-address"])
 411                                 iface_found = True
 412                                 break
 413                         if iface_found:
 414                             break
 415                     else:
 416                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 417                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 418                                            "connection-point".format(vnf_params["member-vnf-index"],
 419                                                                      icp_params["id-ref"]))
 420
 421         for vld_params in get_iterable(ns_params, "vld"):
 422             if "ip-profile" in vld_params:
 423                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 424                               ip_profile_2_RO(vld_params["ip-profile"]))
 425
 426             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 428                               wim_account_2_RO(vld_params["wimAccountId"])),
 429             if vld_params.get("vim-network-name"):
 430                 RO_vld_sites = []
 431                 if isinstance(vld_params["vim-network-name"], dict):
 432                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 433                         RO_vld_sites.append({
 434                             "netmap-use": vim_net,
 435                             "datacenter": vim_account_2_RO(vim_account)
 436                         })
 437                 else:  # isinstance str
 438                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 439                 if RO_vld_sites:
 440                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 441             if vld_params.get("vim-network-id"):
 442                 RO_vld_sites = []
 443                 if isinstance(vld_params["vim-network-id"], dict):
 444                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 445                         RO_vld_sites.append({
 446                             "netmap-use": vim_net,
 447                             "datacenter": vim_account_2_RO(vim_account)
 448                         })
 449                 else:  # isinstance str
 450                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 451                 if RO_vld_sites:
 452                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 453             if vld_params.get("ns-net"):
 454                 if isinstance(vld_params["ns-net"], dict):
 455                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 456                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 457                 if RO_vld_ns_net:
 458                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 459             if "vnfd-connection-point-ref" in vld_params:
 460                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 461                     # look for interface
 462                     for constituent_vnfd in nsd["constituent-vnfd"]:
 463                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 464                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 465                             break
 466                     else:
 467                         raise LcmException(
 468                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 469                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 470                     match_cp = False
 471                     for vdu_descriptor in vnf_descriptor["vdu"]:
 472                         for interface_descriptor in vdu_descriptor["interface"]:
 473                             if interface_descriptor.get("external-connection-point-ref") == \
 474                                     cp_params["vnfd-connection-point-ref"]:
 475                                 match_cp = True
 476                                 break
 477                         if match_cp:
 478                             break
 479                     else:
 480                         raise LcmException(
 481                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 482                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 483                                 cp_params["member-vnf-index-ref"],
 484                                 cp_params["vnfd-connection-point-ref"],
 485                                 vnf_descriptor["id"]))
 486                     if cp_params.get("ip-address"):
 487                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 488                                                      vdu_descriptor["id"], "interfaces",
 489                                                      interface_descriptor["name"], "ip_address"),
 490                                       cp_params["ip-address"])
 491                     if cp_params.get("mac-address"):
 492                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 493                                                      vdu_descriptor["id"], "interfaces",
 494                                                      interface_descriptor["name"], "mac_address"),
 495                                       cp_params["mac-address"])
 496         return RO_ns_params
 497
 498     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 499         # make a copy to do not change
 500         vdu_create = copy(vdu_create)
 501         vdu_delete = copy(vdu_delete)
 502
 503         vdurs = db_vnfr.get("vdur")
 504         if vdurs is None:
 505             vdurs = []
 506         vdu_index = len(vdurs)
 507         while vdu_index:
 508             vdu_index -= 1
 509             vdur = vdurs[vdu_index]
 510             if vdur.get("pdu-type"):
 511                 continue
 512             vdu_id_ref = vdur["vdu-id-ref"]
 513             if vdu_create and vdu_create.get(vdu_id_ref):
 514                 for index in range(0, vdu_create[vdu_id_ref]):
 515                     vdur = deepcopy(vdur)
 516                     vdur["_id"] = str(uuid4())
 517                     vdur["count-index"] += 1
 518                     vdurs.insert(vdu_index+1+index, vdur)
 519                 del vdu_create[vdu_id_ref]
 520             if vdu_delete and vdu_delete.get(vdu_id_ref):
 521                 del vdurs[vdu_index]
 522                 vdu_delete[vdu_id_ref] -= 1
 523                 if not vdu_delete[vdu_id_ref]:
 524                     del vdu_delete[vdu_id_ref]
 525         # check all operations are done
 526         if vdu_create or vdu_delete:
 527             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 528                 vdu_create))
 529         if vdu_delete:
 530             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 531                 vdu_delete))
 532
 533         vnfr_update = {"vdur": vdurs}
 534         db_vnfr["vdur"] = vdurs
 535         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 536
 537     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 538         """
 539         Updates database nsr with the RO info for the created vld
 540         :param ns_update_nsr: dictionary to be filled with the updated info
 541         :param db_nsr: content of db_nsr. This is also modified
 542         :param nsr_desc_RO: nsr descriptor from RO
 543         :return: Nothing, LcmException is raised on errors
 544         """
 545
 546         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 547             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 548                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 549                     continue
 550                 vld["vim-id"] = net_RO.get("vim_net_id")
 551                 vld["name"] = net_RO.get("vim_name")
 552                 vld["status"] = net_RO.get("status")
 553                 vld["status-detailed"] = net_RO.get("error_msg")
 554                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 555                 break
 556             else:
 557                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 558
 559     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 560         """
 561         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 562         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 563         :param nsr_desc_RO: nsr descriptor from RO
 564         :return: Nothing, LcmException is raised on errors
 565         """
 566         for vnf_index, db_vnfr in db_vnfrs.items():
 567             for vnf_RO in nsr_desc_RO["vnfs"]:
 568                 if vnf_RO["member_vnf_index"] != vnf_index:
 569                     continue
 570                 vnfr_update = {}
 571                 if vnf_RO.get("ip_address"):
 572                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 573                 elif not db_vnfr.get("ip-address"):
 574                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 575
 576                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 577                     vdur_RO_count_index = 0
 578                     if vdur.get("pdu-type"):
 579                         continue
 580                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 581                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 582                             continue
 583                         if vdur["count-index"] != vdur_RO_count_index:
 584                             vdur_RO_count_index += 1
 585                             continue
 586                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 587                         if vdur_RO.get("ip_address"):
 588                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 589                         else:
 590                             vdur["ip-address"] = None
 591                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 592                         vdur["name"] = vdur_RO.get("vim_name")
 593                         vdur["status"] = vdur_RO.get("status")
 594                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 595                         for ifacer in get_iterable(vdur, "interfaces"):
 596                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 597                                 if ifacer["name"] == interface_RO.get("internal_name"):
 598                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 599                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 600                                     break
 601                             else:
 602                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 603                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 604                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 605                         break
 606                     else:
 607                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 608                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 609
 610                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 611                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 612                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 613                             continue
 614                         vld["vim-id"] = net_RO.get("vim_net_id")
 615                         vld["name"] = net_RO.get("vim_name")
 616                         vld["status"] = net_RO.get("status")
 617                         vld["status-detailed"] = net_RO.get("error_msg")
 618                         vnfr_update["vld.{}".format(vld_index)] = vld
 619                         break
 620                     else:
 621                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 622                             vnf_index, vld["id"]))
 623
 624                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 625                 break
 626
 627             else:
 628                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 629
 630     async def instantiate(self, nsr_id, nslcmop_id):
 631         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 632         self.logger.debug(logging_text + "Enter")
 633         # get all needed from database
 634         start_deploy = time()
 635         db_nsr = None
 636         db_nslcmop = None
 637         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 638         db_nslcmop_update = {}
 639         nslcmop_operation_state = None
 640         db_vnfrs = {}
 641         RO_descriptor_number = 0   # number of descriptors created at RO
 642         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 643         n2vc_info = {}
 644         exc = None
 645         try:
 646             step = "Getting nslcmop={} from db".format(nslcmop_id)
 647             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 648             step = "Getting nsr={} from db".format(nsr_id)
 649             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 650             ns_params = db_nslcmop.get("operationParams")
 651             nsd = db_nsr["nsd"]
 652             nsr_name = db_nsr["name"]   # TODO short-name??
 653
 654             # look if previous tasks in process
 655             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
 656             if task_dependency:
 657                 step = db_nslcmop_update["detailed-status"] = \
 658                     "Waiting for related tasks to be completed: {}".format(task_name)
 659                 self.logger.debug(logging_text + step)
 660                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 661                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 662                 if pending:
 663                     raise LcmException("Timeout waiting related tasks to be completed")
 664
 665             step = "Getting vnfrs from db"
 666             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 667             db_vnfds_ref = {}
 668             db_vnfds = {}
 669             for vnfr in db_vnfrs_list:
 670                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 671                 vnfd_id = vnfr["vnfd-id"]
 672                 vnfd_ref = vnfr["vnfd-ref"]
 673                 if vnfd_id not in db_vnfds:
 674                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 675                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 676                     db_vnfds_ref[vnfd_ref] = vnfd
 677                     db_vnfds[vnfd_id] = vnfd
 678
 679             # Get or generates the _admin.deployed,VCA list
 680             vca_deployed_list = None
 681             vca_model_name = None
 682             if db_nsr["_admin"].get("deployed"):
 683                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 684                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 685             if vca_deployed_list is None:
 686                 vca_deployed_list = []
 687                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 688             elif isinstance(vca_deployed_list, dict):
 689                 # maintain backward compatibility. Change a dict to list at database
 690                 vca_deployed_list = list(vca_deployed_list.values())
 691                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 692
 693             db_nsr_update["detailed-status"] = "creating"
 694             db_nsr_update["operational-status"] = "init"
 695             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 696                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 697                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 698                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 699
 700             RO = ROclient.ROClient(self.loop, **self.ro_config)
 701
 702             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 703             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 704             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 705
 706             # get vnfds, instantiate at RO
 707             for c_vnf in nsd.get("constituent-vnfd", ()):
 708                 member_vnf_index = c_vnf["member-vnf-index"]
 709                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 710                 vnfd_ref = vnfd["id"]
 711                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member-vnf-index='{}' at RO".format(
 712                     vnfd_ref, member_vnf_index)
 713                 # self.logger.debug(logging_text + step)
 714                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 715                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 716                 RO_descriptor_number += 1
 717
 718                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 719                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 720                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 721                         break
 722                 else:
 723                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
 724                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
 725
 726                 # look if present
 727                 RO_update = {"member-vnf-index": member_vnf_index}
 728                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
 729                 if vnfd_list:
 730                     RO_update["id"] = vnfd_list[0]["uuid"]
 731                     self.logger.debug(logging_text + "vnfd='{}'  member-vnf-index='{}' exists at RO. Using RO_id={}".
 732                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
 733                 else:
 734                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
 735                                            get("additionalParamsForVnf"), nsr_id)
 736                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
 737                     RO_update["id"] = desc["uuid"]
 738                     self.logger.debug(logging_text + "vnfd='{}' member-vnf-index='{}' created at RO. RO_id={}".format(
 739                         vnfd_ref, member_vnf_index, desc["uuid"]))
 740                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
 741                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
 742                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 743
 744             # create nsd at RO
 745             nsd_ref = nsd["id"]
 746             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
 747             # self.logger.debug(logging_text + step)
 748
 749             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
 750             RO_descriptor_number += 1
 751             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
 752             if nsd_list:
 753                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
 754                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
 755                     nsd_ref, RO_nsd_uuid))
 756             else:
 757                 nsd_RO = deepcopy(nsd)
 758                 nsd_RO["id"] = RO_osm_nsd_id
 759                 nsd_RO.pop("_id", None)
 760                 nsd_RO.pop("_admin", None)
 761                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
 762                     member_vnf_index = c_vnf["member-vnf-index"]
 763                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 764                 for c_vld in nsd_RO.get("vld", ()):
 765                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
 766                         member_vnf_index = cp["member-vnf-index-ref"]
 767                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 768
 769                 desc = await RO.create("nsd", descriptor=nsd_RO)
 770                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 771                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
 772                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
 773             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 774
 775             # Crate ns at RO
 776             # if present use it unless in error status
 777             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
 778             if RO_nsr_id:
 779                 try:
 780                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
 781                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
 782                     desc = await RO.show("ns", RO_nsr_id)
 783                 except ROclient.ROClientException as e:
 784                     if e.http_code != HTTPStatus.NOT_FOUND:
 785                         raise
 786                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 787                 if RO_nsr_id:
 788                     ns_status, ns_status_info = RO.check_ns_status(desc)
 789                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 790                     if ns_status == "ERROR":
 791                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
 792                         self.logger.debug(logging_text + step)
 793                         await RO.delete("ns", RO_nsr_id)
 794                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 795             if not RO_nsr_id:
 796                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
 797                 # self.logger.debug(logging_text + step)
 798
 799                 # check if VIM is creating and wait  look if previous tasks in process
 800                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
 801                 if task_dependency:
 802                     step = "Waiting for related tasks to be completed: {}".format(task_name)
 803                     self.logger.debug(logging_text + step)
 804                     await asyncio.wait(task_dependency, timeout=3600)
 805                 if ns_params.get("vnf"):
 806                     for vnf in ns_params["vnf"]:
 807                         if "vimAccountId" in vnf:
 808                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 809                                                                                         vnf["vimAccountId"])
 810                         if task_dependency:
 811                             step = "Waiting for related tasks to be completed: {}".format(task_name)
 812                             self.logger.debug(logging_text + step)
 813                             await asyncio.wait(task_dependency, timeout=3600)
 814
 815                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
 816
 817                 # feature 1429. Add n2vc public key to needed VMs
 818                 n2vc_key = await self.n2vc.GetPublicKey()
 819                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, [n2vc_key])
 820
 821                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
 822                 desc = await RO.create("ns", descriptor=RO_ns_params,
 823                                        name=db_nsr["name"],
 824                                        scenario=RO_nsd_uuid)
 825                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
 826                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 827                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
 828                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 829             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 830
 831             # wait until NS is ready
 832             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
 833             detailed_status_old = None
 834             self.logger.debug(logging_text + step)
 835
 836             while time() <= start_deploy + self.total_deploy_timeout:
 837                 desc = await RO.show("ns", RO_nsr_id)
 838                 ns_status, ns_status_info = RO.check_ns_status(desc)
 839                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 840                 if ns_status == "ERROR":
 841                     raise ROclient.ROClientException(ns_status_info)
 842                 elif ns_status == "BUILD":
 843                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 844                 elif ns_status == "ACTIVE":
 845                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
 846                     try:
 847                         self.ns_update_vnfr(db_vnfrs, desc)
 848                         break
 849                     except LcmExceptionNoMgmtIP:
 850                         pass
 851                 else:
 852                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 853                 if detailed_status != detailed_status_old:
 854                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
 855                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 856                 await asyncio.sleep(5, loop=self.loop)
 857             else:   # total_deploy_timeout
 858                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 859
 860             step = "Updating NSR"
 861             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 862
 863             db_nsr["detailed-status"] = "Configuring vnfr"
 864             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 865
 866             # The parameters we'll need to deploy a charm
 867             number_to_configure = 0
 868
 869             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm):
 870                 """An inner function to deploy the charm from either ns, vnf or vdu
 871                 For ns both vnf_index and vdu_id are None.
 872                 For vnf only vdu_id is None
 873                 For vdu both vnf_index and vdu_id contain a value
 874                 """
 875                 if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 876                     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 877
 878                 machine_spec = {}
 879                 if native_charm:
 880                     machine_spec["username"] = charm_params.get("username"),
 881                     machine_spec["username"] = charm_params.get("rw_mgmt_ip")
 882
 883                 # Login to the VCA.
 884                 # if number_to_configure == 0:
 885                 #     self.logger.debug("Logging into N2VC...")
 886                 #     task = asyncio.ensure_future(self.n2vc.login())
 887                 #     yield from asyncio.wait_for(task, 30.0)
 888                 #     self.logger.debug("Logged into N2VC!")
 889
 890                 # # await self.n2vc.login()
 891
 892                 # Note: The charm needs to exist on disk at the location
 893                 # specified by charm_path.
 894                 descriptor = vnfd if vnf_index else nsd
 895                 base_folder = descriptor["_admin"]["storage"]
 896                 storage_params = self.fs.get_params()
 897                 charm_path = "{}{}/{}/charms/{}".format(
 898                     storage_params["path"],
 899                     base_folder["folder"],
 900                     base_folder["pkg-dir"],
 901                     proxy_charm
 902                 )
 903
 904                 # ns_name will be ignored in the current version of N2VC
 905                 # but will be implemented for the next point release.
 906                 model_name = nsr_id
 907                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 908                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 909                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 910
 911                 vca_index = len(vca_deployed_list)
 912                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 913                 # 26*26 charm in the same NS
 914                 application_name = application_name[0:48]
 915                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 916                 vca_deployed_ = {
 917                     "member-vnf-index": vnf_index,
 918                     "vdu_id": vdu_id,
 919                     "model": model_name,
 920                     "application": application_name,
 921                     "operational-status": "init",
 922                     "detailed-status": "",
 923                     "vnfd_id": vnfd_id,
 924                     "vdu_name": vdu_name,
 925                     "vdu_count_index": vdu_count_index,
 926                 }
 927                 vca_deployed_list.append(vca_deployed_)
 928                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 929                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 930
 931                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 932                                                                                                 proxy_charm))
 933                 if not n2vc_info:
 934                     n2vc_info["nsr_id"] = nsr_id
 935                     n2vc_info["nslcmop_id"] = nslcmop_id
 936                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 937                     n2vc_info["lcmOperationType"] = "instantiate"
 938                     n2vc_info["deployed"] = vca_deployed_list
 939                     n2vc_info["db_update"] = db_nsr_update
 940                 task = asyncio.ensure_future(
 941                     self.n2vc.DeployCharms(
 942                         model_name,          # The network service name
 943                         application_name,    # The application name
 944                         descriptor,          # The vnf/nsd descriptor
 945                         charm_path,          # Path to charm
 946                         charm_params,        # Runtime params, like mgmt ip
 947                         machine_spec,        # for native charms only
 948                         self.n2vc_callback,  # Callback for status changes
 949                         n2vc_info,           # Callback parameter
 950                         None,                # Callback parameter (task)
 951                     )
 952                 )
 953                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 954                                                          n2vc_info))
 955                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 956
 957             step = "Looking for needed vnfd to configure"
 958             self.logger.debug(logging_text + step)
 959
 960             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 961                 vnfd_id = c_vnf["vnfd-id-ref"]
 962                 vnf_index = str(c_vnf["member-vnf-index"])
 963                 vnfd = db_vnfds_ref[vnfd_id]
 964
 965                 # Get additional parameters
 966                 vnfr_params = {}
 967                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 968                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 969                 for k, v in vnfr_params.items():
 970                     if isinstance(v, str) and v.startswith("!!yaml "):
 971                         vnfr_params[k] = yaml.safe_load(v[7:])
 972
 973                 # Check if this VNF has a charm configuration
 974                 vnf_config = vnfd.get("vnf-configuration")
 975                 if vnf_config and vnf_config.get("juju"):
 976                     proxy_charm = vnf_config["juju"]["charm"]
 977                     native_charm = vnf_config["juju"].get("proxy") is False
 978
 979                     if proxy_charm or native_charm:
 980                         if not vca_model_name:
 981                             step = "creating VCA model name '{}'".format(nsr_id)
 982                             self.logger.debug(logging_text + step)
 983                             await self.n2vc.CreateNetworkService(nsr_id)
 984                             vca_model_name = nsr_id
 985                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 986                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 987                         step = "connecting to N2VC to configure vnf {}".format(vnf_index)
 988                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 989                         charm_params = {
 990                             "user_values": vnfr_params,
 991                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 992                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
 993                         }
 994
 995                         # get username
 996                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
 997                         #  merged. Meanwhile let's get username from initial-config-primitive
 998                         if vnf_config.get("initial-config-primitive"):
 999                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1000                                 if param["name"] == "ssh-username":
1001                                     charm_params["username"] = param["value"]
1002                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1003                             if vnf_config["config-access"]["ssh-access"].get("required"):
1004                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1005
1006                         # Login to the VCA. If there are multiple calls to login(),
1007                         # subsequent calls will be a nop and return immediately.
1008                         await self.n2vc.login()
1009
1010                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1011                         number_to_configure += 1
1012
1013                 # Deploy charms for each VDU that supports one.
1014                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1015                     vdu_config = vdu.get('vdu-configuration')
1016                     proxy_charm = None
1017                     native_charm = None
1018
1019                     if vdu_config and vdu_config.get("juju"):
1020                         proxy_charm = vdu_config["juju"]["charm"]
1021                         native_charm = vdu_config["juju"].get("proxy") is False
1022
1023                         if proxy_charm or native_charm:
1024                             if not vca_model_name:
1025                                 step = "creating VCA model name"
1026                                 await self.n2vc.CreateNetworkService(nsr_id)
1027                                 vca_model_name = nsr_id
1028                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1029                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1030                             step = "connecting to N2VC to configure vdu {} from vnf {}".format(vdu["id"], vnf_index)
1031                             await self.n2vc.login()
1032                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1033                             # TODO for the moment only first vdu_id contains a charm deployed
1034                             if vdur["vdu-id-ref"] != vdu["id"]:
1035                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1036                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1037                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1038                             charm_params = {
1039                                 "user_values": vnfr_params,
1040                                 "rw_mgmt_ip": vdur["ip-address"],
1041                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1042                             }
1043
1044                             # get username
1045                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1046                             #  merged. Meanwhile let's get username from initial-config-primitive
1047                             if vdu_config.get("initial-config-primitive"):
1048                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1049                                     if param["name"] == "ssh-username":
1050                                         charm_params["username"] = param["value"]
1051                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1052                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1053                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1054                                         "default-user")
1055
1056                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1057                                          charm_params, n2vc_info, native_charm)
1058                             number_to_configure += 1
1059
1060             # Check if this NS has a charm configuration
1061
1062             ns_config = nsd.get("ns-configuration")
1063             if ns_config and ns_config.get("juju"):
1064                 proxy_charm = ns_config["juju"]["charm"]
1065                 native_charm = ns_config["juju"].get("proxy") is False
1066
1067                 if proxy_charm or native_charm:
1068                     step = "connecting to N2VC to configure ns"
1069                     # TODO is NS magmt IP address needed?
1070
1071                     # Get additional parameters
1072                     additional_params = {}
1073                     if db_nsr.get("additionalParamsForNs"):
1074                         additional_params = db_nsr["additionalParamsForNs"].copy()
1075                     for k, v in additional_params.items():
1076                         if isinstance(v, str) and v.startswith("!!yaml "):
1077                             additional_params[k] = yaml.safe_load(v[7:])
1078
1079                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1080                     charm_params = {
1081                         "user_values": additional_params,
1082                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1083                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1084                     }
1085
1086                     # get username
1087                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1088                     #  merged. Meanwhile let's get username from initial-config-primitive
1089                     if ns_config.get("initial-config-primitive"):
1090                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1091                             if param["name"] == "ssh-username":
1092                                 charm_params["username"] = param["value"]
1093                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1094                         if ns_config["config-access"]["ssh-access"].get("required"):
1095                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1096
1097                     # Login to the VCA. If there are multiple calls to login(),
1098                     # subsequent calls will be a nop and return immediately.
1099                     await self.n2vc.login()
1100                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1101                     number_to_configure += 1
1102
1103             db_nsr_update["operational-status"] = "running"
1104             configuration_failed = False
1105             if number_to_configure:
1106                 old_status = "configuring: init: {}".format(number_to_configure)
1107                 db_nsr_update["config-status"] = old_status
1108                 db_nsr_update["detailed-status"] = old_status
1109                 db_nslcmop_update["detailed-status"] = old_status
1110
1111                 # wait until all are configured.
1112                 while time() <= start_deploy + self.total_deploy_timeout:
1113                     if db_nsr_update:
1114                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1115                     if db_nslcmop_update:
1116                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1117                     # TODO add a fake task that set n2vc_event after some time
1118                     await n2vc_info["n2vc_event"].wait()
1119                     n2vc_info["n2vc_event"].clear()
1120                     all_active = True
1121                     status_map = {}
1122                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1123                     now = time()
1124                     for vca_deployed in vca_deployed_list:
1125                         vca_status = vca_deployed["operational-status"]
1126                         if vca_status not in status_map:
1127                             # Initialize it
1128                             status_map[vca_status] = 0
1129                         status_map[vca_status] += 1
1130
1131                         if vca_status == "active":
1132                             vca_deployed.pop("time_first_error", None)
1133                             vca_deployed.pop("status_first_error", None)
1134                             continue
1135
1136                         all_active = False
1137                         if vca_status in ("error", "blocked"):
1138                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1139                             # if not first time in this status error
1140                             if not vca_deployed.get("time_first_error"):
1141                                 vca_deployed["time_first_error"] = now
1142                                 continue
1143                         if vca_deployed.get("time_first_error") and \
1144                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1145                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1146                                                    .format(vca_deployed["member-vnf-index"],
1147                                                            vca_deployed["vdu_id"], vca_status,
1148                                                            vca_deployed["detailed-status-error"]))
1149
1150                     if all_active:
1151                         break
1152                     elif n2vc_error_text:
1153                         db_nsr_update["config-status"] = "failed"
1154                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1155                         db_nsr_update["detailed-status"] = error_text
1156                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1157                         db_nslcmop_update["detailed-status"] = error_text
1158                         db_nslcmop_update["statusEnteredTime"] = time()
1159                         configuration_failed = True
1160                         break
1161                     else:
1162                         cs = "configuring: "
1163                         separator = ""
1164                         for status, num in status_map.items():
1165                             cs += separator + "{}: {}".format(status, num)
1166                             separator = ", "
1167                         if old_status != cs:
1168                             db_nsr_update["config-status"] = cs
1169                             db_nsr_update["detailed-status"] = cs
1170                             db_nslcmop_update["detailed-status"] = cs
1171                             old_status = cs
1172                 else:   # total_deploy_timeout
1173                     raise LcmException("Timeout waiting ns to be configured")
1174
1175             if not configuration_failed:
1176                 # all is done
1177                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1178                 db_nslcmop_update["statusEnteredTime"] = time()
1179                 db_nslcmop_update["detailed-status"] = "done"
1180                 db_nsr_update["config-status"] = "configured"
1181                 db_nsr_update["detailed-status"] = "done"
1182
1183             return
1184
1185         except (ROclient.ROClientException, DbException, LcmException) as e:
1186             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1187             exc = e
1188         except asyncio.CancelledError:
1189             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1190             exc = "Operation was cancelled"
1191         except Exception as e:
1192             exc = traceback.format_exc()
1193             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1194                                  exc_info=True)
1195         finally:
1196             if exc:
1197                 if db_nsr:
1198                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1199                     db_nsr_update["operational-status"] = "failed"
1200                 if db_nslcmop:
1201                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1202                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1203                     db_nslcmop_update["statusEnteredTime"] = time()
1204             try:
1205                 if db_nsr:
1206                     db_nsr_update["_admin.nslcmop"] = None
1207                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1208                 if db_nslcmop_update:
1209                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1210             except DbException as e:
1211                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1212             if nslcmop_operation_state:
1213                 try:
1214                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1215                                                                    "operationState": nslcmop_operation_state},
1216                                             loop=self.loop)
1217                 except Exception as e:
1218                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1219
1220             self.logger.debug(logging_text + "Exit")
1221             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1222
1223     async def _destroy_charm(self, model, application):
1224         """
1225         Order N2VC destroy a charm
1226         :param model:
1227         :param application:
1228         :return: True if charm does not exist. False if it exist
1229         """
1230         if not await self.n2vc.HasApplication(model, application):
1231             return True  # Already removed
1232         await self.n2vc.RemoveCharms(model, application)
1233         return False
1234
1235     async def _wait_charm_destroyed(self, model, application, timeout):
1236         """
1237         Wait until charm does not exist
1238         :param model:
1239         :param application:
1240         :param timeout:
1241         :return: True if not exist, False if timeout
1242         """
1243         while True:
1244             if not await self.n2vc.HasApplication(model, application):
1245                 return True
1246             if timeout < 0:
1247                 return False
1248             await asyncio.sleep(10)
1249             timeout -= 10
1250
1251     # Check if this VNFD has a configured terminate action
1252     def _has_terminate_config_primitive(self, vnfd):
1253         vnf_config = vnfd.get("vnf-configuration")
1254         if vnf_config and vnf_config.get("terminate-config-primitive"):
1255             return True
1256         else:
1257             return False
1258
1259     # Get a numerically sorted list of the sequences for this VNFD's terminate action
1260     def _get_terminate_config_primitive_seq_list(self, vnfd):
1261         # No need to check for existing primitive twice, already done before
1262         vnf_config = vnfd.get("vnf-configuration")
1263         seq_list = vnf_config.get("terminate-config-primitive")
1264         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1265         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1266         return seq_list_sorted
1267
1268     @staticmethod
1269     def _create_nslcmop(nsr_id, operation, params):
1270         """
1271         Creates a ns-lcm-opp content to be stored at database.
1272         :param nsr_id: internal id of the instance
1273         :param operation: instantiate, terminate, scale, action, ...
1274         :param params: user parameters for the operation
1275         :return: dictionary following SOL005 format
1276         """
1277         # Raise exception if invalid arguments
1278         if not (nsr_id and operation and params):
1279             raise LcmException(
1280                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1281         now = time()
1282         _id = str(uuid4())
1283         nslcmop = {
1284             "id": _id,
1285             "_id": _id,
1286             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1287             "operationState": "PROCESSING",
1288             "statusEnteredTime": now,
1289             "nsInstanceId": nsr_id,
1290             "lcmOperationType": operation,
1291             "startTime": now,
1292             "isAutomaticInvocation": False,
1293             "operationParams": params,
1294             "isCancelPending": False,
1295             "links": {
1296                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1297                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1298             }
1299         }
1300         return nslcmop
1301
1302     # Create a primitive with params from VNFD
1303     # - Called from terminate() before deleting instance
1304     # - Calls action() to execute the primitive
1305     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1306         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1307         db_vnfds = {}
1308         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1309         # Loop over VNFRs
1310         for vnfr in db_vnfrs_list:
1311             vnfd_id = vnfr["vnfd-id"]
1312             vnf_index = vnfr["member-vnf-index-ref"]
1313             if vnfd_id not in db_vnfds:
1314                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1315                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1316                 db_vnfds[vnfd_id] = vnfd
1317             vnfd = db_vnfds[vnfd_id]
1318             if not self._has_terminate_config_primitive(vnfd):
1319                 continue
1320             # Get the primitive's sorted sequence list
1321             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1322             for seq in seq_list:
1323                 # For each sequence in list, call terminate action
1324                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1325                     vnf_index, seq.get("name"))
1326                 self.logger.debug(logging_text + step)
1327                 # Create the primitive for each sequence
1328                 operation = "action"
1329                 # primitive, i.e. "primitive": "touch"
1330                 primitive = seq.get('name')
1331                 primitive_params = {}
1332                 params = {
1333                     "member_vnf_index": vnf_index,
1334                     "primitive": primitive,
1335                     "primitive_params": primitive_params,
1336                 }
1337                 nslcmop_primitive = self._create_nslcmop(nsr_id, operation, params)
1338                 # Get a copy of db_nslcmop 'admin' part
1339                 db_nslcmop_action = {"_admin": deepcopy(db_nslcmop["_admin"])}
1340                 # Update db_nslcmop with the primitive data
1341                 db_nslcmop_action.update(nslcmop_primitive)
1342                 # Create a new db entry for the created primitive, returns the new ID.
1343                 # (The ID is normally obtained from Kafka.)
1344                 nslcmop_terminate_action_id = self.db.create(
1345                     "nslcmops", db_nslcmop_action)
1346                 # Execute the primitive
1347                 nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1348                     nsr_id, nslcmop_terminate_action_id)
1349                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1350                 nslcmop_operation_states_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1351                 if (nslcmop_operation_state not in nslcmop_operation_states_ok):
1352                     raise LcmException(
1353                         "terminate_primitive_action for vnf_member_index={}",
1354                         " primitive={} fails with error {}".format(
1355                             vnf_index, seq.get("name"), nslcmop_operation_state_detail))
1356
1357     async def terminate(self, nsr_id, nslcmop_id):
1358         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1359         self.logger.debug(logging_text + "Enter")
1360         db_nsr = None
1361         db_nslcmop = None
1362         exc = None
1363         failed_detail = []   # annotates all failed error messages
1364         vca_time_destroy = None   # time of where destroy charm order
1365         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1366         db_nslcmop_update = {}
1367         nslcmop_operation_state = None
1368         autoremove = False  # autoremove after terminated
1369         try:
1370             step = "Getting nslcmop={} from db".format(nslcmop_id)
1371             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1372             step = "Getting nsr={} from db".format(nsr_id)
1373             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1374             # nsd = db_nsr["nsd"]
1375             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1376             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1377                 return
1378             # #TODO check if VIM is creating and wait
1379             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1380             # Call internal terminate action
1381             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1382
1383             db_nsr_update["operational-status"] = "terminating"
1384             db_nsr_update["config-status"] = "terminating"
1385
1386             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1387                 vca_model_name = nsr_deployed["VCA-model-name"]
1388                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1389                 self.logger.debug(logging_text + step)
1390                 try:
1391                     await self.n2vc.DestroyNetworkService(vca_model_name)
1392                 except NetworkServiceDoesNotExist:
1393                     pass
1394                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1395                 if nsr_deployed.get("VCA"):
1396                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1397                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1398                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1399             # for backward compatibility if charm have been created with "default"  model name delete one by one
1400             elif nsr_deployed and nsr_deployed.get("VCA"):
1401                 try:
1402                     step = "Scheduling configuration charms removing"
1403                     db_nsr_update["detailed-status"] = "Deleting charms"
1404                     self.logger.debug(logging_text + step)
1405                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1406                     # for backward compatibility
1407                     if isinstance(nsr_deployed["VCA"], dict):
1408                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1409                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1410                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1411
1412                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1413                         if vca_deployed:
1414                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1415                                 vca_deployed.clear()
1416                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1417                             else:
1418                                 vca_time_destroy = time()
1419                 except Exception as e:
1420                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1421
1422             # remove from RO
1423             RO_fail = False
1424             RO = ROclient.ROClient(self.loop, **self.ro_config)
1425
1426             # Delete ns
1427             RO_nsr_id = RO_delete_action = None
1428             if nsr_deployed and nsr_deployed.get("RO"):
1429                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1430                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1431             try:
1432                 if RO_nsr_id:
1433                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1434                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1435                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1436                     self.logger.debug(logging_text + step)
1437                     desc = await RO.delete("ns", RO_nsr_id)
1438                     RO_delete_action = desc["action_id"]
1439                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1440                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1441                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1442                 if RO_delete_action:
1443                     # wait until NS is deleted from VIM
1444                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1445                         format(RO_nsr_id, RO_delete_action)
1446                     detailed_status_old = None
1447                     self.logger.debug(logging_text + step)
1448
1449                     delete_timeout = 20 * 60   # 20 minutes
1450                     while delete_timeout > 0:
1451                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1452                                              extra_item_id=RO_delete_action)
1453                         ns_status, ns_status_info = RO.check_action_status(desc)
1454                         if ns_status == "ERROR":
1455                             raise ROclient.ROClientException(ns_status_info)
1456                         elif ns_status == "BUILD":
1457                             detailed_status = step + "; {}".format(ns_status_info)
1458                         elif ns_status == "ACTIVE":
1459                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1460                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1461                             break
1462                         else:
1463                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1464                         if detailed_status != detailed_status_old:
1465                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1466                                 db_nsr_update["detailed-status"] = detailed_status
1467                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1468                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1469                         await asyncio.sleep(5, loop=self.loop)
1470                         delete_timeout -= 5
1471                     else:  # delete_timeout <= 0:
1472                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1473
1474             except ROclient.ROClientException as e:
1475                 if e.http_code == 404:  # not found
1476                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1477                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1478                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1479                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1480                 elif e.http_code == 409:   # conflict
1481                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1482                     self.logger.debug(logging_text + failed_detail[-1])
1483                     RO_fail = True
1484                 else:
1485                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1486                     self.logger.error(logging_text + failed_detail[-1])
1487                     RO_fail = True
1488
1489             # Delete nsd
1490             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1491                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1492                 try:
1493                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1494                         "Deleting nsd at RO"
1495                     await RO.delete("nsd", RO_nsd_id)
1496                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1497                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1498                 except ROclient.ROClientException as e:
1499                     if e.http_code == 404:  # not found
1500                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1501                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1502                     elif e.http_code == 409:   # conflict
1503                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1504                         self.logger.debug(logging_text + failed_detail[-1])
1505                         RO_fail = True
1506                     else:
1507                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1508                         self.logger.error(logging_text + failed_detail[-1])
1509                         RO_fail = True
1510
1511             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1512                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1513                     if not vnf_deployed or not vnf_deployed["id"]:
1514                         continue
1515                     try:
1516                         RO_vnfd_id = vnf_deployed["id"]
1517                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1518                             "Deleting member-vnf-index={} RO_vnfd_id={} from RO".format(
1519                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1520                         await RO.delete("vnfd", RO_vnfd_id)
1521                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1522                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1523                     except ROclient.ROClientException as e:
1524                         if e.http_code == 404:  # not found
1525                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1526                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1527                         elif e.http_code == 409:   # conflict
1528                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1529                             self.logger.debug(logging_text + failed_detail[-1])
1530                         else:
1531                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1532                             self.logger.error(logging_text + failed_detail[-1])
1533
1534             # wait until charm deleted
1535             if vca_time_destroy:
1536                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1537                     "Waiting for deletion of configuration charms"
1538                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1539                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1540                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1541                     if not vca_deployed:
1542                         continue
1543                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1544                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1545                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1546                                                             timeout):
1547                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1548                             vca_deployed["application"]))
1549                     else:
1550                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1551
1552             if failed_detail:
1553                 self.logger.error(logging_text + " ;".join(failed_detail))
1554                 db_nsr_update["operational-status"] = "failed"
1555                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1556                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1557                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1558                 db_nslcmop_update["statusEnteredTime"] = time()
1559             else:
1560                 db_nsr_update["operational-status"] = "terminated"
1561                 db_nsr_update["detailed-status"] = "Done"
1562                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1563                 db_nslcmop_update["detailed-status"] = "Done"
1564                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1565                 db_nslcmop_update["statusEnteredTime"] = time()
1566                 if db_nslcmop["operationParams"].get("autoremove"):
1567                     autoremove = True
1568
1569         except (ROclient.ROClientException, DbException, LcmException) as e:
1570             self.logger.error(logging_text + "Exit Exception {}".format(e))
1571             exc = e
1572         except asyncio.CancelledError:
1573             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1574             exc = "Operation was cancelled"
1575         except Exception as e:
1576             exc = traceback.format_exc()
1577             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1578         finally:
1579             if exc and db_nslcmop:
1580                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1581                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1582                 db_nslcmop_update["statusEnteredTime"] = time()
1583             try:
1584                 if db_nslcmop and db_nslcmop_update:
1585                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1586                 if db_nsr:
1587                     db_nsr_update["_admin.nslcmop"] = None
1588                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1589             except DbException as e:
1590                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1591             if nslcmop_operation_state:
1592                 try:
1593                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1594                                                                  "operationState": nslcmop_operation_state,
1595                                                                  "autoremove": autoremove},
1596                                             loop=self.loop)
1597                 except Exception as e:
1598                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1599             self.logger.debug(logging_text + "Exit")
1600             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1601
1602     @staticmethod
1603     def _map_primitive_params(primitive_desc, params, instantiation_params):
1604         """
1605         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1606         The default-value is used. If it is between < > it look for a value at instantiation_params
1607         :param primitive_desc: portion of VNFD/NSD that describes primitive
1608         :param params: Params provided by user
1609         :param instantiation_params: Instantiation params provided by user
1610         :return: a dictionary with the calculated params
1611         """
1612         calculated_params = {}
1613         for parameter in primitive_desc.get("parameter", ()):
1614             param_name = parameter["name"]
1615             if param_name in params:
1616                 calculated_params[param_name] = params[param_name]
1617             elif "default-value" in parameter:
1618                 calculated_params[param_name] = parameter["default-value"]
1619                 if isinstance(parameter["default-value"], str) and parameter["default-value"].startswith("<") and \
1620                         parameter["default-value"].endswith(">"):
1621                     if parameter["default-value"][1:-1] in instantiation_params:
1622                         calculated_params[param_name] = instantiation_params[parameter["default-value"][1:-1]]
1623                     else:
1624                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1625                                            format(parameter["default-value"], primitive_desc["name"]))
1626             else:
1627                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1628                                    format(param_name, primitive_desc["name"]))
1629
1630             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1631                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1632                                                                width=256)
1633             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1634                 calculated_params[param_name] = calculated_params[param_name][7:]
1635         return calculated_params
1636
1637     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1638                                     primitive, primitive_params):
1639         start_primitive_time = time()
1640         try:
1641             for vca_deployed in db_deployed["VCA"]:
1642                 if not vca_deployed:
1643                     continue
1644                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1645                     continue
1646                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1647                     continue
1648                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1649                     continue
1650                 break
1651             else:
1652                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1653                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1654             model_name = vca_deployed.get("model")
1655             application_name = vca_deployed.get("application")
1656             if not model_name or not application_name:
1657                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1658                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1659                                                                        vdu_count_index))
1660             if vca_deployed["operational-status"] != "active":
1661                 raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1662                     member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1663             callback = None  # self.n2vc_callback
1664             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1665             await self.n2vc.login()
1666             primitive_id = await self.n2vc.ExecutePrimitive(
1667                 model_name,
1668                 application_name,
1669                 primitive,
1670                 callback,
1671                 *callback_args,
1672                 **primitive_params
1673             )
1674             while time() - start_primitive_time < self.timeout_primitive:
1675                 primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1676                 if primitive_result_ in ("running", "pending"):
1677                     pass
1678                 elif primitive_result_ in ("completed", "failed"):
1679                     primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1680                     detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1681                     break
1682                 else:
1683                     detailed_result = "Invalid N2VC.GetPrimitiveStatus = {} obtained".format(primitive_result_)
1684                     primitive_result = "FAILED"
1685                     break
1686                 await asyncio.sleep(5)
1687             else:
1688                 raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1689             return primitive_result, detailed_result
1690         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
1691             return "FAILED", str(e)
1692
1693     async def action(self, nsr_id, nslcmop_id):
1694         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
1695         self.logger.debug(logging_text + "Enter")
1696         # get all needed from database
1697         db_nsr = None
1698         db_nslcmop = None
1699         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1700         db_nslcmop_update = {}
1701         nslcmop_operation_state = None
1702         nslcmop_operation_state_detail = None
1703         exc = None
1704         try:
1705             step = "Getting information from database"
1706             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1707             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1708
1709             nsr_deployed = db_nsr["_admin"].get("deployed")
1710             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
1711             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1712             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1713             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1714
1715             if vnf_index:
1716                 step = "Getting vnfr from database"
1717                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1718                 step = "Getting vnfd from database"
1719                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1720             else:
1721                 if db_nsr.get("nsd"):
1722                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
1723                 else:
1724                     step = "Getting nsd from database"
1725                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1726
1727             # look if previous tasks in process
1728             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1729             if task_dependency:
1730                 step = db_nslcmop_update["detailed-status"] = \
1731                     "Waiting for related tasks to be completed: {}".format(task_name)
1732                 self.logger.debug(logging_text + step)
1733                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1734                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1735                 if pending:
1736                     raise LcmException("Timeout waiting related tasks to be completed")
1737
1738             # for backward compatibility
1739             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1740                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1741                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1742                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1743
1744             primitive = db_nslcmop["operationParams"]["primitive"]
1745             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
1746
1747             # look for primitive
1748             config_primitive_desc = None
1749             if vdu_id:
1750                 for vdu in get_iterable(db_vnfd, "vdu"):
1751                     if vdu_id == vdu["id"]:
1752                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
1753                             if config_primitive["name"] == primitive:
1754                                 config_primitive_desc = config_primitive
1755                                 break
1756             elif vnf_index:
1757                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1758                     if config_primitive["name"] == primitive:
1759                         config_primitive_desc = config_primitive
1760                         break
1761             else:
1762                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
1763                     if config_primitive["name"] == primitive:
1764                         config_primitive_desc = config_primitive
1765                         break
1766
1767             if not config_primitive_desc:
1768                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
1769                                    format(primitive))
1770
1771             desc_params = {}
1772             if vnf_index:
1773                 if db_vnfr.get("additionalParamsForVnf"):
1774                     desc_params.update(db_vnfr["additionalParamsForVnf"])
1775             else:
1776                 if db_nsr.get("additionalParamsForVnf"):
1777                     desc_params.update(db_nsr["additionalParamsForNs"])
1778
1779             # TODO check if ns is in a proper status
1780             result, result_detail = await self._ns_execute_primitive(
1781                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1782                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
1783             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
1784             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
1785             db_nslcmop_update["statusEnteredTime"] = time()
1786             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
1787             return  # database update is called inside finally
1788
1789         except (DbException, LcmException) as e:
1790             self.logger.error(logging_text + "Exit Exception {}".format(e))
1791             exc = e
1792         except asyncio.CancelledError:
1793             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1794             exc = "Operation was cancelled"
1795         except Exception as e:
1796             exc = traceback.format_exc()
1797             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1798         finally:
1799             if exc and db_nslcmop:
1800                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
1801                     "FAILED {}: {}".format(step, exc)
1802                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1803                 db_nslcmop_update["statusEnteredTime"] = time()
1804             try:
1805                 if db_nslcmop_update:
1806                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1807                 if db_nsr:
1808                     db_nsr_update["_admin.nslcmop"] = None
1809                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1810             except DbException as e:
1811                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1812             self.logger.debug(logging_text + "Exit")
1813             if nslcmop_operation_state:
1814                 try:
1815                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1816                                                                "operationState": nslcmop_operation_state},
1817                                             loop=self.loop)
1818                 except Exception as e:
1819                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1820             self.logger.debug(logging_text + "Exit")
1821             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
1822             return nslcmop_operation_state, nslcmop_operation_state_detail
1823
1824     async def scale(self, nsr_id, nslcmop_id):
1825         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
1826         self.logger.debug(logging_text + "Enter")
1827         # get all needed from database
1828         db_nsr = None
1829         db_nslcmop = None
1830         db_nslcmop_update = {}
1831         nslcmop_operation_state = None
1832         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1833         exc = None
1834         # in case of error, indicates what part of scale was failed to put nsr at error status
1835         scale_process = None
1836         old_operational_status = ""
1837         old_config_status = ""
1838         vnfr_scaled = False
1839         try:
1840             step = "Getting nslcmop from database"
1841             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1842             step = "Getting nsr from database"
1843             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1844
1845             old_operational_status = db_nsr["operational-status"]
1846             old_config_status = db_nsr["config-status"]
1847
1848             # look if previous tasks in process
1849             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1850             if task_dependency:
1851                 step = db_nslcmop_update["detailed-status"] = \
1852                     "Waiting for related tasks to be completed: {}".format(task_name)
1853                 self.logger.debug(logging_text + step)
1854                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1855                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1856                 if pending:
1857                     raise LcmException("Timeout waiting related tasks to be completed")
1858
1859             step = "Parsing scaling parameters"
1860             db_nsr_update["operational-status"] = "scaling"
1861             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1862             nsr_deployed = db_nsr["_admin"].get("deployed")
1863             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
1864             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
1865             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
1866             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
1867             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
1868
1869             # for backward compatibility
1870             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1871                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1872                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1873                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1874
1875             step = "Getting vnfr from database"
1876             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1877             step = "Getting vnfd from database"
1878             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1879             step = "Getting scaling-group-descriptor"
1880             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
1881                 if scaling_descriptor["name"] == scaling_group:
1882                     break
1883             else:
1884                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
1885                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
1886             # cooldown_time = 0
1887             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
1888             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
1889             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
1890             #         break
1891
1892             # TODO check if ns is in a proper status
1893             step = "Sending scale order to RO"
1894             nb_scale_op = 0
1895             if not db_nsr["_admin"].get("scaling-group"):
1896                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
1897                 admin_scale_index = 0
1898             else:
1899                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
1900                     if admin_scale_info["name"] == scaling_group:
1901                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
1902                         break
1903                 else:  # not found, set index one plus last element and add new entry with the name
1904                     admin_scale_index += 1
1905                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
1906             RO_scaling_info = []
1907             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
1908             if scaling_type == "SCALE_OUT":
1909                 # count if max-instance-count is reached
1910                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
1911                     max_instance_count = int(scaling_descriptor["max-instance-count"])
1912                     if nb_scale_op >= max_instance_count:
1913                         raise LcmException("reached the limit of {} (max-instance-count) scaling-out operations for the"
1914                                            " scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1915                 nb_scale_op = nb_scale_op + 1
1916                 vdu_scaling_info["scaling_direction"] = "OUT"
1917                 vdu_scaling_info["vdu-create"] = {}
1918                 for vdu_scale_info in scaling_descriptor["vdu"]:
1919                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1920                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
1921                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1922             elif scaling_type == "SCALE_IN":
1923                 # count if min-instance-count is reached
1924                 min_instance_count = 0
1925                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
1926                     min_instance_count = int(scaling_descriptor["min-instance-count"])
1927                 if nb_scale_op <= min_instance_count:
1928                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
1929                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1930                 nb_scale_op = nb_scale_op - 1
1931                 vdu_scaling_info["scaling_direction"] = "IN"
1932                 vdu_scaling_info["vdu-delete"] = {}
1933                 for vdu_scale_info in scaling_descriptor["vdu"]:
1934                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1935                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
1936                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1937
1938             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
1939             vdu_create = vdu_scaling_info.get("vdu-create")
1940             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
1941             if vdu_scaling_info["scaling_direction"] == "IN":
1942                 for vdur in reversed(db_vnfr["vdur"]):
1943                     if vdu_delete.get(vdur["vdu-id-ref"]):
1944                         vdu_delete[vdur["vdu-id-ref"]] -= 1
1945                         vdu_scaling_info["vdu"].append({
1946                             "name": vdur["name"],
1947                             "vdu_id": vdur["vdu-id-ref"],
1948                             "interface": []
1949                         })
1950                         for interface in vdur["interfaces"]:
1951                             vdu_scaling_info["vdu"][-1]["interface"].append({
1952                                 "name": interface["name"],
1953                                 "ip_address": interface["ip-address"],
1954                                 "mac_address": interface.get("mac-address"),
1955                             })
1956                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
1957
1958             # execute primitive service PRE-SCALING
1959             step = "Executing pre-scale vnf-config-primitive"
1960             if scaling_descriptor.get("scaling-config-action"):
1961                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1962                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
1963                             and scaling_type == "SCALE_IN":
1964                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1965                         step = db_nslcmop_update["detailed-status"] = \
1966                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
1967
1968                         # look for primitive
1969                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1970                             if config_primitive["name"] == vnf_config_primitive:
1971                                 break
1972                         else:
1973                             raise LcmException(
1974                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
1975                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
1976                                 "primitive".format(scaling_group, config_primitive))
1977
1978                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
1979                         if db_vnfr.get("additionalParamsForVnf"):
1980                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1981
1982                         scale_process = "VCA"
1983                         db_nsr_update["config-status"] = "configuring pre-scaling"
1984                         result, result_detail = await self._ns_execute_primitive(
1985                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1986                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1987                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1988                             vnf_config_primitive, result, result_detail))
1989                         if result == "FAILED":
1990                             raise LcmException(result_detail)
1991                         db_nsr_update["config-status"] = old_config_status
1992                         scale_process = None
1993
1994             if RO_scaling_info:
1995                 scale_process = "RO"
1996                 RO = ROclient.ROClient(self.loop, **self.ro_config)
1997                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
1998                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
1999                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2000                 # wait until ready
2001                 RO_nslcmop_id = RO_desc["instance_action_id"]
2002                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2003
2004                 RO_task_done = False
2005                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2006                 detailed_status_old = None
2007                 self.logger.debug(logging_text + step)
2008
2009                 deployment_timeout = 1 * 3600   # One hour
2010                 while deployment_timeout > 0:
2011                     if not RO_task_done:
2012                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2013                                              extra_item_id=RO_nslcmop_id)
2014                         ns_status, ns_status_info = RO.check_action_status(desc)
2015                         if ns_status == "ERROR":
2016                             raise ROclient.ROClientException(ns_status_info)
2017                         elif ns_status == "BUILD":
2018                             detailed_status = step + "; {}".format(ns_status_info)
2019                         elif ns_status == "ACTIVE":
2020                             RO_task_done = True
2021                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2022                             self.logger.debug(logging_text + step)
2023                         else:
2024                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2025                     else:
2026                         desc = await RO.show("ns", RO_nsr_id)
2027                         ns_status, ns_status_info = RO.check_ns_status(desc)
2028                         if ns_status == "ERROR":
2029                             raise ROclient.ROClientException(ns_status_info)
2030                         elif ns_status == "BUILD":
2031                             detailed_status = step + "; {}".format(ns_status_info)
2032                         elif ns_status == "ACTIVE":
2033                             step = detailed_status = \
2034                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2035                             if not vnfr_scaled:
2036                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2037                                 vnfr_scaled = True
2038                             try:
2039                                 desc = await RO.show("ns", RO_nsr_id)
2040                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2041                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2042                                 break
2043                             except LcmExceptionNoMgmtIP:
2044                                 pass
2045                         else:
2046                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2047                     if detailed_status != detailed_status_old:
2048                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2049                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2050
2051                     await asyncio.sleep(5, loop=self.loop)
2052                     deployment_timeout -= 5
2053                 if deployment_timeout <= 0:
2054                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2055
2056                 # update VDU_SCALING_INFO with the obtained ip_addresses
2057                 if vdu_scaling_info["scaling_direction"] == "OUT":
2058                     for vdur in reversed(db_vnfr["vdur"]):
2059                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2060                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2061                             vdu_scaling_info["vdu"].append({
2062                                 "name": vdur["name"],
2063                                 "vdu_id": vdur["vdu-id-ref"],
2064                                 "interface": []
2065                             })
2066                             for interface in vdur["interfaces"]:
2067                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2068                                     "name": interface["name"],
2069                                     "ip_address": interface["ip-address"],
2070                                     "mac_address": interface.get("mac-address"),
2071                                 })
2072                     del vdu_scaling_info["vdu-create"]
2073
2074             scale_process = None
2075             if db_nsr_update:
2076                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2077
2078             # execute primitive service POST-SCALING
2079             step = "Executing post-scale vnf-config-primitive"
2080             if scaling_descriptor.get("scaling-config-action"):
2081                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2082                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2083                             and scaling_type == "SCALE_OUT":
2084                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2085                         step = db_nslcmop_update["detailed-status"] = \
2086                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2087
2088                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2089                         if db_vnfr.get("additionalParamsForVnf"):
2090                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2091
2092                         # look for primitive
2093                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2094                             if config_primitive["name"] == vnf_config_primitive:
2095                                 break
2096                         else:
2097                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2098                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2099                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2100                                                                                                      config_primitive))
2101                         scale_process = "VCA"
2102                         db_nsr_update["config-status"] = "configuring post-scaling"
2103
2104                         result, result_detail = await self._ns_execute_primitive(
2105                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2106                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2107                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2108                             vnf_config_primitive, result, result_detail))
2109                         if result == "FAILED":
2110                             raise LcmException(result_detail)
2111                         db_nsr_update["config-status"] = old_config_status
2112                         scale_process = None
2113
2114             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2115             db_nslcmop_update["statusEnteredTime"] = time()
2116             db_nslcmop_update["detailed-status"] = "done"
2117             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2118             db_nsr_update["operational-status"] = old_operational_status
2119             db_nsr_update["config-status"] = old_config_status
2120             return
2121         except (ROclient.ROClientException, DbException, LcmException) as e:
2122             self.logger.error(logging_text + "Exit Exception {}".format(e))
2123             exc = e
2124         except asyncio.CancelledError:
2125             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2126             exc = "Operation was cancelled"
2127         except Exception as e:
2128             exc = traceback.format_exc()
2129             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2130         finally:
2131             if exc:
2132                 if db_nslcmop:
2133                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2134                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2135                     db_nslcmop_update["statusEnteredTime"] = time()
2136                 if db_nsr:
2137                     db_nsr_update["operational-status"] = old_operational_status
2138                     db_nsr_update["config-status"] = old_config_status
2139                     db_nsr_update["detailed-status"] = ""
2140                     db_nsr_update["_admin.nslcmop"] = None
2141                     if scale_process:
2142                         if "VCA" in scale_process:
2143                             db_nsr_update["config-status"] = "failed"
2144                         if "RO" in scale_process:
2145                             db_nsr_update["operational-status"] = "failed"
2146                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2147                                                                                                      exc)
2148             try:
2149                 if db_nslcmop and db_nslcmop_update:
2150                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2151                 if db_nsr:
2152                     db_nsr_update["_admin.nslcmop"] = None
2153                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2154             except DbException as e:
2155                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2156             if nslcmop_operation_state:
2157                 try:
2158                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2159                                                              "operationState": nslcmop_operation_state},
2160                                             loop=self.loop)
2161                     # if cooldown_time:
2162                     #     await asyncio.sleep(cooldown_time)
2163                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2164                 except Exception as e:
2165                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2166             self.logger.debug(logging_text + "Exit")
2167             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")