osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Update target_dict creating nested dictionaries with the key_list. Last key_list item is assigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 def deep_get(target_dict, key_list):
  71     """
  72     Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
  73     Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
  74     :param target_dict: dictionary to be read
  75     :param key_list: list of keys to read from  target_dict
  76     :return: The wanted value if exist, None otherwise
  77     """
  78     for key in key_list:
  79         if not isinstance(target_dict, dict) or key not in target_dict:
  80             return None
  81         target_dict = target_dict[key]
  82     return target_dict
  83
  84
  85 class NsLcm(LcmBase):
  86     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  87     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  88     timeout_charm_delete = 10 * 60
  89     timeout_primitive = 10 * 60  # timeout for primitive execution
  90
  91     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  92         """
  93         Init, Connect to database, filesystem storage, and messaging
  94         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  95         :return: None
  96         """
  97         # logging
  98         self.logger = logging.getLogger('lcm.ns')
  99         self.loop = loop
 100         self.lcm_tasks = lcm_tasks
 101
 102         super().__init__(db, msg, fs, self.logger)
 103
 104         self.ro_config = ro_config
 105
 106         self.n2vc = N2VC(
 107             log=self.logger,
 108             server=vca_config['host'],
 109             port=vca_config['port'],
 110             user=vca_config['user'],
 111             secret=vca_config['secret'],
 112             # TODO: This should point to the base folder where charms are stored,
 113             # if there is a common one (like object storage). Otherwise, leave
 114             # it unset and pass it via DeployCharms
 115             # artifacts=vca_config[''],
 116             artifacts=None,
 117             juju_public_key=vca_config.get('pubkey'),
 118             ca_cert=vca_config.get('cacert'),
 119         )
 120         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 121
 122     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 123         """
 124         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 125         :param vnfd: input vnfd
 126         :param new_id: overrides vnf id if provided
 127         :param additionalParams: Instantiation params for VNFs provided
 128         :param nsrId: Id of the NSR
 129         :return: copy of vnfd
 130         """
 131         try:
 132             vnfd_RO = deepcopy(vnfd)
 133             # remove unused by RO configuration, monitoring, scaling and internal keys
 134             vnfd_RO.pop("_id", None)
 135             vnfd_RO.pop("_admin", None)
 136             vnfd_RO.pop("vnf-configuration", None)
 137             vnfd_RO.pop("monitoring-param", None)
 138             vnfd_RO.pop("scaling-group-descriptor", None)
 139             if new_id:
 140                 vnfd_RO["id"] = new_id
 141
 142             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 143             for vdu in get_iterable(vnfd_RO, "vdu"):
 144                 cloud_init_file = None
 145                 if vdu.get("cloud-init-file"):
 146                     base_folder = vnfd["_admin"]["storage"]
 147                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 148                                                                    vdu["cloud-init-file"])
 149                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 150                         cloud_init_content = ci_file.read()
 151                     vdu.pop("cloud-init-file", None)
 152                 elif vdu.get("cloud-init"):
 153                     cloud_init_content = vdu["cloud-init"]
 154                 else:
 155                     continue
 156
 157                 env = Environment()
 158                 ast = env.parse(cloud_init_content)
 159                 mandatory_vars = meta.find_undeclared_variables(ast)
 160                 if mandatory_vars:
 161                     for var in mandatory_vars:
 162                         if not additionalParams or var not in additionalParams.keys():
 163                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 164                                                "file, must be provided in the instantiation parameters inside the "
 165                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 166                 template = Template(cloud_init_content)
 167                 cloud_init_content = template.render(additionalParams or {})
 168                 vdu["cloud-init"] = cloud_init_content
 169
 170             return vnfd_RO
 171         except FsException as e:
 172             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 173                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 174         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 175             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 176                                format(vnfd["id"], vdu["id"], e))
 177
 178     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 179         """
 180         Callback both for charm status change and task completion
 181         :param model_name: Charm model name
 182         :param application_name: Charm application name
 183         :param status: Can be
 184             - blocked: The unit needs manual intervention
 185             - maintenance: The unit is actively deploying/configuring
 186             - waiting: The unit is waiting for another charm to be ready
 187             - active: The unit is deployed, configured, and ready
 188             - error: The charm has failed and needs attention.
 189             - terminated: The charm has been destroyed
 190             - removing,
 191             - removed
 192         :param message: detailed message error
 193         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 194             nsr_id:
 195             nslcmop_id:
 196             lcmOperationType: currently "instantiate"
 197             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 198             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 199             n2vc_event: event used to notify instantiation task that some change has been produced
 200         :param task: None for charm status change, or task for completion task callback
 201         :return:
 202         """
 203         try:
 204             nsr_id = n2vc_info["nsr_id"]
 205             deployed = n2vc_info["deployed"]
 206             db_nsr_update = n2vc_info["db_update"]
 207             nslcmop_id = n2vc_info["nslcmop_id"]
 208             ns_operation = n2vc_info["lcmOperationType"]
 209             n2vc_event = n2vc_info["n2vc_event"]
 210             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 211                                                                                     application_name)
 212             for vca_index, vca_deployed in enumerate(deployed):
 213                 if not vca_deployed:
 214                     continue
 215                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 216                     break
 217             else:
 218                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 219                                   format(model_name))
 220                 return
 221             if task:
 222                 if task.cancelled():
 223                     self.logger.debug(logging_text + " task Cancelled")
 224                     vca_deployed['operational-status'] = "error"
 225                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 226                     vca_deployed['detailed-status'] = "Task Cancelled"
 227                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 228
 229                 elif task.done():
 230                     exc = task.exception()
 231                     if exc:
 232                         self.logger.error(logging_text + " task Exception={}".format(exc))
 233                         vca_deployed['operational-status'] = "error"
 234                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 235                         vca_deployed['detailed-status'] = str(exc)
 236                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 237                     else:
 238                         self.logger.debug(logging_text + " task Done")
 239                         # task is Done, but callback is still ongoing. So ignore
 240                         return
 241             elif status:
 242                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 243                 if vca_deployed['operational-status'] == status:
 244                     return  # same status, ignore
 245                 vca_deployed['operational-status'] = status
 246                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 247                 vca_deployed['detailed-status'] = str(message)
 248                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 249             else:
 250                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 251                 return
 252             # wake up instantiate task
 253             n2vc_event.set()
 254         except Exception as e:
 255             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 256
 257     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 258         """
 259         Creates a RO ns descriptor from OSM ns_instantiate params
 260         :param ns_params: OSM instantiate params
 261         :return: The RO ns descriptor
 262         """
 263         vim_2_RO = {}
 264         wim_2_RO = {}
 265         # TODO feature 1417: Check that no instantiation is set over PDU
 266         # check if PDU forces a concrete vim-network-id and add it
 267         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 268
 269         def vim_account_2_RO(vim_account):
 270             if vim_account in vim_2_RO:
 271                 return vim_2_RO[vim_account]
 272
 273             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 274             if db_vim["_admin"]["operationalState"] != "ENABLED":
 275                 raise LcmException("VIM={} is not available. operationalState={}".format(
 276                     vim_account, db_vim["_admin"]["operationalState"]))
 277             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 278             vim_2_RO[vim_account] = RO_vim_id
 279             return RO_vim_id
 280
 281         def wim_account_2_RO(wim_account):
 282             if isinstance(wim_account, str):
 283                 if wim_account in wim_2_RO:
 284                     return wim_2_RO[wim_account]
 285
 286                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 287                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 288                     raise LcmException("WIM={} is not available. operationalState={}".format(
 289                         wim_account, db_wim["_admin"]["operationalState"]))
 290                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 291                 wim_2_RO[wim_account] = RO_wim_id
 292                 return RO_wim_id
 293             else:
 294                 return wim_account
 295
 296         def ip_profile_2_RO(ip_profile):
 297             RO_ip_profile = deepcopy((ip_profile))
 298             if "dns-server" in RO_ip_profile:
 299                 if isinstance(RO_ip_profile["dns-server"], list):
 300                     RO_ip_profile["dns-address"] = []
 301                     for ds in RO_ip_profile.pop("dns-server"):
 302                         RO_ip_profile["dns-address"].append(ds['address'])
 303                 else:
 304                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 305             if RO_ip_profile.get("ip-version") == "ipv4":
 306                 RO_ip_profile["ip-version"] = "IPv4"
 307             if RO_ip_profile.get("ip-version") == "ipv6":
 308                 RO_ip_profile["ip-version"] = "IPv6"
 309             if "dhcp-params" in RO_ip_profile:
 310                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 311             return RO_ip_profile
 312
 313         if not ns_params:
 314             return None
 315         RO_ns_params = {
 316             # "name": ns_params["nsName"],
 317             # "description": ns_params.get("nsDescription"),
 318             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 319             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 320             # "scenario": ns_params["nsdId"],
 321         }
 322         n2vc_key_list = n2vc_key_list or []
 323         for vnfd_ref, vnfd in vnfd_dict.items():
 324             vdu_needed_access = []
 325             mgmt_cp = None
 326             if vnfd.get("vnf-configuration"):
 327                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 328                 if ssh_required and vnfd.get("mgmt-interface"):
 329                     if vnfd["mgmt-interface"].get("vdu-id"):
 330                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 331                     elif vnfd["mgmt-interface"].get("cp"):
 332                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 333
 334             for vdu in vnfd.get("vdu", ()):
 335                 if vdu.get("vdu-configuration"):
 336                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 337                     if ssh_required:
 338                         vdu_needed_access.append(vdu["id"])
 339                 elif mgmt_cp:
 340                     for vdu_interface in vdu.get("interface"):
 341                         if vdu_interface.get("external-connection-point-ref") and \
 342                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 343                             vdu_needed_access.append(vdu["id"])
 344                             mgmt_cp = None
 345                             break
 346
 347             if vdu_needed_access:
 348                 for vnf_member in nsd.get("constituent-vnfd"):
 349                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 350                         continue
 351                     for vdu in vdu_needed_access:
 352                         populate_dict(RO_ns_params,
 353                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 354                                       n2vc_key_list)
 355
 356         if ns_params.get("vduImage"):
 357             RO_ns_params["vduImage"] = ns_params["vduImage"]
 358
 359         if ns_params.get("ssh_keys"):
 360             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 361         for vnf_params in get_iterable(ns_params, "vnf"):
 362             for constituent_vnfd in nsd["constituent-vnfd"]:
 363                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 364                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 365                     break
 366             else:
 367                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 368                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 369             if vnf_params.get("vimAccountId"):
 370                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 371                               vim_account_2_RO(vnf_params["vimAccountId"]))
 372
 373             for vdu_params in get_iterable(vnf_params, "vdu"):
 374                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 375                 if vdu_params.get("volume"):
 376                     for volume_params in vdu_params["volume"]:
 377                         if volume_params.get("vim-volume-id"):
 378                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 379                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 380                                           volume_params["vim-volume-id"])
 381                 if vdu_params.get("interface"):
 382                     for interface_params in vdu_params["interface"]:
 383                         if interface_params.get("ip-address"):
 384                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 385                                                          vdu_params["id"], "interfaces", interface_params["name"],
 386                                                          "ip_address"),
 387                                           interface_params["ip-address"])
 388                         if interface_params.get("mac-address"):
 389                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 390                                                          vdu_params["id"], "interfaces", interface_params["name"],
 391                                                          "mac_address"),
 392                                           interface_params["mac-address"])
 393                         if interface_params.get("floating-ip-required"):
 394                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 395                                                          vdu_params["id"], "interfaces", interface_params["name"],
 396                                                          "floating-ip"),
 397                                           interface_params["floating-ip-required"])
 398
 399             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 400                 if internal_vld_params.get("vim-network-name"):
 401                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 402                                                  internal_vld_params["name"], "vim-network-name"),
 403                                   internal_vld_params["vim-network-name"])
 404                 if internal_vld_params.get("vim-network-id"):
 405                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 406                                                  internal_vld_params["name"], "vim-network-id"),
 407                                   internal_vld_params["vim-network-id"])
 408                 if internal_vld_params.get("ip-profile"):
 409                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 410                                                  internal_vld_params["name"], "ip-profile"),
 411                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 412
 413                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 414                     # look for interface
 415                     iface_found = False
 416                     for vdu_descriptor in vnf_descriptor["vdu"]:
 417                         for vdu_interface in vdu_descriptor["interface"]:
 418                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 419                                 if icp_params.get("ip-address"):
 420                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 421                                                                  vdu_descriptor["id"], "interfaces",
 422                                                                  vdu_interface["name"], "ip_address"),
 423                                                   icp_params["ip-address"])
 424
 425                                 if icp_params.get("mac-address"):
 426                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 427                                                                  vdu_descriptor["id"], "interfaces",
 428                                                                  vdu_interface["name"], "mac_address"),
 429                                                   icp_params["mac-address"])
 430                                 iface_found = True
 431                                 break
 432                         if iface_found:
 433                             break
 434                     else:
 435                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 436                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 437                                            "connection-point".format(vnf_params["member-vnf-index"],
 438                                                                      icp_params["id-ref"]))
 439
 440         for vld_params in get_iterable(ns_params, "vld"):
 441             if "ip-profile" in vld_params:
 442                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 443                               ip_profile_2_RO(vld_params["ip-profile"]))
 444
 445             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 446                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 447                               wim_account_2_RO(vld_params["wimAccountId"])),
 448             if vld_params.get("vim-network-name"):
 449                 RO_vld_sites = []
 450                 if isinstance(vld_params["vim-network-name"], dict):
 451                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 452                         RO_vld_sites.append({
 453                             "netmap-use": vim_net,
 454                             "datacenter": vim_account_2_RO(vim_account)
 455                         })
 456                 else:  # isinstance str
 457                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 458                 if RO_vld_sites:
 459                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 460             if vld_params.get("vim-network-id"):
 461                 RO_vld_sites = []
 462                 if isinstance(vld_params["vim-network-id"], dict):
 463                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 464                         RO_vld_sites.append({
 465                             "netmap-use": vim_net,
 466                             "datacenter": vim_account_2_RO(vim_account)
 467                         })
 468                 else:  # isinstance str
 469                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 470                 if RO_vld_sites:
 471                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 472             if vld_params.get("ns-net"):
 473                 if isinstance(vld_params["ns-net"], dict):
 474                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 475                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 476                 if RO_vld_ns_net:
 477                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 478             if "vnfd-connection-point-ref" in vld_params:
 479                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 480                     # look for interface
 481                     for constituent_vnfd in nsd["constituent-vnfd"]:
 482                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 483                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 484                             break
 485                     else:
 486                         raise LcmException(
 487                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 488                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 489                     match_cp = False
 490                     for vdu_descriptor in vnf_descriptor["vdu"]:
 491                         for interface_descriptor in vdu_descriptor["interface"]:
 492                             if interface_descriptor.get("external-connection-point-ref") == \
 493                                     cp_params["vnfd-connection-point-ref"]:
 494                                 match_cp = True
 495                                 break
 496                         if match_cp:
 497                             break
 498                     else:
 499                         raise LcmException(
 500                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 501                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 502                                 cp_params["member-vnf-index-ref"],
 503                                 cp_params["vnfd-connection-point-ref"],
 504                                 vnf_descriptor["id"]))
 505                     if cp_params.get("ip-address"):
 506                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 507                                                      vdu_descriptor["id"], "interfaces",
 508                                                      interface_descriptor["name"], "ip_address"),
 509                                       cp_params["ip-address"])
 510                     if cp_params.get("mac-address"):
 511                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 512                                                      vdu_descriptor["id"], "interfaces",
 513                                                      interface_descriptor["name"], "mac_address"),
 514                                       cp_params["mac-address"])
 515         return RO_ns_params
 516
 517     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 518         # make a copy to do not change
 519         vdu_create = copy(vdu_create)
 520         vdu_delete = copy(vdu_delete)
 521
 522         vdurs = db_vnfr.get("vdur")
 523         if vdurs is None:
 524             vdurs = []
 525         vdu_index = len(vdurs)
 526         while vdu_index:
 527             vdu_index -= 1
 528             vdur = vdurs[vdu_index]
 529             if vdur.get("pdu-type"):
 530                 continue
 531             vdu_id_ref = vdur["vdu-id-ref"]
 532             if vdu_create and vdu_create.get(vdu_id_ref):
 533                 for index in range(0, vdu_create[vdu_id_ref]):
 534                     vdur = deepcopy(vdur)
 535                     vdur["_id"] = str(uuid4())
 536                     vdur["count-index"] += 1
 537                     vdurs.insert(vdu_index+1+index, vdur)
 538                 del vdu_create[vdu_id_ref]
 539             if vdu_delete and vdu_delete.get(vdu_id_ref):
 540                 del vdurs[vdu_index]
 541                 vdu_delete[vdu_id_ref] -= 1
 542                 if not vdu_delete[vdu_id_ref]:
 543                     del vdu_delete[vdu_id_ref]
 544         # check all operations are done
 545         if vdu_create or vdu_delete:
 546             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 547                 vdu_create))
 548         if vdu_delete:
 549             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 550                 vdu_delete))
 551
 552         vnfr_update = {"vdur": vdurs}
 553         db_vnfr["vdur"] = vdurs
 554         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 555
 556     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 557         """
 558         Updates database nsr with the RO info for the created vld
 559         :param ns_update_nsr: dictionary to be filled with the updated info
 560         :param db_nsr: content of db_nsr. This is also modified
 561         :param nsr_desc_RO: nsr descriptor from RO
 562         :return: Nothing, LcmException is raised on errors
 563         """
 564
 565         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 566             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 567                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 568                     continue
 569                 vld["vim-id"] = net_RO.get("vim_net_id")
 570                 vld["name"] = net_RO.get("vim_name")
 571                 vld["status"] = net_RO.get("status")
 572                 vld["status-detailed"] = net_RO.get("error_msg")
 573                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 574                 break
 575             else:
 576                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 577
 578     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 579         """
 580         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 581         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 582         :param nsr_desc_RO: nsr descriptor from RO
 583         :return: Nothing, LcmException is raised on errors
 584         """
 585         for vnf_index, db_vnfr in db_vnfrs.items():
 586             for vnf_RO in nsr_desc_RO["vnfs"]:
 587                 if vnf_RO["member_vnf_index"] != vnf_index:
 588                     continue
 589                 vnfr_update = {}
 590                 if vnf_RO.get("ip_address"):
 591                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 592                 elif not db_vnfr.get("ip-address"):
 593                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 594
 595                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 596                     vdur_RO_count_index = 0
 597                     if vdur.get("pdu-type"):
 598                         continue
 599                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 600                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 601                             continue
 602                         if vdur["count-index"] != vdur_RO_count_index:
 603                             vdur_RO_count_index += 1
 604                             continue
 605                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 606                         if vdur_RO.get("ip_address"):
 607                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 608                         else:
 609                             vdur["ip-address"] = None
 610                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 611                         vdur["name"] = vdur_RO.get("vim_name")
 612                         vdur["status"] = vdur_RO.get("status")
 613                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 614                         for ifacer in get_iterable(vdur, "interfaces"):
 615                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 616                                 if ifacer["name"] == interface_RO.get("internal_name"):
 617                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 618                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 619                                     break
 620                             else:
 621                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 622                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 623                                                                           ifacer["name"]))
 624                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 625                         break
 626                     else:
 627                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 628                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 629
 630                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 631                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 632                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 633                             continue
 634                         vld["vim-id"] = net_RO.get("vim_net_id")
 635                         vld["name"] = net_RO.get("vim_name")
 636                         vld["status"] = net_RO.get("status")
 637                         vld["status-detailed"] = net_RO.get("error_msg")
 638                         vnfr_update["vld.{}".format(vld_index)] = vld
 639                         break
 640                     else:
 641                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 642                             vnf_index, vld["id"]))
 643
 644                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 645                 break
 646
 647             else:
 648                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 649
 650     async def instantiate(self, nsr_id, nslcmop_id):
 651
 652         # Try to lock HA task here
 653         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 654         if not task_is_locked_by_me:
 655             return
 656
 657         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 658         self.logger.debug(logging_text + "Enter")
 659         # get all needed from database
 660         start_deploy = time()
 661         db_nsr = None
 662         db_nslcmop = None
 663         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 664         db_nslcmop_update = {}
 665         nslcmop_operation_state = None
 666         db_vnfrs = {}
 667         RO_descriptor_number = 0   # number of descriptors created at RO
 668         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 669         n2vc_info = {}
 670         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 671         exc = None
 672         try:
 673             # wait for any previous tasks in process
 674             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 675
 676             step = "Getting nslcmop={} from db".format(nslcmop_id)
 677             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 678             step = "Getting nsr={} from db".format(nsr_id)
 679             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 680             ns_params = db_nslcmop.get("operationParams")
 681             nsd = db_nsr["nsd"]
 682             nsr_name = db_nsr["name"]   # TODO short-name??
 683
 684             step = "Getting vnfrs from db"
 685             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 686             db_vnfds_ref = {}
 687             db_vnfds = {}
 688             db_vnfds_index = {}
 689             for vnfr in db_vnfrs_list:
 690                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 691                 vnfd_id = vnfr["vnfd-id"]
 692                 vnfd_ref = vnfr["vnfd-ref"]
 693                 if vnfd_id not in db_vnfds:
 694                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 695                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 696                     db_vnfds_ref[vnfd_ref] = vnfd
 697                     db_vnfds[vnfd_id] = vnfd
 698                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 699
 700             # Get or generates the _admin.deployed,VCA list
 701             vca_deployed_list = None
 702             vca_model_name = None
 703             if db_nsr["_admin"].get("deployed"):
 704                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 705                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 706             if vca_deployed_list is None:
 707                 vca_deployed_list = []
 708                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 709                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 710             elif isinstance(vca_deployed_list, dict):
 711                 # maintain backward compatibility. Change a dict to list at database
 712                 vca_deployed_list = list(vca_deployed_list.values())
 713                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 714                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 715
 716             db_nsr_update["detailed-status"] = "creating"
 717             db_nsr_update["operational-status"] = "init"
 718             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
 719                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 720                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 721
 722             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 723             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 724             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 725
 726             # Deploy charms
 727             # The parameters we'll need to deploy a charm
 728             number_to_configure = 0
 729
 730             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 731                 """An inner function to deploy the charm from either ns, vnf or vdu
 732                 For ns both vnf_index and vdu_id are None.
 733                 For vnf only vdu_id is None
 734                 For vdu both vnf_index and vdu_id contain a value
 735                 """
 736                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 737                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 738
 739                 machine_spec = {}
 740                 if native_charm:
 741                     machine_spec["username"] = charm_params.get("username"),
 742                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 743
 744                 # Note: The charm needs to exist on disk at the location
 745                 # specified by charm_path.
 746                 descriptor = vnfd if vnf_index else nsd
 747                 base_folder = descriptor["_admin"]["storage"]
 748                 storage_params = self.fs.get_params()
 749                 charm_path = "{}{}/{}/charms/{}".format(
 750                     storage_params["path"],
 751                     base_folder["folder"],
 752                     base_folder["pkg-dir"],
 753                     proxy_charm
 754                 )
 755
 756                 # ns_name will be ignored in the current version of N2VC
 757                 # but will be implemented for the next point release.
 758                 model_name = nsr_id
 759                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 760                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 761                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 762
 763                 vca_index = len(vca_deployed_list)
 764                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 765                 # 26*26 charm in the same NS
 766                 application_name = application_name[0:48]
 767                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 768                 vca_deployed_ = {
 769                     "member-vnf-index": vnf_index,
 770                     "vdu_id": vdu_id,
 771                     "model": model_name,
 772                     "application": application_name,
 773                     "operational-status": "init",
 774                     "detailed-status": "",
 775                     "step": "initial-deploy",
 776                     "vnfd_id": vnfd_id,
 777                     "vdu_name": vdu_name,
 778                     "vdu_count_index": vdu_count_index,
 779                 }
 780                 vca_deployed_list.append(vca_deployed_)
 781                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 782                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 783
 784                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 785                                                                                                 proxy_charm))
 786                 if not n2vc_info:
 787                     n2vc_info["nsr_id"] = nsr_id
 788                     n2vc_info["nslcmop_id"] = nslcmop_id
 789                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 790                     n2vc_info["lcmOperationType"] = "instantiate"
 791                     n2vc_info["deployed"] = vca_deployed_list
 792                     n2vc_info["db_update"] = db_nsr_update
 793                 task = asyncio.ensure_future(
 794                     self.n2vc.DeployCharms(
 795                         model_name,          # The network service name
 796                         application_name,    # The application name
 797                         descriptor,          # The vnf/nsd descriptor
 798                         charm_path,          # Path to charm
 799                         charm_params,        # Runtime params, like mgmt ip
 800                         machine_spec,        # for native charms only
 801                         self.n2vc_callback,  # Callback for status changes
 802                         n2vc_info,           # Callback parameter
 803                         None,                # Callback parameter (task)
 804                     )
 805                 )
 806                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 807                                                          n2vc_info))
 808                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 809
 810             step = "Looking for needed vnfd to configure with proxy charm"
 811             self.logger.debug(logging_text + step)
 812
 813             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 814                 vnfd_id = c_vnf["vnfd-id-ref"]
 815                 vnf_index = str(c_vnf["member-vnf-index"])
 816                 vnfd = db_vnfds_ref[vnfd_id]
 817
 818                 # Get additional parameters
 819                 vnfr_params = {}
 820                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 821                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 822                 for k, v in vnfr_params.items():
 823                     if isinstance(v, str) and v.startswith("!!yaml "):
 824                         vnfr_params[k] = yaml.safe_load(v[7:])
 825
 826                 step = "deploying proxy charms for configuration"
 827                 # Check if this VNF has a charm configuration
 828                 vnf_config = vnfd.get("vnf-configuration")
 829                 if vnf_config and vnf_config.get("juju"):
 830                     proxy_charm = vnf_config["juju"]["charm"]
 831                     if vnf_config["juju"].get("proxy") is False:
 832                         # native_charm, will be deployed after VM. Skip
 833                         proxy_charm = None
 834
 835                     if proxy_charm:
 836                         if not vca_model_name:
 837                             step = "creating VCA model name '{}'".format(nsr_id)
 838                             self.logger.debug(logging_text + step)
 839                             await self.n2vc.CreateNetworkService(nsr_id)
 840                             vca_model_name = nsr_id
 841                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 842                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 843                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 844                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 845                         charm_params = {
 846                             "user_values": vnfr_params,
 847                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 848                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 849                         }
 850
 851                         # Login to the VCA. If there are multiple calls to login(),
 852                         # subsequent calls will be a nop and return immediately.
 853                         await self.n2vc.login()
 854
 855                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 856                         number_to_configure += 1
 857
 858                 # Deploy charms for each VDU that supports one.
 859                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 860                     vdu_config = vdu.get('vdu-configuration')
 861                     proxy_charm = None
 862
 863                     if vdu_config and vdu_config.get("juju"):
 864                         proxy_charm = vdu_config["juju"]["charm"]
 865                         if vdu_config["juju"].get("proxy") is False:
 866                             # native_charm, will be deployed after VM. Skip
 867                             proxy_charm = None
 868
 869                         if proxy_charm:
 870                             if not vca_model_name:
 871                                 step = "creating VCA model name"
 872                                 await self.n2vc.CreateNetworkService(nsr_id)
 873                                 vca_model_name = nsr_id
 874                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 875                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 876                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 877                                                                                                           vdu["id"])
 878                             await self.n2vc.login()
 879                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 880                             # TODO for the moment only first vdu_id contains a charm deployed
 881                             if vdur["vdu-id-ref"] != vdu["id"]:
 882                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 883                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 884                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 885                             charm_params = {
 886                                 "user_values": vnfr_params,
 887                                 "rw_mgmt_ip": vdur["ip-address"],
 888                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 889                             }
 890                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 891                                          charm_params, n2vc_info)
 892                             number_to_configure += 1
 893
 894             # Check if this NS has a charm configuration
 895
 896             ns_config = nsd.get("ns-configuration")
 897             if ns_config and ns_config.get("juju"):
 898                 proxy_charm = ns_config["juju"]["charm"]
 899                 if ns_config["juju"].get("proxy") is False:
 900                     # native_charm, will be deployed after VM. Skip
 901                     proxy_charm = None
 902
 903                 if proxy_charm:
 904                     step = "deploying proxy charm to configure ns"
 905                     # TODO is NS magmt IP address needed?
 906
 907                     # Get additional parameters
 908                     additional_params = {}
 909                     if db_nsr.get("additionalParamsForNs"):
 910                         additional_params = db_nsr["additionalParamsForNs"].copy()
 911                     for k, v in additional_params.items():
 912                         if isinstance(v, str) and v.startswith("!!yaml "):
 913                             additional_params[k] = yaml.safe_load(v[7:])
 914
 915                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 916                     charm_params = {
 917                         "user_values": additional_params,
 918                         # "rw_mgmt_ip": db_nsr["ip-address"],
 919                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 920                     }
 921
 922                     # Login to the VCA. If there are multiple calls to login(),
 923                     # subsequent calls will be a nop and return immediately.
 924                     await self.n2vc.login()
 925                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 926                     number_to_configure += 1
 927
 928             db_nsr_update["operational-status"] = "running"
 929
 930             # Wait until all charms has reached blocked or active status
 931             step = "waiting proxy charms to be ready"
 932             if number_to_configure:
 933                 # wait until all charms are configured.
 934                 # steps are:
 935                 #       initial-deploy
 936                 #       get-ssh-public-key
 937                 #       generate-ssh-key
 938                 #       retry-get-ssh-public-key
 939                 #       ssh-public-key-obtained
 940                 while time() <= start_deploy + self.total_deploy_timeout:
 941                     if db_nsr_update:
 942                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 943                     if db_nslcmop_update:
 944                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 945
 946                     all_active = True
 947                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 948                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 949                         if vca_deployed["step"] == "initial-deploy":
 950                             if vca_deployed["operational-status"] in ("active", "blocked"):
 951                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 952                                        .format(vca_deployed["member-vnf-index"],
 953                                                vca_deployed["vdu_id"])
 954                                 self.logger.debug(logging_text + step)
 955                                 try:
 956                                     primitive_id = await self.n2vc.ExecutePrimitive(
 957                                         vca_deployed["model"],
 958                                         vca_deployed["application"],
 959                                         "get-ssh-public-key",
 960                                         None,
 961                                     )
 962                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 963                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 964                                         primitive_id
 965                                     db_nsr_update[database_entry + "operational-status"] =\
 966                                         vca_deployed["operational-status"]
 967                                 except PrimitiveDoesNotExist:
 968                                     ssh_public_key = None
 969                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 970                                         "ssh-public-key-obtained"
 971                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 972                                         ssh_public_key
 973                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 974                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 975                                     self.logger.debug(logging_text + step)
 976
 977                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 978                             primitive_id = vca_deployed["primitive_id"]
 979                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 980                                                                                   primitive_id)
 981                             if primitive_status in ("completed", "failed"):
 982                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 983                                                                                       primitive_id)
 984                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 985                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 986                                         primitive_result.get("pubkey"):
 987                                     ssh_public_key = primitive_result.get("pubkey")
 988                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 989                                         "ssh-public-key-obtained"
 990                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 991                                         ssh_public_key
 992                                     n2vc_key_list.append(ssh_public_key)
 993                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 994                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 995                                     self.logger.debug(logging_text + step)
 996                                 else:  # primitive_status == "failed":
 997                                     if vca_deployed["step"] == "get-ssh-public-key":
 998                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 999                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
1000                                                                      vca_deployed["vdu_id"])
1001                                         self.logger.debug(logging_text + step)
1002                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1003                                             "generate-ssh-key"
1004                                         primitive_id = await self.n2vc.ExecutePrimitive(
1005                                             vca_deployed["model"],
1006                                             vca_deployed["application"],
1007                                             "generate-ssh-key",
1008                                             None,
1009                                         )
1010                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1011                                             primitive_id
1012                                     else:  # failed for second time
1013                                         raise LcmException(
1014                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
1015
1016                         elif vca_deployed["step"] == "generate-ssh-key":
1017                             primitive_id = vca_deployed["primitive_id"]
1018                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1019                                                                                   primitive_id)
1020                             if primitive_status in ("completed", "failed"):
1021                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1022                                                                                       primitive_id)
1023                                 vca_deployed["primitive_id"] = db_nsr_update[
1024                                     database_entry + "primitive_id"] = None
1025                                 if primitive_status == "completed":
1026                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1027                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1028                                                               vca_deployed["vdu_id"])
1029                                     self.logger.debug(logging_text + step)
1030                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1031                                         "retry-get-ssh-public-key"
1032                                     primitive_id = await self.n2vc.ExecutePrimitive(
1033                                         vca_deployed["model"],
1034                                         vca_deployed["application"],
1035                                         "get-ssh-public-key",
1036                                         None,
1037                                     )
1038                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1039                                         primitive_id
1040
1041                                 else:  # primitive_status == "failed":
1042                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1043                                                        .format(primitive_result))
1044
1045                         if vca_deployed["step"] != "ssh-public-key-obtained":
1046                             all_active = False
1047
1048                     if all_active:
1049                         break
1050                     await asyncio.sleep(5)
1051                 else:   # total_deploy_timeout
1052                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1053                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1054
1055             # deploy RO
1056             # get vnfds, instantiate at RO
1057             for c_vnf in nsd.get("constituent-vnfd", ()):
1058                 member_vnf_index = c_vnf["member-vnf-index"]
1059                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1060                 vnfd_ref = vnfd["id"]
1061                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1062                     vnfd_ref, member_vnf_index)
1063                 # self.logger.debug(logging_text + step)
1064                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1065                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1066                 RO_descriptor_number += 1
1067
1068                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1069                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1070                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1071                         break
1072                 else:
1073                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1074                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1075
1076                 # look if present
1077                 RO_update = {"member-vnf-index": member_vnf_index}
1078                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1079                 if vnfd_list:
1080                     RO_update["id"] = vnfd_list[0]["uuid"]
1081                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1082                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1083                 else:
1084                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1085                                            get("additionalParamsForVnf"), nsr_id)
1086                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1087                     RO_update["id"] = desc["uuid"]
1088                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1089                         vnfd_ref, member_vnf_index, desc["uuid"]))
1090                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1091                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1092                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1093
1094             # create nsd at RO
1095             nsd_ref = nsd["id"]
1096             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1097             # self.logger.debug(logging_text + step)
1098
1099             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1100             RO_descriptor_number += 1
1101             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1102             if nsd_list:
1103                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1104                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1105                     nsd_ref, RO_nsd_uuid))
1106             else:
1107                 nsd_RO = deepcopy(nsd)
1108                 nsd_RO["id"] = RO_osm_nsd_id
1109                 nsd_RO.pop("_id", None)
1110                 nsd_RO.pop("_admin", None)
1111                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1112                     member_vnf_index = c_vnf["member-vnf-index"]
1113                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1114                 for c_vld in nsd_RO.get("vld", ()):
1115                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1116                         member_vnf_index = cp["member-vnf-index-ref"]
1117                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1118
1119                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1120                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1121                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1122                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1123             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1124
1125             # Crate ns at RO
1126             # if present use it unless in error status
1127             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1128             if RO_nsr_id:
1129                 try:
1130                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1131                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1132                     desc = await self.RO.show("ns", RO_nsr_id)
1133                 except ROclient.ROClientException as e:
1134                     if e.http_code != HTTPStatus.NOT_FOUND:
1135                         raise
1136                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1137                 if RO_nsr_id:
1138                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1139                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1140                     if ns_status == "ERROR":
1141                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1142                         self.logger.debug(logging_text + step)
1143                         await self.RO.delete("ns", RO_nsr_id)
1144                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1145             if not RO_nsr_id:
1146                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1147                 # self.logger.debug(logging_text + step)
1148
1149                 # check if VIM is creating and wait  look if previous tasks in process
1150                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1151                 if task_dependency:
1152                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1153                     self.logger.debug(logging_text + step)
1154                     await asyncio.wait(task_dependency, timeout=3600)
1155                 if ns_params.get("vnf"):
1156                     for vnf in ns_params["vnf"]:
1157                         if "vimAccountId" in vnf:
1158                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1159                                                                                         vnf["vimAccountId"])
1160                         if task_dependency:
1161                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1162                             self.logger.debug(logging_text + step)
1163                             await asyncio.wait(task_dependency, timeout=3600)
1164
1165                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1166
1167                 # feature 1429. Add n2vc public key to needed VMs
1168                 n2vc_key = await self.n2vc.GetPublicKey()
1169                 n2vc_key_list.append(n2vc_key)
1170                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1171
1172                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1173                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1174                                             name=db_nsr["name"],
1175                                             scenario=RO_nsd_uuid)
1176                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1177                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1178                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1179                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1180             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1181
1182             # wait until NS is ready
1183             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1184             detailed_status_old = None
1185             self.logger.debug(logging_text + step)
1186
1187             while time() <= start_deploy + self.total_deploy_timeout:
1188                 desc = await self.RO.show("ns", RO_nsr_id)
1189                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1190                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1191                 if ns_status == "ERROR":
1192                     raise ROclient.ROClientException(ns_status_info)
1193                 elif ns_status == "BUILD":
1194                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1195                 elif ns_status == "ACTIVE":
1196                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1197                     try:
1198                         self.ns_update_vnfr(db_vnfrs, desc)
1199                         break
1200                     except LcmExceptionNoMgmtIP:
1201                         pass
1202                 else:
1203                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1204                 if detailed_status != detailed_status_old:
1205                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1206                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1207                 await asyncio.sleep(5, loop=self.loop)
1208             else:   # total_deploy_timeout
1209                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1210
1211             step = "Updating NSR"
1212             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1213
1214             db_nsr_update["operational-status"] = "running"
1215             db_nsr["detailed-status"] = "Configuring vnfr"
1216             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1217
1218             # Configure proxy charms once VMs are up
1219             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1220                 vnf_index = vca_deployed.get("member-vnf-index")
1221                 vdu_id = vca_deployed.get("vdu_id")
1222                 vdu_name = None
1223                 vdu_count_index = None
1224
1225                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1226                                                                                                            vdu_id)
1227                 add_params = {}
1228                 initial_config_primitive_list = []
1229                 if vnf_index:
1230                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1231                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1232                     vnfd = db_vnfds_index[vnf_index]
1233
1234                     if vdu_id:
1235                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1236                             if vdu["id"] == vdu_id:
1237                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1238                                     'initial-config-primitive', [])
1239                                 break
1240                         else:
1241                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1242                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1243                         # TODO for the moment only first vdu_id contains a charm deployed
1244                         if vdur["vdu-id-ref"] != vdu["id"]:
1245                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1246                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1247                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1248                     else:
1249                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1250                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1251                 else:
1252                     if db_nsr.get("additionalParamsForNs"):
1253                         add_params = db_nsr["additionalParamsForNs"].copy()
1254                     for k, v in add_params.items():
1255                         if isinstance(v, str) and v.startswith("!!yaml "):
1256                             add_params[k] = yaml.safe_load(v[7:])
1257                     add_params["rw_mgmt_ip"] = None
1258                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1259
1260                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1261                 initial_config_primitive_list = initial_config_primitive_list.copy()
1262                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1263                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1264
1265                 for initial_config_primitive in initial_config_primitive_list:
1266                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1267                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1268                                       .format(initial_config_primitive["name"], primitive_params_))
1269                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1270                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1271                         initial_config_primitive["name"],
1272                         primitive_params_,
1273                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1274                         retries_interval=30)
1275                     if primitive_result != "COMPLETED":
1276                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1277                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1278                                                    vca_deployed["vdu_id"], primitive_detail))
1279
1280             # Deploy native charms
1281             step = "Looking for needed vnfd to configure with native charm"
1282             self.logger.debug(logging_text + step)
1283
1284             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1285                 vnfd_id = c_vnf["vnfd-id-ref"]
1286                 vnf_index = str(c_vnf["member-vnf-index"])
1287                 vnfd = db_vnfds_ref[vnfd_id]
1288
1289                 # Get additional parameters
1290                 vnfr_params = {}
1291                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1292                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1293                 for k, v in vnfr_params.items():
1294                     if isinstance(v, str) and v.startswith("!!yaml "):
1295                         vnfr_params[k] = yaml.safe_load(v[7:])
1296
1297                 # Check if this VNF has a charm configuration
1298                 vnf_config = vnfd.get("vnf-configuration")
1299                 if vnf_config and vnf_config.get("juju"):
1300                     native_charm = vnf_config["juju"].get("proxy") is False
1301
1302                     if native_charm:
1303                         if not vca_model_name:
1304                             step = "creating VCA model name '{}'".format(nsr_id)
1305                             self.logger.debug(logging_text + step)
1306                             await self.n2vc.CreateNetworkService(nsr_id)
1307                             vca_model_name = nsr_id
1308                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1309                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1310                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1311                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1312                         charm_params = {
1313                             "user_values": vnfr_params,
1314                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1315                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1316                         }
1317
1318                         # get username
1319                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1320                         #  merged. Meanwhile let's get username from initial-config-primitive
1321                         if vnf_config.get("initial-config-primitive"):
1322                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1323                                 if param["name"] == "ssh-username":
1324                                     charm_params["username"] = param["value"]
1325                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1326                             if vnf_config["config-access"]["ssh-access"].get("required"):
1327                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1328
1329                         # Login to the VCA. If there are multiple calls to login(),
1330                         # subsequent calls will be a nop and return immediately.
1331                         await self.n2vc.login()
1332
1333                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1334                         number_to_configure += 1
1335
1336                 # Deploy charms for each VDU that supports one.
1337                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1338                     vdu_config = vdu.get('vdu-configuration')
1339                     native_charm = False
1340
1341                     if vdu_config and vdu_config.get("juju"):
1342                         native_charm = vdu_config["juju"].get("proxy") is False
1343
1344                         if native_charm:
1345                             if not vca_model_name:
1346                                 step = "creating VCA model name"
1347                                 await self.n2vc.CreateNetworkService(nsr_id)
1348                                 vca_model_name = nsr_id
1349                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1350                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1351                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1352                                                                                                      vdu["id"])
1353                             await self.n2vc.login()
1354                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1355                             # TODO for the moment only first vdu_id contains a charm deployed
1356                             if vdur["vdu-id-ref"] != vdu["id"]:
1357                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1358                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1359                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1360                             charm_params = {
1361                                 "user_values": vnfr_params,
1362                                 "rw_mgmt_ip": vdur["ip-address"],
1363                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1364                             }
1365
1366                             # get username
1367                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1368                             #  merged. Meanwhile let's get username from initial-config-primitive
1369                             if vdu_config.get("initial-config-primitive"):
1370                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1371                                     if param["name"] == "ssh-username":
1372                                         charm_params["username"] = param["value"]
1373                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1374                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1375                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1376                                         "default-user")
1377
1378                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1379                                          charm_params, n2vc_info, native_charm)
1380                             number_to_configure += 1
1381
1382             # Check if this NS has a charm configuration
1383
1384             ns_config = nsd.get("ns-configuration")
1385             if ns_config and ns_config.get("juju"):
1386                 native_charm = ns_config["juju"].get("proxy") is False
1387
1388                 if native_charm:
1389                     step = "deploying native charm to configure ns"
1390                     # TODO is NS magmt IP address needed?
1391
1392                     # Get additional parameters
1393                     additional_params = {}
1394                     if db_nsr.get("additionalParamsForNs"):
1395                         additional_params = db_nsr["additionalParamsForNs"].copy()
1396                     for k, v in additional_params.items():
1397                         if isinstance(v, str) and v.startswith("!!yaml "):
1398                             additional_params[k] = yaml.safe_load(v[7:])
1399
1400                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1401                     charm_params = {
1402                         "user_values": additional_params,
1403                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1404                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1405                     }
1406
1407                     # get username
1408                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1409                     #  merged. Meanwhile let's get username from initial-config-primitive
1410                     if ns_config.get("initial-config-primitive"):
1411                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1412                             if param["name"] == "ssh-username":
1413                                 charm_params["username"] = param["value"]
1414                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1415                         if ns_config["config-access"]["ssh-access"].get("required"):
1416                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1417
1418                     # Login to the VCA. If there are multiple calls to login(),
1419                     # subsequent calls will be a nop and return immediately.
1420                     await self.n2vc.login()
1421                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1422                     number_to_configure += 1
1423
1424             # waiting all charms are ok
1425             configuration_failed = False
1426             if number_to_configure:
1427                 step = "Waiting all charms are active"
1428                 old_status = "configuring: init: {}".format(number_to_configure)
1429                 db_nsr_update["config-status"] = old_status
1430                 db_nsr_update["detailed-status"] = old_status
1431                 db_nslcmop_update["detailed-status"] = old_status
1432
1433                 # wait until all are configured.
1434                 while time() <= start_deploy + self.total_deploy_timeout:
1435                     if db_nsr_update:
1436                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1437                     if db_nslcmop_update:
1438                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1439                     # TODO add a fake task that set n2vc_event after some time
1440                     await n2vc_info["n2vc_event"].wait()
1441                     n2vc_info["n2vc_event"].clear()
1442                     all_active = True
1443                     status_map = {}
1444                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1445                     now = time()
1446                     for vca_deployed in vca_deployed_list:
1447                         vca_status = vca_deployed["operational-status"]
1448                         if vca_status not in status_map:
1449                             # Initialize it
1450                             status_map[vca_status] = 0
1451                         status_map[vca_status] += 1
1452
1453                         if vca_status == "active":
1454                             vca_deployed.pop("time_first_error", None)
1455                             vca_deployed.pop("status_first_error", None)
1456                             continue
1457
1458                         all_active = False
1459                         if vca_status in ("error", "blocked"):
1460                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1461                             # if not first time in this status error
1462                             if not vca_deployed.get("time_first_error"):
1463                                 vca_deployed["time_first_error"] = now
1464                                 continue
1465                         if vca_deployed.get("time_first_error") and \
1466                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1467                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1468                                                    .format(vca_deployed["member-vnf-index"],
1469                                                            vca_deployed["vdu_id"], vca_status,
1470                                                            vca_deployed["detailed-status-error"]))
1471
1472                     if all_active:
1473                         break
1474                     elif n2vc_error_text:
1475                         db_nsr_update["config-status"] = "failed"
1476                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1477                         db_nsr_update["detailed-status"] = error_text
1478                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1479                         db_nslcmop_update["detailed-status"] = error_text
1480                         db_nslcmop_update["statusEnteredTime"] = time()
1481                         configuration_failed = True
1482                         break
1483                     else:
1484                         cs = "configuring: "
1485                         separator = ""
1486                         for status, num in status_map.items():
1487                             cs += separator + "{}: {}".format(status, num)
1488                             separator = ", "
1489                         if old_status != cs:
1490                             db_nsr_update["config-status"] = cs
1491                             db_nsr_update["detailed-status"] = cs
1492                             db_nslcmop_update["detailed-status"] = cs
1493                             old_status = cs
1494                 else:   # total_deploy_timeout
1495                     raise LcmException("Timeout waiting ns to be configured")
1496
1497             if not configuration_failed:
1498                 # all is done
1499                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1500                 db_nslcmop_update["statusEnteredTime"] = time()
1501                 db_nslcmop_update["detailed-status"] = "done"
1502                 db_nsr_update["config-status"] = "configured"
1503                 db_nsr_update["detailed-status"] = "done"
1504
1505             return
1506
1507         except (ROclient.ROClientException, DbException, LcmException) as e:
1508             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1509             exc = e
1510         except asyncio.CancelledError:
1511             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1512             exc = "Operation was cancelled"
1513         except Exception as e:
1514             exc = traceback.format_exc()
1515             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1516                                  exc_info=True)
1517         finally:
1518             if exc:
1519                 if db_nsr:
1520                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1521                     db_nsr_update["operational-status"] = "failed"
1522                 if db_nslcmop:
1523                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1524                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1525                     db_nslcmop_update["statusEnteredTime"] = time()
1526             try:
1527                 if db_nsr:
1528                     db_nsr_update["_admin.nslcmop"] = None
1529                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1530                 if db_nslcmop_update:
1531                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1532             except DbException as e:
1533                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1534             if nslcmop_operation_state:
1535                 try:
1536                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1537                                                                    "operationState": nslcmop_operation_state},
1538                                             loop=self.loop)
1539                 except Exception as e:
1540                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1541
1542             self.logger.debug(logging_text + "Exit")
1543             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1544
1545     async def _destroy_charm(self, model, application):
1546         """
1547         Order N2VC destroy a charm
1548         :param model:
1549         :param application:
1550         :return: True if charm does not exist. False if it exist
1551         """
1552         if not await self.n2vc.HasApplication(model, application):
1553             return True  # Already removed
1554         await self.n2vc.RemoveCharms(model, application)
1555         return False
1556
1557     async def _wait_charm_destroyed(self, model, application, timeout):
1558         """
1559         Wait until charm does not exist
1560         :param model:
1561         :param application:
1562         :param timeout:
1563         :return: True if not exist, False if timeout
1564         """
1565         while True:
1566             if not await self.n2vc.HasApplication(model, application):
1567                 return True
1568             if timeout < 0:
1569                 return False
1570             await asyncio.sleep(10)
1571             timeout -= 10
1572
1573     # Check if this VNFD has a configured terminate action
1574     def _has_terminate_config_primitive(self, vnfd):
1575         vnf_config = vnfd.get("vnf-configuration")
1576         if vnf_config and vnf_config.get("terminate-config-primitive"):
1577             return True
1578         else:
1579             return False
1580
1581     @staticmethod
1582     def _get_terminate_config_primitive_seq_list(vnfd):
1583         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1584         # No need to check for existing primitive twice, already done before
1585         vnf_config = vnfd.get("vnf-configuration")
1586         seq_list = vnf_config.get("terminate-config-primitive")
1587         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1588         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1589         return seq_list_sorted
1590
1591     @staticmethod
1592     def _create_nslcmop(nsr_id, operation, params):
1593         """
1594         Creates a ns-lcm-opp content to be stored at database.
1595         :param nsr_id: internal id of the instance
1596         :param operation: instantiate, terminate, scale, action, ...
1597         :param params: user parameters for the operation
1598         :return: dictionary following SOL005 format
1599         """
1600         # Raise exception if invalid arguments
1601         if not (nsr_id and operation and params):
1602             raise LcmException(
1603                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1604         now = time()
1605         _id = str(uuid4())
1606         nslcmop = {
1607             "id": _id,
1608             "_id": _id,
1609             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1610             "operationState": "PROCESSING",
1611             "statusEnteredTime": now,
1612             "nsInstanceId": nsr_id,
1613             "lcmOperationType": operation,
1614             "startTime": now,
1615             "isAutomaticInvocation": False,
1616             "operationParams": params,
1617             "isCancelPending": False,
1618             "links": {
1619                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1620                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1621             }
1622         }
1623         return nslcmop
1624
1625     def _get_terminate_primitive_params(self, seq, vnf_index):
1626         primitive = seq.get('name')
1627         primitive_params = {}
1628         params = {
1629             "member_vnf_index": vnf_index,
1630             "primitive": primitive,
1631             "primitive_params": primitive_params,
1632         }
1633         desc_params = {}
1634         return self._map_primitive_params(seq, params, desc_params)
1635
1636     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1637                           vdu_name, primitive, mapped_primitive_params):
1638         # Create the "_admin.operations" array, or append operation if array already exists
1639         key_admin = '_admin'
1640         key_operations = 'operations'
1641         db_nslcmop_update = {}
1642         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1643         op_list = db_nslcmop_admin.get(key_operations)
1644         new_op = {'member_vnf_index': vnf_index,
1645                   'vdu_id': vdu_id,
1646                   'vdu_count_index': vdu_count_index,
1647                   'primitive': primitive,
1648                   'primitive_params': mapped_primitive_params}
1649         if db_nslcmop_admin:
1650             if not op_list:
1651                 # First operation, create key 'operations' with current operation as first list element
1652                 db_nslcmop_admin.update({key_operations: [new_op]})
1653                 op_list = db_nslcmop_admin.get(key_operations)
1654             else:
1655                 # Not first operation, append operation to existing list
1656                 op_list.append(new_op)
1657
1658         db_nslcmop_update['_admin.operations'] = op_list
1659         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1660
1661     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1662         """ Create a primitive with params from VNFD
1663             Called from terminate() before deleting instance
1664             Calls action() to execute the primitive """
1665         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1666         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1667         db_vnfds = {}
1668         # Loop over VNFRs
1669         for vnfr in db_vnfrs_list:
1670             vnfd_id = vnfr["vnfd-id"]
1671             vnf_index = vnfr["member-vnf-index-ref"]
1672             if vnfd_id not in db_vnfds:
1673                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1674                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1675                 db_vnfds[vnfd_id] = vnfd
1676             vnfd = db_vnfds[vnfd_id]
1677             if not self._has_terminate_config_primitive(vnfd):
1678                 continue
1679             # Get the primitive's sorted sequence list
1680             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1681             for seq in seq_list:
1682                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1683                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1684                     vnf_index, seq.get("name"))
1685                 self.logger.debug(logging_text + step)
1686                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1687                 primitive = seq.get('name')
1688                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1689                 # The following 3 parameters are currently set to None for 'terminate':
1690                 # vdu_id, vdu_count_index, vdu_name
1691                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1692                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1693                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1694                 # Add suboperation
1695                 self._add_suboperation(db_nslcmop,
1696                                        nslcmop_id,
1697                                        vnf_index,
1698                                        vdu_id,
1699                                        vdu_count_index,
1700                                        vdu_name,
1701                                        primitive,
1702                                        mapped_primitive_params)
1703                 # Suboperations: Call _ns_execute_primitive() instead of action()
1704                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1705                 nsr_deployed = db_nsr["_admin"]["deployed"]
1706                 result, result_detail = await self._ns_execute_primitive(
1707                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1708                     mapped_primitive_params)
1709
1710                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1711                 #    nsr_id, nslcmop_terminate_action_id)
1712                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1713                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1714                 if result not in result_ok:
1715                     raise LcmException(
1716                         "terminate_primitive_action for vnf_member_index={}",
1717                         " primitive={} fails with error {}".format(
1718                             vnf_index, seq.get("name"), result_detail))
1719
1720     async def terminate(self, nsr_id, nslcmop_id):
1721
1722         # Try to lock HA task here
1723         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1724         if not task_is_locked_by_me:
1725             return
1726
1727         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1728         self.logger.debug(logging_text + "Enter")
1729         db_nsr = None
1730         db_nslcmop = None
1731         exc = None
1732         failed_detail = []   # annotates all failed error messages
1733         vca_time_destroy = None   # time of where destroy charm order
1734         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1735         db_nslcmop_update = {}
1736         nslcmop_operation_state = None
1737         autoremove = False  # autoremove after terminated
1738         try:
1739             # wait for any previous tasks in process
1740             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1741
1742             step = "Getting nslcmop={} from db".format(nslcmop_id)
1743             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1744             step = "Getting nsr={} from db".format(nsr_id)
1745             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1746             # nsd = db_nsr["nsd"]
1747             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1748             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1749                 return
1750             # #TODO check if VIM is creating and wait
1751             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1752             # Call internal terminate action
1753             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1754
1755             db_nsr_update["operational-status"] = "terminating"
1756             db_nsr_update["config-status"] = "terminating"
1757
1758             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1759                 vca_model_name = nsr_deployed["VCA-model-name"]
1760                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1761                 self.logger.debug(logging_text + step)
1762                 try:
1763                     await self.n2vc.DestroyNetworkService(vca_model_name)
1764                 except NetworkServiceDoesNotExist:
1765                     pass
1766                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1767                 if nsr_deployed.get("VCA"):
1768                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1769                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1770                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1771             # for backward compatibility if charm have been created with "default"  model name delete one by one
1772             elif nsr_deployed and nsr_deployed.get("VCA"):
1773                 try:
1774                     step = "Scheduling configuration charms removing"
1775                     db_nsr_update["detailed-status"] = "Deleting charms"
1776                     self.logger.debug(logging_text + step)
1777                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1778                     # for backward compatibility
1779                     if isinstance(nsr_deployed["VCA"], dict):
1780                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1781                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1782                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1783
1784                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1785                         if vca_deployed:
1786                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1787                                 vca_deployed.clear()
1788                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1789                             else:
1790                                 vca_time_destroy = time()
1791                 except Exception as e:
1792                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1793
1794             # remove from RO
1795             RO_fail = False
1796
1797             # Delete ns
1798             RO_nsr_id = RO_delete_action = None
1799             if nsr_deployed and nsr_deployed.get("RO"):
1800                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1801                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1802             try:
1803                 if RO_nsr_id:
1804                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1805                         "Deleting ns from VIM"
1806                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1807                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1808                     self.logger.debug(logging_text + step)
1809                     desc = await self.RO.delete("ns", RO_nsr_id)
1810                     RO_delete_action = desc["action_id"]
1811                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1812                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1813                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1814                 if RO_delete_action:
1815                     # wait until NS is deleted from VIM
1816                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1817                         format(RO_nsr_id, RO_delete_action)
1818                     detailed_status_old = None
1819                     self.logger.debug(logging_text + step)
1820
1821                     delete_timeout = 20 * 60   # 20 minutes
1822                     while delete_timeout > 0:
1823                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1824                                                   extra_item_id=RO_delete_action)
1825                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1826                         if ns_status == "ERROR":
1827                             raise ROclient.ROClientException(ns_status_info)
1828                         elif ns_status == "BUILD":
1829                             detailed_status = step + "; {}".format(ns_status_info)
1830                         elif ns_status == "ACTIVE":
1831                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1832                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1833                             break
1834                         else:
1835                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1836                         if detailed_status != detailed_status_old:
1837                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1838                                 db_nsr_update["detailed-status"] = detailed_status
1839                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1840                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1841                         await asyncio.sleep(5, loop=self.loop)
1842                         delete_timeout -= 5
1843                     else:  # delete_timeout <= 0:
1844                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1845
1846             except ROclient.ROClientException as e:
1847                 if e.http_code == 404:  # not found
1848                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1849                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1850                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1851                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1852                 elif e.http_code == 409:   # conflict
1853                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1854                     self.logger.debug(logging_text + failed_detail[-1])
1855                     RO_fail = True
1856                 else:
1857                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1858                     self.logger.error(logging_text + failed_detail[-1])
1859                     RO_fail = True
1860
1861             # Delete nsd
1862             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1863                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1864                 try:
1865                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1866                         "Deleting nsd from RO"
1867                     await self.RO.delete("nsd", RO_nsd_id)
1868                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1869                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1870                 except ROclient.ROClientException as e:
1871                     if e.http_code == 404:  # not found
1872                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1873                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1874                     elif e.http_code == 409:   # conflict
1875                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1876                         self.logger.debug(logging_text + failed_detail[-1])
1877                         RO_fail = True
1878                     else:
1879                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1880                         self.logger.error(logging_text + failed_detail[-1])
1881                         RO_fail = True
1882
1883             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1884                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1885                     if not vnf_deployed or not vnf_deployed["id"]:
1886                         continue
1887                     try:
1888                         RO_vnfd_id = vnf_deployed["id"]
1889                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1890                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1891                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1892                         await self.RO.delete("vnfd", RO_vnfd_id)
1893                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1894                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1895                     except ROclient.ROClientException as e:
1896                         if e.http_code == 404:  # not found
1897                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1898                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1899                         elif e.http_code == 409:   # conflict
1900                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1901                             self.logger.debug(logging_text + failed_detail[-1])
1902                         else:
1903                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1904                             self.logger.error(logging_text + failed_detail[-1])
1905
1906             # wait until charm deleted
1907             if vca_time_destroy:
1908                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1909                     "Waiting for deletion of configuration charms"
1910                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1911                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1912                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1913                     if not vca_deployed:
1914                         continue
1915                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1916                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1917                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1918                                                             timeout):
1919                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1920                             vca_deployed["application"]))
1921                     else:
1922                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1923
1924             if failed_detail:
1925                 self.logger.error(logging_text + " ;".join(failed_detail))
1926                 db_nsr_update["operational-status"] = "failed"
1927                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1928                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1929                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1930                 db_nslcmop_update["statusEnteredTime"] = time()
1931             else:
1932                 db_nsr_update["operational-status"] = "terminated"
1933                 db_nsr_update["detailed-status"] = "Done"
1934                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1935                 db_nslcmop_update["detailed-status"] = "Done"
1936                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1937                 db_nslcmop_update["statusEnteredTime"] = time()
1938                 if db_nslcmop["operationParams"].get("autoremove"):
1939                     autoremove = True
1940
1941         except (ROclient.ROClientException, DbException, LcmException) as e:
1942             self.logger.error(logging_text + "Exit Exception {}".format(e))
1943             exc = e
1944         except asyncio.CancelledError:
1945             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1946             exc = "Operation was cancelled"
1947         except Exception as e:
1948             exc = traceback.format_exc()
1949             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1950         finally:
1951             if exc and db_nslcmop:
1952                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1953                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1954                 db_nslcmop_update["statusEnteredTime"] = time()
1955             try:
1956                 if db_nslcmop and db_nslcmop_update:
1957                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1958                 if db_nsr:
1959                     db_nsr_update["_admin.nslcmop"] = None
1960                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1961             except DbException as e:
1962                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1963             if nslcmop_operation_state:
1964                 try:
1965                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1966                                                                  "operationState": nslcmop_operation_state,
1967                                                                  "autoremove": autoremove},
1968                                             loop=self.loop)
1969                 except Exception as e:
1970                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1971             self.logger.debug(logging_text + "Exit")
1972             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1973
1974     @staticmethod
1975     def _map_primitive_params(primitive_desc, params, instantiation_params):
1976         """
1977         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1978         The default-value is used. If it is between < > it look for a value at instantiation_params
1979         :param primitive_desc: portion of VNFD/NSD that describes primitive
1980         :param params: Params provided by user
1981         :param instantiation_params: Instantiation params provided by user
1982         :return: a dictionary with the calculated params
1983         """
1984         calculated_params = {}
1985         for parameter in primitive_desc.get("parameter", ()):
1986             param_name = parameter["name"]
1987             if param_name in params:
1988                 calculated_params[param_name] = params[param_name]
1989             elif "default-value" in parameter or "value" in parameter:
1990                 if "value" in parameter:
1991                     calculated_params[param_name] = parameter["value"]
1992                 else:
1993                     calculated_params[param_name] = parameter["default-value"]
1994                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1995                         and calculated_params[param_name].endswith(">"):
1996                     if calculated_params[param_name][1:-1] in instantiation_params:
1997                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1998                     else:
1999                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
2000                                            format(parameter["default-value"], primitive_desc["name"]))
2001             else:
2002                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
2003                                    format(param_name, primitive_desc["name"]))
2004
2005             if isinstance(calculated_params[param_name], (dict, list, tuple)):
2006                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
2007                                                                width=256)
2008             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
2009                 calculated_params[param_name] = calculated_params[param_name][7:]
2010         return calculated_params
2011
2012     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
2013                                     primitive, primitive_params, retries=0, retries_interval=30):
2014         start_primitive_time = time()
2015         try:
2016             for vca_deployed in db_deployed["VCA"]:
2017                 if not vca_deployed:
2018                     continue
2019                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2020                     continue
2021                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2022                     continue
2023                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2024                     continue
2025                 break
2026             else:
2027                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2028                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2029             model_name = vca_deployed.get("model")
2030             application_name = vca_deployed.get("application")
2031             if not model_name or not application_name:
2032                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2033                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2034                                                                        vdu_count_index))
2035             # if vca_deployed["operational-status"] != "active":
2036             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2037             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2038             callback = None  # self.n2vc_callback
2039             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2040             await self.n2vc.login()
2041             if primitive == "config":
2042                 primitive_params = {"params": primitive_params}
2043             while retries >= 0:
2044                 primitive_id = await self.n2vc.ExecutePrimitive(
2045                     model_name,
2046                     application_name,
2047                     primitive,
2048                     callback,
2049                     *callback_args,
2050                     **primitive_params
2051                 )
2052                 while time() - start_primitive_time < self.timeout_primitive:
2053                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2054                     if primitive_result_ in ("completed", "failed"):
2055                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2056                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2057                         break
2058                     elif primitive_result_ is None and primitive == "config":
2059                         primitive_result = "COMPLETED"
2060                         detailed_result = None
2061                         break
2062                     else:  # ("running", "pending", None):
2063                         pass
2064                     await asyncio.sleep(5)
2065                 else:
2066                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2067                 if primitive_result == "COMPLETED":
2068                     break
2069                 retries -= 1
2070                 if retries >= 0:
2071                     await asyncio.sleep(retries_interval)
2072
2073             return primitive_result, detailed_result
2074         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2075             return "FAILED", str(e)
2076
2077     async def action(self, nsr_id, nslcmop_id):
2078
2079         # Try to lock HA task here
2080         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2081         if not task_is_locked_by_me:
2082             return
2083
2084         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2085         self.logger.debug(logging_text + "Enter")
2086         # get all needed from database
2087         db_nsr = None
2088         db_nslcmop = None
2089         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2090         db_nslcmop_update = {}
2091         nslcmop_operation_state = None
2092         nslcmop_operation_state_detail = None
2093         exc = None
2094         try:
2095             # wait for any previous tasks in process
2096             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2097
2098             step = "Getting information from database"
2099             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2100             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2101
2102             nsr_deployed = db_nsr["_admin"].get("deployed")
2103             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2104             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2105             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2106             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2107
2108             if vnf_index:
2109                 step = "Getting vnfr from database"
2110                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2111                 step = "Getting vnfd from database"
2112                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2113             else:
2114                 if db_nsr.get("nsd"):
2115                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2116                 else:
2117                     step = "Getting nsd from database"
2118                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2119
2120             # for backward compatibility
2121             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2122                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2123                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2124                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2125
2126             primitive = db_nslcmop["operationParams"]["primitive"]
2127             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2128
2129             # look for primitive
2130             config_primitive_desc = None
2131             if vdu_id:
2132                 for vdu in get_iterable(db_vnfd, "vdu"):
2133                     if vdu_id == vdu["id"]:
2134                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2135                             if config_primitive["name"] == primitive:
2136                                 config_primitive_desc = config_primitive
2137                                 break
2138             elif vnf_index:
2139                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2140                     if config_primitive["name"] == primitive:
2141                         config_primitive_desc = config_primitive
2142                         break
2143             else:
2144                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2145                     if config_primitive["name"] == primitive:
2146                         config_primitive_desc = config_primitive
2147                         break
2148
2149             if not config_primitive_desc:
2150                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2151                                    format(primitive))
2152
2153             desc_params = {}
2154             if vnf_index:
2155                 if db_vnfr.get("additionalParamsForVnf"):
2156                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2157             else:
2158                 if db_nsr.get("additionalParamsForVnf"):
2159                     desc_params.update(db_nsr["additionalParamsForNs"])
2160
2161             # TODO check if ns is in a proper status
2162             result, result_detail = await self._ns_execute_primitive(
2163                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2164                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2165             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2166             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2167             db_nslcmop_update["statusEnteredTime"] = time()
2168             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2169             return  # database update is called inside finally
2170
2171         except (DbException, LcmException) as e:
2172             self.logger.error(logging_text + "Exit Exception {}".format(e))
2173             exc = e
2174         except asyncio.CancelledError:
2175             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2176             exc = "Operation was cancelled"
2177         except Exception as e:
2178             exc = traceback.format_exc()
2179             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2180         finally:
2181             if exc and db_nslcmop:
2182                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2183                     "FAILED {}: {}".format(step, exc)
2184                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2185                 db_nslcmop_update["statusEnteredTime"] = time()
2186             try:
2187                 if db_nslcmop_update:
2188                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2189                 if db_nsr:
2190                     db_nsr_update["_admin.nslcmop"] = None
2191                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2192             except DbException as e:
2193                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2194             self.logger.debug(logging_text + "Exit")
2195             if nslcmop_operation_state:
2196                 try:
2197                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2198                                                                "operationState": nslcmop_operation_state},
2199                                             loop=self.loop)
2200                 except Exception as e:
2201                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2202             self.logger.debug(logging_text + "Exit")
2203             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2204             return nslcmop_operation_state, nslcmop_operation_state_detail
2205
2206     async def scale(self, nsr_id, nslcmop_id):
2207
2208         # Try to lock HA task here
2209         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2210         if not task_is_locked_by_me:
2211             return
2212
2213         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2214         self.logger.debug(logging_text + "Enter")
2215         # get all needed from database
2216         db_nsr = None
2217         db_nslcmop = None
2218         db_nslcmop_update = {}
2219         nslcmop_operation_state = None
2220         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2221         exc = None
2222         # in case of error, indicates what part of scale was failed to put nsr at error status
2223         scale_process = None
2224         old_operational_status = ""
2225         old_config_status = ""
2226         vnfr_scaled = False
2227         try:
2228             # wait for any previous tasks in process
2229             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2230
2231             step = "Getting nslcmop from database"
2232             self.logger.debug(step + " after having waited for previous tasks to be completed")
2233             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2234             step = "Getting nsr from database"
2235             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2236
2237             old_operational_status = db_nsr["operational-status"]
2238             old_config_status = db_nsr["config-status"]
2239             step = "Parsing scaling parameters"
2240             # self.logger.debug(step)
2241             db_nsr_update["operational-status"] = "scaling"
2242             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2243             nsr_deployed = db_nsr["_admin"].get("deployed")
2244             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2245             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2246             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2247             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2248             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2249
2250             # for backward compatibility
2251             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2252                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2253                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2254                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2255
2256             step = "Getting vnfr from database"
2257             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2258             step = "Getting vnfd from database"
2259             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2260
2261             step = "Getting scaling-group-descriptor"
2262             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2263                 if scaling_descriptor["name"] == scaling_group:
2264                     break
2265             else:
2266                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2267                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2268
2269             # cooldown_time = 0
2270             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2271             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2272             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2273             #         break
2274
2275             # TODO check if ns is in a proper status
2276             step = "Sending scale order to VIM"
2277             nb_scale_op = 0
2278             if not db_nsr["_admin"].get("scaling-group"):
2279                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2280                 admin_scale_index = 0
2281             else:
2282                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2283                     if admin_scale_info["name"] == scaling_group:
2284                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2285                         break
2286                 else:  # not found, set index one plus last element and add new entry with the name
2287                     admin_scale_index += 1
2288                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2289             RO_scaling_info = []
2290             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2291             if scaling_type == "SCALE_OUT":
2292                 # count if max-instance-count is reached
2293                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2294                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2295                 if nb_scale_op >= max_instance_count:
2296                     raise LcmException("reached the limit of {} (max-instance-count) "
2297                                        "scaling-out operations for the "
2298                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2299
2300                 nb_scale_op += 1
2301                 vdu_scaling_info["scaling_direction"] = "OUT"
2302                 vdu_scaling_info["vdu-create"] = {}
2303                 for vdu_scale_info in scaling_descriptor["vdu"]:
2304                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2305                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2306                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2307
2308             elif scaling_type == "SCALE_IN":
2309                 # count if min-instance-count is reached
2310                 min_instance_count = 0
2311                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2312                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2313                 if nb_scale_op <= min_instance_count:
2314                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2315                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2316                 nb_scale_op -= 1
2317                 vdu_scaling_info["scaling_direction"] = "IN"
2318                 vdu_scaling_info["vdu-delete"] = {}
2319                 for vdu_scale_info in scaling_descriptor["vdu"]:
2320                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2321                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2322                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2323
2324             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2325             vdu_create = vdu_scaling_info.get("vdu-create")
2326             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2327             if vdu_scaling_info["scaling_direction"] == "IN":
2328                 for vdur in reversed(db_vnfr["vdur"]):
2329                     if vdu_delete.get(vdur["vdu-id-ref"]):
2330                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2331                         vdu_scaling_info["vdu"].append({
2332                             "name": vdur["name"],
2333                             "vdu_id": vdur["vdu-id-ref"],
2334                             "interface": []
2335                         })
2336                         for interface in vdur["interfaces"]:
2337                             vdu_scaling_info["vdu"][-1]["interface"].append({
2338                                 "name": interface["name"],
2339                                 "ip_address": interface["ip-address"],
2340                                 "mac_address": interface.get("mac-address"),
2341                             })
2342                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2343
2344             # execute primitive service PRE-SCALING
2345             step = "Executing pre-scale vnf-config-primitive"
2346             if scaling_descriptor.get("scaling-config-action"):
2347                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2348                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2349                             and scaling_type == "SCALE_IN":
2350                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2351                         step = db_nslcmop_update["detailed-status"] = \
2352                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2353
2354                         # look for primitive
2355                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2356                             if config_primitive["name"] == vnf_config_primitive:
2357                                 break
2358                         else:
2359                             raise LcmException(
2360                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2361                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2362                                 "primitive".format(scaling_group, config_primitive))
2363
2364                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2365                         if db_vnfr.get("additionalParamsForVnf"):
2366                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2367
2368                         scale_process = "VCA"
2369                         db_nsr_update["config-status"] = "configuring pre-scaling"
2370                         result, result_detail = await self._ns_execute_primitive(
2371                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2372                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2373                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2374                             vnf_config_primitive, result, result_detail))
2375                         if result == "FAILED":
2376                             raise LcmException(result_detail)
2377                         db_nsr_update["config-status"] = old_config_status
2378                         scale_process = None
2379
2380             if RO_scaling_info:
2381                 scale_process = "RO"
2382                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2383                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2384                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2385                 # wait until ready
2386                 RO_nslcmop_id = RO_desc["instance_action_id"]
2387                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2388
2389                 RO_task_done = False
2390                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2391                 detailed_status_old = None
2392                 self.logger.debug(logging_text + step)
2393
2394                 deployment_timeout = 1 * 3600   # One hour
2395                 while deployment_timeout > 0:
2396                     if not RO_task_done:
2397                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2398                                                   extra_item_id=RO_nslcmop_id)
2399                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2400                         if ns_status == "ERROR":
2401                             raise ROclient.ROClientException(ns_status_info)
2402                         elif ns_status == "BUILD":
2403                             detailed_status = step + "; {}".format(ns_status_info)
2404                         elif ns_status == "ACTIVE":
2405                             RO_task_done = True
2406                             step = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
2407                             self.logger.debug(logging_text + step)
2408                         else:
2409                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2410                     else:
2411                         desc = await self.RO.show("ns", RO_nsr_id)
2412                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2413                         if ns_status == "ERROR":
2414                             raise ROclient.ROClientException(ns_status_info)
2415                         elif ns_status == "BUILD":
2416                             detailed_status = step + "; {}".format(ns_status_info)
2417                         elif ns_status == "ACTIVE":
2418                             step = detailed_status = \
2419                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2420                             if not vnfr_scaled:
2421                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2422                                 vnfr_scaled = True
2423                             try:
2424                                 desc = await self.RO.show("ns", RO_nsr_id)
2425                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2426                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2427                                 break
2428                             except LcmExceptionNoMgmtIP:
2429                                 pass
2430                         else:
2431                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2432                     if detailed_status != detailed_status_old:
2433                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2434                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2435
2436                     await asyncio.sleep(5, loop=self.loop)
2437                     deployment_timeout -= 5
2438                 if deployment_timeout <= 0:
2439                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2440
2441                 # update VDU_SCALING_INFO with the obtained ip_addresses
2442                 if vdu_scaling_info["scaling_direction"] == "OUT":
2443                     for vdur in reversed(db_vnfr["vdur"]):
2444                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2445                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2446                             vdu_scaling_info["vdu"].append({
2447                                 "name": vdur["name"],
2448                                 "vdu_id": vdur["vdu-id-ref"],
2449                                 "interface": []
2450                             })
2451                             for interface in vdur["interfaces"]:
2452                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2453                                     "name": interface["name"],
2454                                     "ip_address": interface["ip-address"],
2455                                     "mac_address": interface.get("mac-address"),
2456                                 })
2457                     del vdu_scaling_info["vdu-create"]
2458
2459             scale_process = None
2460             if db_nsr_update:
2461                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2462
2463             # execute primitive service POST-SCALING
2464             step = "Executing post-scale vnf-config-primitive"
2465             if scaling_descriptor.get("scaling-config-action"):
2466                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2467                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2468                             and scaling_type == "SCALE_OUT":
2469                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2470                         step = db_nslcmop_update["detailed-status"] = \
2471                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2472
2473                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2474                         if db_vnfr.get("additionalParamsForVnf"):
2475                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2476
2477                         # look for primitive
2478                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2479                             if config_primitive["name"] == vnf_config_primitive:
2480                                 break
2481                         else:
2482                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2483                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2484                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2485                                                                                                      config_primitive))
2486                         scale_process = "VCA"
2487                         db_nsr_update["config-status"] = "configuring post-scaling"
2488
2489                         result, result_detail = await self._ns_execute_primitive(
2490                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2491                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2492                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2493                             vnf_config_primitive, result, result_detail))
2494                         if result == "FAILED":
2495                             raise LcmException(result_detail)
2496                         db_nsr_update["config-status"] = old_config_status
2497                         scale_process = None
2498
2499             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2500             db_nslcmop_update["statusEnteredTime"] = time()
2501             db_nslcmop_update["detailed-status"] = "done"
2502             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2503             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2504                 else old_operational_status
2505             db_nsr_update["config-status"] = old_config_status
2506             return
2507         except (ROclient.ROClientException, DbException, LcmException) as e:
2508             self.logger.error(logging_text + "Exit Exception {}".format(e))
2509             exc = e
2510         except asyncio.CancelledError:
2511             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2512             exc = "Operation was cancelled"
2513         except Exception as e:
2514             exc = traceback.format_exc()
2515             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2516         finally:
2517             if exc:
2518                 if db_nslcmop:
2519                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2520                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2521                     db_nslcmop_update["statusEnteredTime"] = time()
2522                 if db_nsr:
2523                     db_nsr_update["operational-status"] = old_operational_status
2524                     db_nsr_update["config-status"] = old_config_status
2525                     db_nsr_update["detailed-status"] = ""
2526                     db_nsr_update["_admin.nslcmop"] = None
2527                     if scale_process:
2528                         if "VCA" in scale_process:
2529                             db_nsr_update["config-status"] = "failed"
2530                         if "RO" in scale_process:
2531                             db_nsr_update["operational-status"] = "failed"
2532                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2533                                                                                                      exc)
2534             try:
2535                 if db_nslcmop and db_nslcmop_update:
2536                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2537                 if db_nsr:
2538                     db_nsr_update["_admin.nslcmop"] = None
2539                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2540             except DbException as e:
2541                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2542             if nslcmop_operation_state:
2543                 try:
2544                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2545                                                              "operationState": nslcmop_operation_state},
2546                                             loop=self.loop)
2547                     # if cooldown_time:
2548                     #     await asyncio.sleep(cooldown_time)
2549                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2550                 except Exception as e:
2551                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2552             self.logger.debug(logging_text + "Exit")
2553             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")