osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Update target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 def deep_get(target_dict, key_list):
  71     """
  72     Get a value from target_dict entering in the nested keys. If keys does not exist, it returns None
  73     Example target_dict={a: {b: 5}}; key_list=[a,b] returns 5; both key_list=[a,b,c] and key_list=[f,h] return None
  74     :param target_dict: dictionary to be read
  75     :param key_list: list of keys to read from  target_dict
  76     :return: The wanted value if exist, None otherwise
  77     """
  78     for key in key_list:
  79         if not isinstance(target_dict, dict) or key not in target_dict:
  80             return None
  81         target_dict = target_dict[key]
  82     return target_dict
  83
  84
  85 class NsLcm(LcmBase):
  86     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  87     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  88     timeout_charm_delete = 10 * 60
  89     timeout_primitive = 10 * 60  # timeout for primitive execution
  90
  91     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  92         """
  93         Init, Connect to database, filesystem storage, and messaging
  94         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  95         :return: None
  96         """
  97         # logging
  98         self.logger = logging.getLogger('lcm.ns')
  99         self.loop = loop
 100         self.lcm_tasks = lcm_tasks
 101
 102         super().__init__(db, msg, fs, self.logger)
 103
 104         self.ro_config = ro_config
 105
 106         self.n2vc = N2VC(
 107             log=self.logger,
 108             server=vca_config['host'],
 109             port=vca_config['port'],
 110             user=vca_config['user'],
 111             secret=vca_config['secret'],
 112             # TODO: This should point to the base folder where charms are stored,
 113             # if there is a common one (like object storage). Otherwise, leave
 114             # it unset and pass it via DeployCharms
 115             # artifacts=vca_config[''],
 116             artifacts=None,
 117             juju_public_key=vca_config.get('pubkey'),
 118             ca_cert=vca_config.get('cacert'),
 119             api_proxy=vca_config.get('apiproxy')
 120         )
 121         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 122
 123     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 124         """
 125         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 126         :param vnfd: input vnfd
 127         :param new_id: overrides vnf id if provided
 128         :param additionalParams: Instantiation params for VNFs provided
 129         :param nsrId: Id of the NSR
 130         :return: copy of vnfd
 131         """
 132         try:
 133             vnfd_RO = deepcopy(vnfd)
 134             # remove unused by RO configuration, monitoring, scaling and internal keys
 135             vnfd_RO.pop("_id", None)
 136             vnfd_RO.pop("_admin", None)
 137             vnfd_RO.pop("vnf-configuration", None)
 138             vnfd_RO.pop("monitoring-param", None)
 139             vnfd_RO.pop("scaling-group-descriptor", None)
 140             if new_id:
 141                 vnfd_RO["id"] = new_id
 142
 143             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 144             for vdu in get_iterable(vnfd_RO, "vdu"):
 145                 cloud_init_file = None
 146                 if vdu.get("cloud-init-file"):
 147                     base_folder = vnfd["_admin"]["storage"]
 148                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 149                                                                    vdu["cloud-init-file"])
 150                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 151                         cloud_init_content = ci_file.read()
 152                     vdu.pop("cloud-init-file", None)
 153                 elif vdu.get("cloud-init"):
 154                     cloud_init_content = vdu["cloud-init"]
 155                 else:
 156                     continue
 157
 158                 env = Environment()
 159                 ast = env.parse(cloud_init_content)
 160                 mandatory_vars = meta.find_undeclared_variables(ast)
 161                 if mandatory_vars:
 162                     for var in mandatory_vars:
 163                         if not additionalParams or var not in additionalParams.keys():
 164                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 165                                                "file, must be provided in the instantiation parameters inside the "
 166                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 167                 template = Template(cloud_init_content)
 168                 cloud_init_content = template.render(additionalParams or {})
 169                 vdu["cloud-init"] = cloud_init_content
 170
 171             return vnfd_RO
 172         except FsException as e:
 173             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 174                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 175         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 176             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 177                                format(vnfd["id"], vdu["id"], e))
 178
 179     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 180         """
 181         Callback both for charm status change and task completion
 182         :param model_name: Charm model name
 183         :param application_name: Charm application name
 184         :param status: Can be
 185             - blocked: The unit needs manual intervention
 186             - maintenance: The unit is actively deploying/configuring
 187             - waiting: The unit is waiting for another charm to be ready
 188             - active: The unit is deployed, configured, and ready
 189             - error: The charm has failed and needs attention.
 190             - terminated: The charm has been destroyed
 191             - removing,
 192             - removed
 193         :param message: detailed message error
 194         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 195             nsr_id:
 196             nslcmop_id:
 197             lcmOperationType: currently "instantiate"
 198             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 199             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 200             n2vc_event: event used to notify instantiation task that some change has been produced
 201         :param task: None for charm status change, or task for completion task callback
 202         :return:
 203         """
 204         try:
 205             nsr_id = n2vc_info["nsr_id"]
 206             deployed = n2vc_info["deployed"]
 207             db_nsr_update = n2vc_info["db_update"]
 208             nslcmop_id = n2vc_info["nslcmop_id"]
 209             ns_operation = n2vc_info["lcmOperationType"]
 210             n2vc_event = n2vc_info["n2vc_event"]
 211             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 212                                                                                     application_name)
 213             for vca_index, vca_deployed in enumerate(deployed):
 214                 if not vca_deployed:
 215                     continue
 216                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 217                     break
 218             else:
 219                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 220                                   format(model_name))
 221                 return
 222             if task:
 223                 if task.cancelled():
 224                     self.logger.debug(logging_text + " task Cancelled")
 225                     vca_deployed['operational-status'] = "error"
 226                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 227                     vca_deployed['detailed-status'] = "Task Cancelled"
 228                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 229
 230                 elif task.done():
 231                     exc = task.exception()
 232                     if exc:
 233                         self.logger.error(logging_text + " task Exception={}".format(exc))
 234                         vca_deployed['operational-status'] = "error"
 235                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 236                         vca_deployed['detailed-status'] = str(exc)
 237                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 238                     else:
 239                         self.logger.debug(logging_text + " task Done")
 240                         # task is Done, but callback is still ongoing. So ignore
 241                         return
 242             elif status:
 243                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 244                 if vca_deployed['operational-status'] == status:
 245                     return  # same status, ignore
 246                 vca_deployed['operational-status'] = status
 247                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 248                 vca_deployed['detailed-status'] = str(message)
 249                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 250             else:
 251                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 252                 return
 253             # wake up instantiate task
 254             n2vc_event.set()
 255         except Exception as e:
 256             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 257
 258     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 259         """
 260         Creates a RO ns descriptor from OSM ns_instantiate params
 261         :param ns_params: OSM instantiate params
 262         :return: The RO ns descriptor
 263         """
 264         vim_2_RO = {}
 265         wim_2_RO = {}
 266         # TODO feature 1417: Check that no instantiation is set over PDU
 267         # check if PDU forces a concrete vim-network-id and add it
 268         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 269
 270         def vim_account_2_RO(vim_account):
 271             if vim_account in vim_2_RO:
 272                 return vim_2_RO[vim_account]
 273
 274             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 275             if db_vim["_admin"]["operationalState"] != "ENABLED":
 276                 raise LcmException("VIM={} is not available. operationalState={}".format(
 277                     vim_account, db_vim["_admin"]["operationalState"]))
 278             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 279             vim_2_RO[vim_account] = RO_vim_id
 280             return RO_vim_id
 281
 282         def wim_account_2_RO(wim_account):
 283             if isinstance(wim_account, str):
 284                 if wim_account in wim_2_RO:
 285                     return wim_2_RO[wim_account]
 286
 287                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 288                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 289                     raise LcmException("WIM={} is not available. operationalState={}".format(
 290                         wim_account, db_wim["_admin"]["operationalState"]))
 291                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 292                 wim_2_RO[wim_account] = RO_wim_id
 293                 return RO_wim_id
 294             else:
 295                 return wim_account
 296
 297         def ip_profile_2_RO(ip_profile):
 298             RO_ip_profile = deepcopy((ip_profile))
 299             if "dns-server" in RO_ip_profile:
 300                 if isinstance(RO_ip_profile["dns-server"], list):
 301                     RO_ip_profile["dns-address"] = []
 302                     for ds in RO_ip_profile.pop("dns-server"):
 303                         RO_ip_profile["dns-address"].append(ds['address'])
 304                 else:
 305                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 306             if RO_ip_profile.get("ip-version") == "ipv4":
 307                 RO_ip_profile["ip-version"] = "IPv4"
 308             if RO_ip_profile.get("ip-version") == "ipv6":
 309                 RO_ip_profile["ip-version"] = "IPv6"
 310             if "dhcp-params" in RO_ip_profile:
 311                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 312             return RO_ip_profile
 313
 314         if not ns_params:
 315             return None
 316         RO_ns_params = {
 317             # "name": ns_params["nsName"],
 318             # "description": ns_params.get("nsDescription"),
 319             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 320             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 321             # "scenario": ns_params["nsdId"],
 322         }
 323         n2vc_key_list = n2vc_key_list or []
 324         for vnfd_ref, vnfd in vnfd_dict.items():
 325             vdu_needed_access = []
 326             mgmt_cp = None
 327             if vnfd.get("vnf-configuration"):
 328                 ssh_required = deep_get(vnfd, ("vnf-configuration", "config-access", "ssh-access", "required"))
 329                 if ssh_required and vnfd.get("mgmt-interface"):
 330                     if vnfd["mgmt-interface"].get("vdu-id"):
 331                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 332                     elif vnfd["mgmt-interface"].get("cp"):
 333                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 334
 335             for vdu in vnfd.get("vdu", ()):
 336                 if vdu.get("vdu-configuration"):
 337                     ssh_required = deep_get(vdu, ("vdu-configuration", "config-access", "ssh-access", "required"))
 338                     if ssh_required:
 339                         vdu_needed_access.append(vdu["id"])
 340                 elif mgmt_cp:
 341                     for vdu_interface in vdu.get("interface"):
 342                         if vdu_interface.get("external-connection-point-ref") and \
 343                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 344                             vdu_needed_access.append(vdu["id"])
 345                             mgmt_cp = None
 346                             break
 347
 348             if vdu_needed_access:
 349                 for vnf_member in nsd.get("constituent-vnfd"):
 350                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 351                         continue
 352                     for vdu in vdu_needed_access:
 353                         populate_dict(RO_ns_params,
 354                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 355                                       n2vc_key_list)
 356
 357         if ns_params.get("vduImage"):
 358             RO_ns_params["vduImage"] = ns_params["vduImage"]
 359
 360         if ns_params.get("ssh_keys"):
 361             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 362         for vnf_params in get_iterable(ns_params, "vnf"):
 363             for constituent_vnfd in nsd["constituent-vnfd"]:
 364                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 365                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 366                     break
 367             else:
 368                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 369                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 370             if vnf_params.get("vimAccountId"):
 371                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 372                               vim_account_2_RO(vnf_params["vimAccountId"]))
 373
 374             for vdu_params in get_iterable(vnf_params, "vdu"):
 375                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 376                 if vdu_params.get("volume"):
 377                     for volume_params in vdu_params["volume"]:
 378                         if volume_params.get("vim-volume-id"):
 379                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 380                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 381                                           volume_params["vim-volume-id"])
 382                 if vdu_params.get("interface"):
 383                     for interface_params in vdu_params["interface"]:
 384                         if interface_params.get("ip-address"):
 385                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 386                                                          vdu_params["id"], "interfaces", interface_params["name"],
 387                                                          "ip_address"),
 388                                           interface_params["ip-address"])
 389                         if interface_params.get("mac-address"):
 390                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 391                                                          vdu_params["id"], "interfaces", interface_params["name"],
 392                                                          "mac_address"),
 393                                           interface_params["mac-address"])
 394                         if interface_params.get("floating-ip-required"):
 395                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 396                                                          vdu_params["id"], "interfaces", interface_params["name"],
 397                                                          "floating-ip"),
 398                                           interface_params["floating-ip-required"])
 399
 400             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 401                 if internal_vld_params.get("vim-network-name"):
 402                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 403                                                  internal_vld_params["name"], "vim-network-name"),
 404                                   internal_vld_params["vim-network-name"])
 405                 if internal_vld_params.get("vim-network-id"):
 406                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 407                                                  internal_vld_params["name"], "vim-network-id"),
 408                                   internal_vld_params["vim-network-id"])
 409                 if internal_vld_params.get("ip-profile"):
 410                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 411                                                  internal_vld_params["name"], "ip-profile"),
 412                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 413
 414                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 415                     # look for interface
 416                     iface_found = False
 417                     for vdu_descriptor in vnf_descriptor["vdu"]:
 418                         for vdu_interface in vdu_descriptor["interface"]:
 419                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 420                                 if icp_params.get("ip-address"):
 421                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 422                                                                  vdu_descriptor["id"], "interfaces",
 423                                                                  vdu_interface["name"], "ip_address"),
 424                                                   icp_params["ip-address"])
 425
 426                                 if icp_params.get("mac-address"):
 427                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 428                                                                  vdu_descriptor["id"], "interfaces",
 429                                                                  vdu_interface["name"], "mac_address"),
 430                                                   icp_params["mac-address"])
 431                                 iface_found = True
 432                                 break
 433                         if iface_found:
 434                             break
 435                     else:
 436                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 437                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 438                                            "connection-point".format(vnf_params["member-vnf-index"],
 439                                                                      icp_params["id-ref"]))
 440
 441         for vld_params in get_iterable(ns_params, "vld"):
 442             if "ip-profile" in vld_params:
 443                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 444                               ip_profile_2_RO(vld_params["ip-profile"]))
 445
 446             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 447                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 448                               wim_account_2_RO(vld_params["wimAccountId"])),
 449             if vld_params.get("vim-network-name"):
 450                 RO_vld_sites = []
 451                 if isinstance(vld_params["vim-network-name"], dict):
 452                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 453                         RO_vld_sites.append({
 454                             "netmap-use": vim_net,
 455                             "datacenter": vim_account_2_RO(vim_account)
 456                         })
 457                 else:  # isinstance str
 458                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 459                 if RO_vld_sites:
 460                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 461             if vld_params.get("vim-network-id"):
 462                 RO_vld_sites = []
 463                 if isinstance(vld_params["vim-network-id"], dict):
 464                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 465                         RO_vld_sites.append({
 466                             "netmap-use": vim_net,
 467                             "datacenter": vim_account_2_RO(vim_account)
 468                         })
 469                 else:  # isinstance str
 470                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 471                 if RO_vld_sites:
 472                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 473             if vld_params.get("ns-net"):
 474                 if isinstance(vld_params["ns-net"], dict):
 475                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 476                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 477                 if RO_vld_ns_net:
 478                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 479             if "vnfd-connection-point-ref" in vld_params:
 480                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 481                     # look for interface
 482                     for constituent_vnfd in nsd["constituent-vnfd"]:
 483                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 484                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 485                             break
 486                     else:
 487                         raise LcmException(
 488                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 489                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 490                     match_cp = False
 491                     for vdu_descriptor in vnf_descriptor["vdu"]:
 492                         for interface_descriptor in vdu_descriptor["interface"]:
 493                             if interface_descriptor.get("external-connection-point-ref") == \
 494                                     cp_params["vnfd-connection-point-ref"]:
 495                                 match_cp = True
 496                                 break
 497                         if match_cp:
 498                             break
 499                     else:
 500                         raise LcmException(
 501                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 502                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 503                                 cp_params["member-vnf-index-ref"],
 504                                 cp_params["vnfd-connection-point-ref"],
 505                                 vnf_descriptor["id"]))
 506                     if cp_params.get("ip-address"):
 507                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 508                                                      vdu_descriptor["id"], "interfaces",
 509                                                      interface_descriptor["name"], "ip_address"),
 510                                       cp_params["ip-address"])
 511                     if cp_params.get("mac-address"):
 512                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 513                                                      vdu_descriptor["id"], "interfaces",
 514                                                      interface_descriptor["name"], "mac_address"),
 515                                       cp_params["mac-address"])
 516         return RO_ns_params
 517
 518     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 519         # make a copy to do not change
 520         vdu_create = copy(vdu_create)
 521         vdu_delete = copy(vdu_delete)
 522
 523         vdurs = db_vnfr.get("vdur")
 524         if vdurs is None:
 525             vdurs = []
 526         vdu_index = len(vdurs)
 527         while vdu_index:
 528             vdu_index -= 1
 529             vdur = vdurs[vdu_index]
 530             if vdur.get("pdu-type"):
 531                 continue
 532             vdu_id_ref = vdur["vdu-id-ref"]
 533             if vdu_create and vdu_create.get(vdu_id_ref):
 534                 for index in range(0, vdu_create[vdu_id_ref]):
 535                     vdur = deepcopy(vdur)
 536                     vdur["_id"] = str(uuid4())
 537                     vdur["count-index"] += 1
 538                     vdurs.insert(vdu_index+1+index, vdur)
 539                 del vdu_create[vdu_id_ref]
 540             if vdu_delete and vdu_delete.get(vdu_id_ref):
 541                 del vdurs[vdu_index]
 542                 vdu_delete[vdu_id_ref] -= 1
 543                 if not vdu_delete[vdu_id_ref]:
 544                     del vdu_delete[vdu_id_ref]
 545         # check all operations are done
 546         if vdu_create or vdu_delete:
 547             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 548                 vdu_create))
 549         if vdu_delete:
 550             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 551                 vdu_delete))
 552
 553         vnfr_update = {"vdur": vdurs}
 554         db_vnfr["vdur"] = vdurs
 555         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 556
 557     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 558         """
 559         Updates database nsr with the RO info for the created vld
 560         :param ns_update_nsr: dictionary to be filled with the updated info
 561         :param db_nsr: content of db_nsr. This is also modified
 562         :param nsr_desc_RO: nsr descriptor from RO
 563         :return: Nothing, LcmException is raised on errors
 564         """
 565
 566         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 567             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 568                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 569                     continue
 570                 vld["vim-id"] = net_RO.get("vim_net_id")
 571                 vld["name"] = net_RO.get("vim_name")
 572                 vld["status"] = net_RO.get("status")
 573                 vld["status-detailed"] = net_RO.get("error_msg")
 574                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 575                 break
 576             else:
 577                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 578
 579     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 580         """
 581         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 582         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 583         :param nsr_desc_RO: nsr descriptor from RO
 584         :return: Nothing, LcmException is raised on errors
 585         """
 586         for vnf_index, db_vnfr in db_vnfrs.items():
 587             for vnf_RO in nsr_desc_RO["vnfs"]:
 588                 if vnf_RO["member_vnf_index"] != vnf_index:
 589                     continue
 590                 vnfr_update = {}
 591                 if vnf_RO.get("ip_address"):
 592                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 593                 elif not db_vnfr.get("ip-address"):
 594                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 595
 596                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 597                     vdur_RO_count_index = 0
 598                     if vdur.get("pdu-type"):
 599                         continue
 600                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 601                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 602                             continue
 603                         if vdur["count-index"] != vdur_RO_count_index:
 604                             vdur_RO_count_index += 1
 605                             continue
 606                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 607                         if vdur_RO.get("ip_address"):
 608                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 609                         else:
 610                             vdur["ip-address"] = None
 611                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 612                         vdur["name"] = vdur_RO.get("vim_name")
 613                         vdur["status"] = vdur_RO.get("status")
 614                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 615                         for ifacer in get_iterable(vdur, "interfaces"):
 616                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 617                                 if ifacer["name"] == interface_RO.get("internal_name"):
 618                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 619                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 620                                     break
 621                             else:
 622                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 623                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 624                                                                           ifacer["name"]))
 625                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 626                         break
 627                     else:
 628                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 629                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 630
 631                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 632                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 633                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 634                             continue
 635                         vld["vim-id"] = net_RO.get("vim_net_id")
 636                         vld["name"] = net_RO.get("vim_name")
 637                         vld["status"] = net_RO.get("status")
 638                         vld["status-detailed"] = net_RO.get("error_msg")
 639                         vnfr_update["vld.{}".format(vld_index)] = vld
 640                         break
 641                     else:
 642                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 643                             vnf_index, vld["id"]))
 644
 645                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 646                 break
 647
 648             else:
 649                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 650
 651     async def instantiate(self, nsr_id, nslcmop_id):
 652
 653         # Try to lock HA task here
 654         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 655         if not task_is_locked_by_me:
 656             return
 657
 658         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 659         self.logger.debug(logging_text + "Enter")
 660         # get all needed from database
 661         start_deploy = time()
 662         db_nsr = None
 663         db_nslcmop = None
 664         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 665         db_nslcmop_update = {}
 666         nslcmop_operation_state = None
 667         db_vnfrs = {}
 668         RO_descriptor_number = 0   # number of descriptors created at RO
 669         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 670         n2vc_info = {}
 671         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 672         exc = None
 673         try:
 674             # wait for any previous tasks in process
 675             step = "Waiting for previous tasks"
 676             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 677
 678             step = "Getting nslcmop={} from db".format(nslcmop_id)
 679             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 680             step = "Getting nsr={} from db".format(nsr_id)
 681             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 682             ns_params = db_nslcmop.get("operationParams")
 683             nsd = db_nsr["nsd"]
 684             nsr_name = db_nsr["name"]   # TODO short-name??
 685
 686             step = "Getting vnfrs from db"
 687             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 688             db_vnfds_ref = {}
 689             db_vnfds = {}
 690             db_vnfds_index = {}
 691             for vnfr in db_vnfrs_list:
 692                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 693                 vnfd_id = vnfr["vnfd-id"]
 694                 vnfd_ref = vnfr["vnfd-ref"]
 695                 if vnfd_id not in db_vnfds:
 696                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 697                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 698                     db_vnfds_ref[vnfd_ref] = vnfd
 699                     db_vnfds[vnfd_id] = vnfd
 700                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 701
 702             # Get or generates the _admin.deployed,VCA list
 703             vca_deployed_list = None
 704             vca_model_name = None
 705             if db_nsr["_admin"].get("deployed"):
 706                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 707                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 708             if vca_deployed_list is None:
 709                 vca_deployed_list = []
 710                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 711                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 712             elif isinstance(vca_deployed_list, dict):
 713                 # maintain backward compatibility. Change a dict to list at database
 714                 vca_deployed_list = list(vca_deployed_list.values())
 715                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 716                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 717
 718             db_nsr_update["detailed-status"] = "creating"
 719             db_nsr_update["operational-status"] = "init"
 720             if not isinstance(deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list):
 721                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 722                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 723
 724             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 725             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 726             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 727
 728             # Deploy charms
 729             # The parameters we'll need to deploy a charm
 730             number_to_configure = 0
 731
 732             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 733                 """An inner function to deploy the charm from either ns, vnf or vdu
 734                 For ns both vnf_index and vdu_id are None.
 735                 For vnf only vdu_id is None
 736                 For vdu both vnf_index and vdu_id contain a value
 737                 """
 738                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 739                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 740
 741                 machine_spec = {}
 742                 if native_charm:
 743                     machine_spec["username"] = charm_params.get("username")
 744                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 745
 746                 # Note: The charm needs to exist on disk at the location
 747                 # specified by charm_path.
 748                 descriptor = vnfd if vnf_index else nsd
 749                 base_folder = descriptor["_admin"]["storage"]
 750                 storage_params = self.fs.get_params()
 751                 charm_path = "{}{}/{}/charms/{}".format(
 752                     storage_params["path"],
 753                     base_folder["folder"],
 754                     base_folder["pkg-dir"],
 755                     proxy_charm
 756                 )
 757
 758                 # ns_name will be ignored in the current version of N2VC
 759                 # but will be implemented for the next point release.
 760                 model_name = nsr_id
 761                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 762                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 763                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 764
 765                 vca_index = len(vca_deployed_list)
 766                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 767                 # 26*26 charm in the same NS
 768                 application_name = application_name[0:48]
 769                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 770                 vca_deployed_ = {
 771                     "member-vnf-index": vnf_index,
 772                     "vdu_id": vdu_id,
 773                     "model": model_name,
 774                     "application": application_name,
 775                     "operational-status": "init",
 776                     "detailed-status": "",
 777                     "step": "initial-deploy",
 778                     "vnfd_id": vnfd_id,
 779                     "vdu_name": vdu_name,
 780                     "vdu_count_index": vdu_count_index,
 781                 }
 782                 vca_deployed_list.append(vca_deployed_)
 783                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 784                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 785
 786                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 787                                                                                                 proxy_charm))
 788                 if not n2vc_info:
 789                     n2vc_info["nsr_id"] = nsr_id
 790                     n2vc_info["nslcmop_id"] = nslcmop_id
 791                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 792                     n2vc_info["lcmOperationType"] = "instantiate"
 793                     n2vc_info["deployed"] = vca_deployed_list
 794                     n2vc_info["db_update"] = db_nsr_update
 795                 task = asyncio.ensure_future(
 796                     self.n2vc.DeployCharms(
 797                         model_name,          # The network service name
 798                         application_name,    # The application name
 799                         descriptor,          # The vnf/nsd descriptor
 800                         charm_path,          # Path to charm
 801                         charm_params,        # Runtime params, like mgmt ip
 802                         machine_spec,        # for native charms only
 803                         self.n2vc_callback,  # Callback for status changes
 804                         n2vc_info,           # Callback parameter
 805                         None,                # Callback parameter (task)
 806                     )
 807                 )
 808                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 809                                                          n2vc_info))
 810                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 811
 812             step = "Looking for needed vnfd to configure with proxy charm"
 813             self.logger.debug(logging_text + step)
 814
 815             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 816                 vnfd_id = c_vnf["vnfd-id-ref"]
 817                 vnf_index = str(c_vnf["member-vnf-index"])
 818                 vnfd = db_vnfds_ref[vnfd_id]
 819
 820                 # Get additional parameters
 821                 vnfr_params = {}
 822                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 823                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 824                 for k, v in vnfr_params.items():
 825                     if isinstance(v, str) and v.startswith("!!yaml "):
 826                         vnfr_params[k] = yaml.safe_load(v[7:])
 827
 828                 step = "deploying proxy charms for configuration"
 829                 # Check if this VNF has a charm configuration
 830                 vnf_config = vnfd.get("vnf-configuration")
 831                 if vnf_config and vnf_config.get("juju"):
 832                     proxy_charm = vnf_config["juju"]["charm"]
 833                     if vnf_config["juju"].get("proxy") is False:
 834                         # native_charm, will be deployed after VM. Skip
 835                         proxy_charm = None
 836
 837                     if proxy_charm:
 838                         if not vca_model_name:
 839                             step = "creating VCA model name '{}'".format(nsr_id)
 840                             self.logger.debug(logging_text + step)
 841                             await self.n2vc.CreateNetworkService(nsr_id)
 842                             vca_model_name = nsr_id
 843                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 844                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 845                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 846                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 847                         charm_params = {
 848                             "user_values": vnfr_params,
 849                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 850                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 851                         }
 852
 853                         # Login to the VCA. If there are multiple calls to login(),
 854                         # subsequent calls will be a nop and return immediately.
 855                         await self.n2vc.login()
 856
 857                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 858                         number_to_configure += 1
 859
 860                 # Deploy charms for each VDU that supports one.
 861                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 862                     vdu_config = vdu.get('vdu-configuration')
 863                     proxy_charm = None
 864
 865                     if vdu_config and vdu_config.get("juju"):
 866                         proxy_charm = vdu_config["juju"]["charm"]
 867                         if vdu_config["juju"].get("proxy") is False:
 868                             # native_charm, will be deployed after VM. Skip
 869                             proxy_charm = None
 870                         if proxy_charm:
 871                             if not vca_model_name:
 872                                 step = "creating VCA model name"
 873                                 await self.n2vc.CreateNetworkService(nsr_id)
 874                                 vca_model_name = nsr_id
 875                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 876                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 877                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 878                                                                                                           vdu["id"])
 879                             await self.n2vc.login()
 880                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 881                             # TODO for the moment only first vdu_id contains a charm deployed
 882                             if vdur["vdu-id-ref"] != vdu["id"]:
 883                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 884                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 885                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 886                             charm_params = {
 887                                 "user_values": vnfr_params,
 888                                 "rw_mgmt_ip": vdur["ip-address"],
 889                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 890                             }
 891                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 892                                          charm_params, n2vc_info)
 893                             number_to_configure += 1
 894
 895             # Check if this NS has a charm configuration
 896
 897             ns_config = nsd.get("ns-configuration")
 898             if ns_config and ns_config.get("juju"):
 899                 proxy_charm = ns_config["juju"]["charm"]
 900                 if ns_config["juju"].get("proxy") is False:
 901                     # native_charm, will be deployed after VM. Skip
 902                     proxy_charm = None
 903                 if proxy_charm:
 904                     step = "deploying proxy charm to configure ns"
 905                     # TODO is NS magmt IP address needed?
 906
 907                     # Get additional parameters
 908                     additional_params = {}
 909                     if db_nsr.get("additionalParamsForNs"):
 910                         additional_params = db_nsr["additionalParamsForNs"].copy()
 911                     for k, v in additional_params.items():
 912                         if isinstance(v, str) and v.startswith("!!yaml "):
 913                             additional_params[k] = yaml.safe_load(v[7:])
 914
 915                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 916                     charm_params = {
 917                         "user_values": additional_params,
 918                         # "rw_mgmt_ip": db_nsr["ip-address"],
 919                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 920                     }
 921
 922                     # Login to the VCA. If there are multiple calls to login(),
 923                     # subsequent calls will be a nop and return immediately.
 924                     await self.n2vc.login()
 925                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 926                     number_to_configure += 1
 927
 928             db_nsr_update["operational-status"] = "running"
 929
 930             # Wait until all charms has reached blocked or active status
 931             step = "waiting proxy charms to be ready"
 932             if number_to_configure:
 933                 # wait until all charms are configured.
 934                 # steps are:
 935                 #       initial-deploy
 936                 #       get-ssh-public-key
 937                 #       generate-ssh-key
 938                 #       retry-get-ssh-public-key
 939                 #       ssh-public-key-obtained
 940                 while time() <= start_deploy + self.total_deploy_timeout:
 941                     if db_nsr_update:
 942                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 943                     if db_nslcmop_update:
 944                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 945
 946                     all_active = True
 947                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 948                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 949                         if vca_deployed["step"] == "initial-deploy":
 950                             if vca_deployed["operational-status"] in ("active", "blocked"):
 951                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 952                                        .format(vca_deployed["member-vnf-index"],
 953                                                vca_deployed["vdu_id"])
 954                                 self.logger.debug(logging_text + step)
 955                                 try:
 956                                     primitive_id = await self.n2vc.ExecutePrimitive(
 957                                         vca_deployed["model"],
 958                                         vca_deployed["application"],
 959                                         "get-ssh-public-key",
 960                                         None,
 961                                     )
 962                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 963                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 964                                         primitive_id
 965                                     db_nsr_update[database_entry + "operational-status"] =\
 966                                         vca_deployed["operational-status"]
 967                                 except PrimitiveDoesNotExist:
 968                                     ssh_public_key = None
 969                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 970                                         "ssh-public-key-obtained"
 971                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 972                                         ssh_public_key
 973                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 974                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 975                                     self.logger.debug(logging_text + step)
 976
 977                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 978                             primitive_id = vca_deployed["primitive_id"]
 979                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 980                                                                                   primitive_id)
 981                             if primitive_status in ("completed", "failed"):
 982                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 983                                                                                       primitive_id)
 984                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 985                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 986                                         primitive_result.get("pubkey"):
 987                                     ssh_public_key = primitive_result.get("pubkey")
 988                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 989                                         "ssh-public-key-obtained"
 990                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 991                                         ssh_public_key
 992                                     n2vc_key_list.append(ssh_public_key)
 993                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 994                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 995                                     self.logger.debug(logging_text + step)
 996                                 else:  # primitive_status == "failed":
 997                                     if vca_deployed["step"] == "get-ssh-public-key":
 998                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 999                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
1000                                                                      vca_deployed["vdu_id"])
1001                                         self.logger.debug(logging_text + step)
1002                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
1003                                             "generate-ssh-key"
1004                                         primitive_id = await self.n2vc.ExecutePrimitive(
1005                                             vca_deployed["model"],
1006                                             vca_deployed["application"],
1007                                             "generate-ssh-key",
1008                                             None,
1009                                         )
1010                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1011                                             primitive_id
1012                                     else:  # failed for second time
1013                                         raise LcmException(
1014                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
1015
1016                         elif vca_deployed["step"] == "generate-ssh-key":
1017                             primitive_id = vca_deployed["primitive_id"]
1018                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1019                                                                                   primitive_id)
1020                             if primitive_status in ("completed", "failed"):
1021                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1022                                                                                       primitive_id)
1023                                 vca_deployed["primitive_id"] = db_nsr_update[
1024                                     database_entry + "primitive_id"] = None
1025                                 if primitive_status == "completed":
1026                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1027                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1028                                                               vca_deployed["vdu_id"])
1029                                     self.logger.debug(logging_text + step)
1030                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1031                                         "retry-get-ssh-public-key"
1032                                     primitive_id = await self.n2vc.ExecutePrimitive(
1033                                         vca_deployed["model"],
1034                                         vca_deployed["application"],
1035                                         "get-ssh-public-key",
1036                                         None,
1037                                     )
1038                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1039                                         primitive_id
1040
1041                                 else:  # primitive_status == "failed":
1042                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1043                                                        .format(primitive_result))
1044
1045                         if vca_deployed["step"] != "ssh-public-key-obtained":
1046                             all_active = False
1047
1048                     if all_active:
1049                         break
1050                     await asyncio.sleep(5)
1051                 else:   # total_deploy_timeout
1052                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1053                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1054
1055             # deploy RO
1056             # get vnfds, instantiate at RO
1057             for c_vnf in nsd.get("constituent-vnfd", ()):
1058                 member_vnf_index = c_vnf["member-vnf-index"]
1059                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1060                 vnfd_ref = vnfd["id"]
1061                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1062                     vnfd_ref, member_vnf_index)
1063                 # self.logger.debug(logging_text + step)
1064                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1065                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1066                 RO_descriptor_number += 1
1067
1068                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1069                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1070                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1071                         break
1072                 else:
1073                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1074                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1075
1076                 # look if present
1077                 RO_update = {"member-vnf-index": member_vnf_index}
1078                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1079                 if vnfd_list:
1080                     RO_update["id"] = vnfd_list[0]["uuid"]
1081                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1082                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1083                 else:
1084                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1085                                            get("additionalParamsForVnf"), nsr_id)
1086                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1087                     RO_update["id"] = desc["uuid"]
1088                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1089                         vnfd_ref, member_vnf_index, desc["uuid"]))
1090                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1091                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1092                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1093
1094             # create nsd at RO
1095             nsd_ref = nsd["id"]
1096             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1097             # self.logger.debug(logging_text + step)
1098
1099             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1100             RO_descriptor_number += 1
1101             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1102             if nsd_list:
1103                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1104                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1105                     nsd_ref, RO_nsd_uuid))
1106             else:
1107                 nsd_RO = deepcopy(nsd)
1108                 nsd_RO["id"] = RO_osm_nsd_id
1109                 nsd_RO.pop("_id", None)
1110                 nsd_RO.pop("_admin", None)
1111                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1112                     member_vnf_index = c_vnf["member-vnf-index"]
1113                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1114                 for c_vld in nsd_RO.get("vld", ()):
1115                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1116                         member_vnf_index = cp["member-vnf-index-ref"]
1117                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1118
1119                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1120                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1121                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1122                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1123             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1124
1125             # Crate ns at RO
1126             # if present use it unless in error status
1127             RO_nsr_id = deep_get(db_nsr, ("_admin", "deployed", "RO", "nsr_id"))
1128             if RO_nsr_id:
1129                 try:
1130                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1131                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1132                     desc = await self.RO.show("ns", RO_nsr_id)
1133                 except ROclient.ROClientException as e:
1134                     if e.http_code != HTTPStatus.NOT_FOUND:
1135                         raise
1136                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1137                 if RO_nsr_id:
1138                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1139                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1140                     if ns_status == "ERROR":
1141                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1142                         self.logger.debug(logging_text + step)
1143                         await self.RO.delete("ns", RO_nsr_id)
1144                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1145             if not RO_nsr_id:
1146                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1147                 # self.logger.debug(logging_text + step)
1148
1149                 # check if VIM is creating and wait  look if previous tasks in process
1150                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1151                 if task_dependency:
1152                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1153                     self.logger.debug(logging_text + step)
1154                     await asyncio.wait(task_dependency, timeout=3600)
1155                 if ns_params.get("vnf"):
1156                     for vnf in ns_params["vnf"]:
1157                         if "vimAccountId" in vnf:
1158                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1159                                                                                         vnf["vimAccountId"])
1160                         if task_dependency:
1161                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1162                             self.logger.debug(logging_text + step)
1163                             await asyncio.wait(task_dependency, timeout=3600)
1164
1165                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1166
1167                 # feature 1429. Add n2vc public key to needed VMs
1168                 n2vc_key = await self.n2vc.GetPublicKey()
1169                 n2vc_key_list.append(n2vc_key)
1170                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1171
1172                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1173                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1174                                             name=db_nsr["name"],
1175                                             scenario=RO_nsd_uuid)
1176                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1177                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1178                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1179                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1180             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1181
1182             # wait until NS is ready
1183             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1184             detailed_status_old = None
1185             self.logger.debug(logging_text + step)
1186
1187             while time() <= start_deploy + self.total_deploy_timeout:
1188                 desc = await self.RO.show("ns", RO_nsr_id)
1189                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1190                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1191                 if ns_status == "ERROR":
1192                     raise ROclient.ROClientException(ns_status_info)
1193                 elif ns_status == "BUILD":
1194                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1195                 elif ns_status == "ACTIVE":
1196                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1197                     try:
1198                         self.ns_update_vnfr(db_vnfrs, desc)
1199                         break
1200                     except LcmExceptionNoMgmtIP:
1201                         pass
1202                 else:
1203                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1204                 if detailed_status != detailed_status_old:
1205                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1206                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1207                 await asyncio.sleep(5, loop=self.loop)
1208             else:   # total_deploy_timeout
1209                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1210
1211             step = "Updating NSR"
1212             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1213
1214             db_nsr_update["operational-status"] = "running"
1215             db_nsr["detailed-status"] = "Configuring vnfr"
1216             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1217
1218             # Configure proxy charms once VMs are up
1219             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1220                 vnf_index = vca_deployed.get("member-vnf-index")
1221                 vdu_id = vca_deployed.get("vdu_id")
1222                 vdu_name = None
1223                 vdu_count_index = None
1224
1225                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1226                                                                                                            vdu_id)
1227                 add_params = {}
1228                 initial_config_primitive_list = []
1229                 if vnf_index:
1230                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1231                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1232                     vnfd = db_vnfds_index[vnf_index]
1233
1234                     if vdu_id:
1235                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1236                             if vdu["id"] == vdu_id:
1237                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1238                                     'initial-config-primitive', [])
1239                                 break
1240                         else:
1241                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1242                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1243                         # TODO for the moment only first vdu_id contains a charm deployed
1244                         if vdur["vdu-id-ref"] != vdu["id"]:
1245                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1246                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1247                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1248                     else:
1249                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1250                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1251                 else:
1252                     if db_nsr.get("additionalParamsForNs"):
1253                         add_params = db_nsr["additionalParamsForNs"].copy()
1254                     for k, v in add_params.items():
1255                         if isinstance(v, str) and v.startswith("!!yaml "):
1256                             add_params[k] = yaml.safe_load(v[7:])
1257                     add_params["rw_mgmt_ip"] = None
1258                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1259
1260                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1261                 initial_config_primitive_list = initial_config_primitive_list.copy()
1262                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1263                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1264
1265                 for initial_config_primitive in initial_config_primitive_list:
1266                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1267                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1268                                       .format(initial_config_primitive["name"], primitive_params_))
1269                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1270                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1271                         initial_config_primitive["name"],
1272                         primitive_params_,
1273                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1274                         retries_interval=30)
1275                     if primitive_result != "COMPLETED":
1276                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1277                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1278                                                    vca_deployed["vdu_id"], primitive_detail))
1279
1280             # Deploy native charms
1281             step = "Looking for needed vnfd to configure with native charm"
1282             self.logger.debug(logging_text + step)
1283
1284             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1285                 vnfd_id = c_vnf["vnfd-id-ref"]
1286                 vnf_index = str(c_vnf["member-vnf-index"])
1287                 vnfd = db_vnfds_ref[vnfd_id]
1288
1289                 # Get additional parameters
1290                 vnfr_params = {}
1291                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1292                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1293                 for k, v in vnfr_params.items():
1294                     if isinstance(v, str) and v.startswith("!!yaml "):
1295                         vnfr_params[k] = yaml.safe_load(v[7:])
1296
1297                 # Check if this VNF has a charm configuration
1298                 vnf_config = vnfd.get("vnf-configuration")
1299                 if vnf_config and vnf_config.get("juju"):
1300                     native_charm = vnf_config["juju"].get("proxy") is False
1301                     proxy_charm = vnf_config["juju"]["charm"]
1302                     if native_charm and proxy_charm:
1303                         if not vca_model_name:
1304                             step = "creating VCA model name '{}'".format(nsr_id)
1305                             self.logger.debug(logging_text + step)
1306                             await self.n2vc.CreateNetworkService(nsr_id)
1307                             vca_model_name = nsr_id
1308                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1309                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1310                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1311                         self.logger.debug(logging_text + step)
1312
1313                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1314                         charm_params = {
1315                             "user_values": vnfr_params,
1316                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1317                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1318                         }
1319
1320                         # get username
1321                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1322                         #  merged. Meanwhile let's get username from initial-config-primitive
1323                         if vnf_config.get("initial-config-primitive"):
1324                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1325                                 if param["name"] == "ssh-username":
1326                                     charm_params["username"] = param["value"]
1327                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1328                             if vnf_config["config-access"]["ssh-access"].get("required"):
1329                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1330
1331                         # Login to the VCA. If there are multiple calls to login(),
1332                         # subsequent calls will be a nop and return immediately.
1333                         await self.n2vc.login()
1334
1335                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1336                         number_to_configure += 1
1337
1338                 # Deploy charms for each VDU that supports one.
1339                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1340                     vdu_config = vdu.get('vdu-configuration')
1341                     native_charm = False
1342
1343                     if vdu_config and vdu_config.get("juju"):
1344                         native_charm = vdu_config["juju"].get("proxy") is False
1345                         proxy_charm = vdu_config["juju"]["charm"]
1346                         if native_charm and proxy_charm:
1347                             if not vca_model_name:
1348                                 step = "creating VCA model name"
1349                                 await self.n2vc.CreateNetworkService(nsr_id)
1350                                 vca_model_name = nsr_id
1351                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1352                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1353                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1354                                                                                                      vdu["id"])
1355
1356                             self.logger.debug(logging_text + step)
1357
1358                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1359
1360                             # TODO for the moment only first vdu_id contains a charm deployed
1361                             if vdur["vdu-id-ref"] != vdu["id"]:
1362                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1363                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1364                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1365                             charm_params = {
1366                                 "user_values": vnfr_params,
1367                                 "rw_mgmt_ip": vdur["ip-address"],
1368                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1369                             }
1370
1371                             # get username
1372                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1373                             #  merged. Meanwhile let's get username from initial-config-primitive
1374                             if vdu_config.get("initial-config-primitive"):
1375                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1376                                     if param["name"] == "ssh-username":
1377                                         charm_params["username"] = param["value"]
1378                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1379                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1380                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1381                                         "default-user")
1382
1383                             await self.n2vc.login()
1384
1385                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1386                                          charm_params, n2vc_info, native_charm)
1387                             number_to_configure += 1
1388
1389             # Check if this NS has a charm configuration
1390
1391             ns_config = nsd.get("ns-configuration")
1392             if ns_config and ns_config.get("juju"):
1393                 native_charm = ns_config["juju"].get("proxy") is False
1394                 proxy_charm = ns_config["juju"]["charm"]
1395                 if native_charm and proxy_charm:
1396                     step = "deploying native charm to configure ns"
1397                     # TODO is NS magmt IP address needed?
1398
1399                     # Get additional parameters
1400                     additional_params = {}
1401                     if db_nsr.get("additionalParamsForNs"):
1402                         additional_params = db_nsr["additionalParamsForNs"].copy()
1403                     for k, v in additional_params.items():
1404                         if isinstance(v, str) and v.startswith("!!yaml "):
1405                             additional_params[k] = yaml.safe_load(v[7:])
1406
1407                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1408                     charm_params = {
1409                         "user_values": additional_params,
1410                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1411                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1412                     }
1413
1414                     # get username
1415                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1416                     #  merged. Meanwhile let's get username from initial-config-primitive
1417                     if ns_config.get("initial-config-primitive"):
1418                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1419                             if param["name"] == "ssh-username":
1420                                 charm_params["username"] = param["value"]
1421                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1422                         if ns_config["config-access"]["ssh-access"].get("required"):
1423                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1424
1425                     # Login to the VCA. If there are multiple calls to login(),
1426                     # subsequent calls will be a nop and return immediately.
1427                     await self.n2vc.login()
1428                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1429                     number_to_configure += 1
1430
1431             # waiting all charms are ok
1432             configuration_failed = False
1433             if number_to_configure:
1434                 step = "Waiting all charms are active"
1435                 old_status = "configuring: init: {}".format(number_to_configure)
1436                 db_nsr_update["config-status"] = old_status
1437                 db_nsr_update["detailed-status"] = old_status
1438                 db_nslcmop_update["detailed-status"] = old_status
1439
1440                 # wait until all are configured.
1441                 while time() <= start_deploy + self.total_deploy_timeout:
1442                     if db_nsr_update:
1443                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1444                     if db_nslcmop_update:
1445                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1446                     # TODO add a fake task that set n2vc_event after some time
1447                     await n2vc_info["n2vc_event"].wait()
1448                     n2vc_info["n2vc_event"].clear()
1449                     all_active = True
1450                     status_map = {}
1451                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1452                     now = time()
1453                     for vca_deployed in vca_deployed_list:
1454                         vca_status = vca_deployed["operational-status"]
1455                         if vca_status not in status_map:
1456                             # Initialize it
1457                             status_map[vca_status] = 0
1458                         status_map[vca_status] += 1
1459
1460                         if vca_status == "active":
1461                             vca_deployed.pop("time_first_error", None)
1462                             vca_deployed.pop("status_first_error", None)
1463                             continue
1464
1465                         all_active = False
1466                         if vca_status in ("error", "blocked"):
1467                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1468                             # if not first time in this status error
1469                             if not vca_deployed.get("time_first_error"):
1470                                 vca_deployed["time_first_error"] = now
1471                                 continue
1472                         if vca_deployed.get("time_first_error") and \
1473                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1474                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1475                                                    .format(vca_deployed["member-vnf-index"],
1476                                                            vca_deployed["vdu_id"], vca_status,
1477                                                            vca_deployed["detailed-status-error"]))
1478
1479                     if all_active:
1480                         break
1481                     elif n2vc_error_text:
1482                         db_nsr_update["config-status"] = "failed"
1483                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1484                         db_nsr_update["detailed-status"] = error_text
1485                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1486                         db_nslcmop_update["detailed-status"] = error_text
1487                         db_nslcmop_update["statusEnteredTime"] = time()
1488                         configuration_failed = True
1489                         break
1490                     else:
1491                         cs = "configuring: "
1492                         separator = ""
1493                         for status, num in status_map.items():
1494                             cs += separator + "{}: {}".format(status, num)
1495                             separator = ", "
1496                         if old_status != cs:
1497                             db_nsr_update["config-status"] = cs
1498                             db_nsr_update["detailed-status"] = cs
1499                             db_nslcmop_update["detailed-status"] = cs
1500                             old_status = cs
1501                 else:   # total_deploy_timeout
1502                     raise LcmException("Timeout waiting ns to be configured")
1503
1504             if not configuration_failed:
1505                 # all is done
1506                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1507                 db_nslcmop_update["statusEnteredTime"] = time()
1508                 db_nslcmop_update["detailed-status"] = "done"
1509                 db_nsr_update["config-status"] = "configured"
1510                 db_nsr_update["detailed-status"] = "done"
1511
1512             return
1513
1514         except (ROclient.ROClientException, DbException, LcmException) as e:
1515             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1516             exc = e
1517         except asyncio.CancelledError:
1518             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1519             exc = "Operation was cancelled"
1520         except Exception as e:
1521             exc = traceback.format_exc()
1522             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1523                                  exc_info=True)
1524         finally:
1525             if exc:
1526                 if db_nsr:
1527                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1528                     db_nsr_update["operational-status"] = "failed"
1529                 if db_nslcmop:
1530                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1531                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1532                     db_nslcmop_update["statusEnteredTime"] = time()
1533             try:
1534                 if db_nsr:
1535                     db_nsr_update["_admin.nslcmop"] = None
1536                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1537                 if db_nslcmop_update:
1538                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1539             except DbException as e:
1540                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1541             if nslcmop_operation_state:
1542                 try:
1543                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1544                                                                    "operationState": nslcmop_operation_state},
1545                                             loop=self.loop)
1546                 except Exception as e:
1547                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1548
1549             self.logger.debug(logging_text + "Exit")
1550             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1551
1552     async def _destroy_charm(self, model, application):
1553         """
1554         Order N2VC destroy a charm
1555         :param model:
1556         :param application:
1557         :return: True if charm does not exist. False if it exist
1558         """
1559         if not await self.n2vc.HasApplication(model, application):
1560             return True  # Already removed
1561         await self.n2vc.RemoveCharms(model, application)
1562         return False
1563
1564     async def _wait_charm_destroyed(self, model, application, timeout):
1565         """
1566         Wait until charm does not exist
1567         :param model:
1568         :param application:
1569         :param timeout:
1570         :return: True if not exist, False if timeout
1571         """
1572         while True:
1573             if not await self.n2vc.HasApplication(model, application):
1574                 return True
1575             if timeout < 0:
1576                 return False
1577             await asyncio.sleep(10)
1578             timeout -= 10
1579
1580     # Check if this VNFD has a configured terminate action
1581     def _has_terminate_config_primitive(self, vnfd):
1582         vnf_config = vnfd.get("vnf-configuration")
1583         if vnf_config and vnf_config.get("terminate-config-primitive"):
1584             return True
1585         else:
1586             return False
1587
1588     @staticmethod
1589     def _get_terminate_config_primitive_seq_list(vnfd):
1590         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1591         # No need to check for existing primitive twice, already done before
1592         vnf_config = vnfd.get("vnf-configuration")
1593         seq_list = vnf_config.get("terminate-config-primitive")
1594         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1595         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1596         return seq_list_sorted
1597
1598     @staticmethod
1599     def _create_nslcmop(nsr_id, operation, params):
1600         """
1601         Creates a ns-lcm-opp content to be stored at database.
1602         :param nsr_id: internal id of the instance
1603         :param operation: instantiate, terminate, scale, action, ...
1604         :param params: user parameters for the operation
1605         :return: dictionary following SOL005 format
1606         """
1607         # Raise exception if invalid arguments
1608         if not (nsr_id and operation and params):
1609             raise LcmException(
1610                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1611         now = time()
1612         _id = str(uuid4())
1613         nslcmop = {
1614             "id": _id,
1615             "_id": _id,
1616             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1617             "operationState": "PROCESSING",
1618             "statusEnteredTime": now,
1619             "nsInstanceId": nsr_id,
1620             "lcmOperationType": operation,
1621             "startTime": now,
1622             "isAutomaticInvocation": False,
1623             "operationParams": params,
1624             "isCancelPending": False,
1625             "links": {
1626                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1627                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1628             }
1629         }
1630         return nslcmop
1631
1632     def _get_terminate_primitive_params(self, seq, vnf_index):
1633         primitive = seq.get('name')
1634         primitive_params = {}
1635         params = {
1636             "member_vnf_index": vnf_index,
1637             "primitive": primitive,
1638             "primitive_params": primitive_params,
1639         }
1640         desc_params = {}
1641         return self._map_primitive_params(seq, params, desc_params)
1642
1643     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1644                           vdu_name, primitive, mapped_primitive_params):
1645         # Create the "_admin.operations" array, or append operation if array already exists
1646         key_admin = '_admin'
1647         key_operations = 'operations'
1648         db_nslcmop_update = {}
1649         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1650         op_list = db_nslcmop_admin.get(key_operations)
1651         new_op = {'member_vnf_index': vnf_index,
1652                   'vdu_id': vdu_id,
1653                   'vdu_count_index': vdu_count_index,
1654                   'primitive': primitive,
1655                   'primitive_params': mapped_primitive_params}
1656         if db_nslcmop_admin:
1657             if not op_list:
1658                 # First operation, create key 'operations' with current operation as first list element
1659                 db_nslcmop_admin.update({key_operations: [new_op]})
1660                 op_list = db_nslcmop_admin.get(key_operations)
1661             else:
1662                 # Not first operation, append operation to existing list
1663                 op_list.append(new_op)
1664
1665         db_nslcmop_update['_admin.operations'] = op_list
1666         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1667
1668     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1669         """ Create a primitive with params from VNFD
1670             Called from terminate() before deleting instance
1671             Calls action() to execute the primitive """
1672         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1673         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1674         db_vnfds = {}
1675         # Loop over VNFRs
1676         for vnfr in db_vnfrs_list:
1677             vnfd_id = vnfr["vnfd-id"]
1678             vnf_index = vnfr["member-vnf-index-ref"]
1679             if vnfd_id not in db_vnfds:
1680                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1681                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1682                 db_vnfds[vnfd_id] = vnfd
1683             vnfd = db_vnfds[vnfd_id]
1684             if not self._has_terminate_config_primitive(vnfd):
1685                 continue
1686             # Get the primitive's sorted sequence list
1687             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1688             for seq in seq_list:
1689                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1690                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1691                     vnf_index, seq.get("name"))
1692                 self.logger.debug(logging_text + step)
1693                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1694                 primitive = seq.get('name')
1695                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1696                 # The following 3 parameters are currently set to None for 'terminate':
1697                 # vdu_id, vdu_count_index, vdu_name
1698                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1699                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1700                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1701                 # Add suboperation
1702                 self._add_suboperation(db_nslcmop,
1703                                        nslcmop_id,
1704                                        vnf_index,
1705                                        vdu_id,
1706                                        vdu_count_index,
1707                                        vdu_name,
1708                                        primitive,
1709                                        mapped_primitive_params)
1710                 # Suboperations: Call _ns_execute_primitive() instead of action()
1711                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1712                 nsr_deployed = db_nsr["_admin"]["deployed"]
1713                 result, result_detail = await self._ns_execute_primitive(
1714                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1715                     mapped_primitive_params)
1716
1717                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1718                 #    nsr_id, nslcmop_terminate_action_id)
1719                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1720                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1721                 if result not in result_ok:
1722                     raise LcmException(
1723                         "terminate_primitive_action for vnf_member_index={}",
1724                         " primitive={} fails with error {}".format(
1725                             vnf_index, seq.get("name"), result_detail))
1726
1727     async def terminate(self, nsr_id, nslcmop_id):
1728
1729         # Try to lock HA task here
1730         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1731         if not task_is_locked_by_me:
1732             return
1733
1734         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1735         self.logger.debug(logging_text + "Enter")
1736         db_nsr = None
1737         db_nslcmop = None
1738         exc = None
1739         failed_detail = []   # annotates all failed error messages
1740         vca_time_destroy = None   # time of where destroy charm order
1741         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1742         db_nslcmop_update = {}
1743         nslcmop_operation_state = None
1744         autoremove = False  # autoremove after terminated
1745         try:
1746             # wait for any previous tasks in process
1747             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1748
1749             step = "Getting nslcmop={} from db".format(nslcmop_id)
1750             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1751             step = "Getting nsr={} from db".format(nsr_id)
1752             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1753             # nsd = db_nsr["nsd"]
1754             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1755             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1756                 return
1757             # #TODO check if VIM is creating and wait
1758             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1759             # Call internal terminate action
1760             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1761
1762             db_nsr_update["operational-status"] = "terminating"
1763             db_nsr_update["config-status"] = "terminating"
1764
1765             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1766                 vca_model_name = nsr_deployed["VCA-model-name"]
1767                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1768                 self.logger.debug(logging_text + step)
1769                 try:
1770                     await self.n2vc.DestroyNetworkService(vca_model_name)
1771                 except NetworkServiceDoesNotExist:
1772                     pass
1773                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1774                 if nsr_deployed.get("VCA"):
1775                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1776                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1777                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1778             # for backward compatibility if charm have been created with "default"  model name delete one by one
1779             elif nsr_deployed and nsr_deployed.get("VCA"):
1780                 try:
1781                     step = "Scheduling configuration charms removing"
1782                     db_nsr_update["detailed-status"] = "Deleting charms"
1783                     self.logger.debug(logging_text + step)
1784                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1785                     # for backward compatibility
1786                     if isinstance(nsr_deployed["VCA"], dict):
1787                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1788                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1789                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1790
1791                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1792                         if vca_deployed:
1793                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1794                                 vca_deployed.clear()
1795                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1796                             else:
1797                                 vca_time_destroy = time()
1798                 except Exception as e:
1799                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1800
1801             # remove from RO
1802             RO_fail = False
1803
1804             # Delete ns
1805             RO_nsr_id = RO_delete_action = None
1806             if nsr_deployed and nsr_deployed.get("RO"):
1807                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1808                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1809             try:
1810                 if RO_nsr_id:
1811                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1812                         "Deleting ns from VIM"
1813                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1814                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1815                     self.logger.debug(logging_text + step)
1816                     desc = await self.RO.delete("ns", RO_nsr_id)
1817                     RO_delete_action = desc["action_id"]
1818                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1819                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1820                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1821                 if RO_delete_action:
1822                     # wait until NS is deleted from VIM
1823                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1824                         format(RO_nsr_id, RO_delete_action)
1825                     detailed_status_old = None
1826                     self.logger.debug(logging_text + step)
1827
1828                     delete_timeout = 20 * 60   # 20 minutes
1829                     while delete_timeout > 0:
1830                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1831                                                   extra_item_id=RO_delete_action)
1832                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1833                         if ns_status == "ERROR":
1834                             raise ROclient.ROClientException(ns_status_info)
1835                         elif ns_status == "BUILD":
1836                             detailed_status = step + "; {}".format(ns_status_info)
1837                         elif ns_status == "ACTIVE":
1838                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1839                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1840                             break
1841                         else:
1842                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1843                         if detailed_status != detailed_status_old:
1844                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1845                                 db_nsr_update["detailed-status"] = detailed_status
1846                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1847                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1848                         await asyncio.sleep(5, loop=self.loop)
1849                         delete_timeout -= 5
1850                     else:  # delete_timeout <= 0:
1851                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1852
1853             except ROclient.ROClientException as e:
1854                 if e.http_code == 404:  # not found
1855                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1856                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1857                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1858                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1859                 elif e.http_code == 409:   # conflict
1860                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1861                     self.logger.debug(logging_text + failed_detail[-1])
1862                     RO_fail = True
1863                 else:
1864                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1865                     self.logger.error(logging_text + failed_detail[-1])
1866                     RO_fail = True
1867
1868             # Delete nsd
1869             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1870                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1871                 try:
1872                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1873                         "Deleting nsd from RO"
1874                     await self.RO.delete("nsd", RO_nsd_id)
1875                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1876                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1877                 except ROclient.ROClientException as e:
1878                     if e.http_code == 404:  # not found
1879                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1880                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1881                     elif e.http_code == 409:   # conflict
1882                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1883                         self.logger.debug(logging_text + failed_detail[-1])
1884                         RO_fail = True
1885                     else:
1886                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1887                         self.logger.error(logging_text + failed_detail[-1])
1888                         RO_fail = True
1889
1890             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1891                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1892                     if not vnf_deployed or not vnf_deployed["id"]:
1893                         continue
1894                     try:
1895                         RO_vnfd_id = vnf_deployed["id"]
1896                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1897                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1898                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1899                         await self.RO.delete("vnfd", RO_vnfd_id)
1900                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1901                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1902                     except ROclient.ROClientException as e:
1903                         if e.http_code == 404:  # not found
1904                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1905                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1906                         elif e.http_code == 409:   # conflict
1907                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1908                             self.logger.debug(logging_text + failed_detail[-1])
1909                         else:
1910                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1911                             self.logger.error(logging_text + failed_detail[-1])
1912
1913             # wait until charm deleted
1914             if vca_time_destroy:
1915                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1916                     "Waiting for deletion of configuration charms"
1917                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1918                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1919                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1920                     if not vca_deployed:
1921                         continue
1922                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1923                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1924                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1925                                                             timeout):
1926                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1927                             vca_deployed["application"]))
1928                     else:
1929                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1930
1931             if failed_detail:
1932                 self.logger.error(logging_text + " ;".join(failed_detail))
1933                 db_nsr_update["operational-status"] = "failed"
1934                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1935                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1936                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1937                 db_nslcmop_update["statusEnteredTime"] = time()
1938             else:
1939                 db_nsr_update["operational-status"] = "terminated"
1940                 db_nsr_update["detailed-status"] = "Done"
1941                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1942                 db_nslcmop_update["detailed-status"] = "Done"
1943                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1944                 db_nslcmop_update["statusEnteredTime"] = time()
1945                 if db_nslcmop["operationParams"].get("autoremove"):
1946                     autoremove = True
1947
1948         except (ROclient.ROClientException, DbException, LcmException) as e:
1949             self.logger.error(logging_text + "Exit Exception {}".format(e))
1950             exc = e
1951         except asyncio.CancelledError:
1952             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1953             exc = "Operation was cancelled"
1954         except Exception as e:
1955             exc = traceback.format_exc()
1956             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1957         finally:
1958             if exc and db_nslcmop:
1959                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1960                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1961                 db_nslcmop_update["statusEnteredTime"] = time()
1962             try:
1963                 if db_nslcmop and db_nslcmop_update:
1964                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1965                 if db_nsr:
1966                     db_nsr_update["_admin.nslcmop"] = None
1967                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1968             except DbException as e:
1969                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1970             if nslcmop_operation_state:
1971                 try:
1972                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1973                                                                  "operationState": nslcmop_operation_state,
1974                                                                  "autoremove": autoremove},
1975                                             loop=self.loop)
1976                 except Exception as e:
1977                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1978             self.logger.debug(logging_text + "Exit")
1979             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1980
1981     @staticmethod
1982     def _map_primitive_params(primitive_desc, params, instantiation_params):
1983         """
1984         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1985         The default-value is used. If it is between < > it look for a value at instantiation_params
1986         :param primitive_desc: portion of VNFD/NSD that describes primitive
1987         :param params: Params provided by user
1988         :param instantiation_params: Instantiation params provided by user
1989         :return: a dictionary with the calculated params
1990         """
1991         calculated_params = {}
1992         for parameter in primitive_desc.get("parameter", ()):
1993             param_name = parameter["name"]
1994             if param_name in params:
1995                 calculated_params[param_name] = params[param_name]
1996             elif "default-value" in parameter or "value" in parameter:
1997                 if "value" in parameter:
1998                     calculated_params[param_name] = parameter["value"]
1999                 else:
2000                     calculated_params[param_name] = parameter["default-value"]
2001                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
2002                         and calculated_params[param_name].endswith(">"):
2003                     if calculated_params[param_name][1:-1] in instantiation_params:
2004                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
2005                     else:
2006                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
2007                                            format(parameter["default-value"], primitive_desc["name"]))
2008             else:
2009                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
2010                                    format(param_name, primitive_desc["name"]))
2011
2012             if isinstance(calculated_params[param_name], (dict, list, tuple)):
2013                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
2014                                                                width=256)
2015             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
2016                 calculated_params[param_name] = calculated_params[param_name][7:]
2017         return calculated_params
2018
2019     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
2020                                     primitive, primitive_params, retries=0, retries_interval=30):
2021         start_primitive_time = time()
2022         try:
2023             for vca_deployed in db_deployed["VCA"]:
2024                 if not vca_deployed:
2025                     continue
2026                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2027                     continue
2028                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2029                     continue
2030                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2031                     continue
2032                 break
2033             else:
2034                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2035                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2036             model_name = vca_deployed.get("model")
2037             application_name = vca_deployed.get("application")
2038             if not model_name or not application_name:
2039                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2040                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2041                                                                        vdu_count_index))
2042             # if vca_deployed["operational-status"] != "active":
2043             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2044             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2045             callback = None  # self.n2vc_callback
2046             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2047             await self.n2vc.login()
2048             if primitive == "config":
2049                 primitive_params = {"params": primitive_params}
2050             while retries >= 0:
2051                 primitive_id = await self.n2vc.ExecutePrimitive(
2052                     model_name,
2053                     application_name,
2054                     primitive,
2055                     callback,
2056                     *callback_args,
2057                     **primitive_params
2058                 )
2059                 while time() - start_primitive_time < self.timeout_primitive:
2060                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2061                     if primitive_result_ in ("completed", "failed"):
2062                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2063                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2064                         break
2065                     elif primitive_result_ is None and primitive == "config":
2066                         primitive_result = "COMPLETED"
2067                         detailed_result = None
2068                         break
2069                     else:  # ("running", "pending", None):
2070                         pass
2071                     await asyncio.sleep(5)
2072                 else:
2073                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2074                 if primitive_result == "COMPLETED":
2075                     break
2076                 retries -= 1
2077                 if retries >= 0:
2078                     await asyncio.sleep(retries_interval)
2079
2080             return primitive_result, detailed_result
2081         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2082             return "FAILED", str(e)
2083
2084     async def action(self, nsr_id, nslcmop_id):
2085
2086         # Try to lock HA task here
2087         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2088         if not task_is_locked_by_me:
2089             return
2090
2091         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2092         self.logger.debug(logging_text + "Enter")
2093         # get all needed from database
2094         db_nsr = None
2095         db_nslcmop = None
2096         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2097         db_nslcmop_update = {}
2098         nslcmop_operation_state = None
2099         nslcmop_operation_state_detail = None
2100         exc = None
2101         try:
2102             # wait for any previous tasks in process
2103             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2104
2105             step = "Getting information from database"
2106             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2107             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2108
2109             nsr_deployed = db_nsr["_admin"].get("deployed")
2110             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2111             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2112             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2113             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2114
2115             if vnf_index:
2116                 step = "Getting vnfr from database"
2117                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2118                 step = "Getting vnfd from database"
2119                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2120             else:
2121                 if db_nsr.get("nsd"):
2122                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2123                 else:
2124                     step = "Getting nsd from database"
2125                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2126
2127             # for backward compatibility
2128             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2129                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2130                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2131                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2132
2133             primitive = db_nslcmop["operationParams"]["primitive"]
2134             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2135
2136             # look for primitive
2137             config_primitive_desc = None
2138             if vdu_id:
2139                 for vdu in get_iterable(db_vnfd, "vdu"):
2140                     if vdu_id == vdu["id"]:
2141                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2142                             if config_primitive["name"] == primitive:
2143                                 config_primitive_desc = config_primitive
2144                                 break
2145             elif vnf_index:
2146                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2147                     if config_primitive["name"] == primitive:
2148                         config_primitive_desc = config_primitive
2149                         break
2150             else:
2151                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2152                     if config_primitive["name"] == primitive:
2153                         config_primitive_desc = config_primitive
2154                         break
2155
2156             if not config_primitive_desc:
2157                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2158                                    format(primitive))
2159
2160             desc_params = {}
2161             if vnf_index:
2162                 if db_vnfr.get("additionalParamsForVnf"):
2163                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2164             else:
2165                 if db_nsr.get("additionalParamsForVnf"):
2166                     desc_params.update(db_nsr["additionalParamsForNs"])
2167
2168             # TODO check if ns is in a proper status
2169             result, result_detail = await self._ns_execute_primitive(
2170                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2171                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2172             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2173             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2174             db_nslcmop_update["statusEnteredTime"] = time()
2175             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2176             return  # database update is called inside finally
2177
2178         except (DbException, LcmException) as e:
2179             self.logger.error(logging_text + "Exit Exception {}".format(e))
2180             exc = e
2181         except asyncio.CancelledError:
2182             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2183             exc = "Operation was cancelled"
2184         except Exception as e:
2185             exc = traceback.format_exc()
2186             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2187         finally:
2188             if exc and db_nslcmop:
2189                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2190                     "FAILED {}: {}".format(step, exc)
2191                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2192                 db_nslcmop_update["statusEnteredTime"] = time()
2193             try:
2194                 if db_nslcmop_update:
2195                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2196                 if db_nsr:
2197                     db_nsr_update["_admin.nslcmop"] = None
2198                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2199             except DbException as e:
2200                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2201             self.logger.debug(logging_text + "Exit")
2202             if nslcmop_operation_state:
2203                 try:
2204                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2205                                                                "operationState": nslcmop_operation_state},
2206                                             loop=self.loop)
2207                 except Exception as e:
2208                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2209             self.logger.debug(logging_text + "Exit")
2210             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2211             return nslcmop_operation_state, nslcmop_operation_state_detail
2212
2213     async def scale(self, nsr_id, nslcmop_id):
2214
2215         # Try to lock HA task here
2216         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2217         if not task_is_locked_by_me:
2218             return
2219
2220         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2221         self.logger.debug(logging_text + "Enter")
2222         # get all needed from database
2223         db_nsr = None
2224         db_nslcmop = None
2225         db_nslcmop_update = {}
2226         nslcmop_operation_state = None
2227         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2228         exc = None
2229         # in case of error, indicates what part of scale was failed to put nsr at error status
2230         scale_process = None
2231         old_operational_status = ""
2232         old_config_status = ""
2233         vnfr_scaled = False
2234         try:
2235             # wait for any previous tasks in process
2236             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2237
2238             step = "Getting nslcmop from database"
2239             self.logger.debug(step + " after having waited for previous tasks to be completed")
2240             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2241             step = "Getting nsr from database"
2242             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2243
2244             old_operational_status = db_nsr["operational-status"]
2245             old_config_status = db_nsr["config-status"]
2246             step = "Parsing scaling parameters"
2247             # self.logger.debug(step)
2248             db_nsr_update["operational-status"] = "scaling"
2249             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2250             nsr_deployed = db_nsr["_admin"].get("deployed")
2251             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2252             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2253             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2254             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2255             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2256
2257             # for backward compatibility
2258             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2259                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2260                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2261                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2262
2263             step = "Getting vnfr from database"
2264             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2265             step = "Getting vnfd from database"
2266             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2267
2268             step = "Getting scaling-group-descriptor"
2269             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2270                 if scaling_descriptor["name"] == scaling_group:
2271                     break
2272             else:
2273                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2274                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2275
2276             # cooldown_time = 0
2277             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2278             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2279             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2280             #         break
2281
2282             # TODO check if ns is in a proper status
2283             step = "Sending scale order to VIM"
2284             nb_scale_op = 0
2285             if not db_nsr["_admin"].get("scaling-group"):
2286                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2287                 admin_scale_index = 0
2288             else:
2289                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2290                     if admin_scale_info["name"] == scaling_group:
2291                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2292                         break
2293                 else:  # not found, set index one plus last element and add new entry with the name
2294                     admin_scale_index += 1
2295                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2296             RO_scaling_info = []
2297             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2298             if scaling_type == "SCALE_OUT":
2299                 # count if max-instance-count is reached
2300                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2301                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2302                 if nb_scale_op >= max_instance_count:
2303                     raise LcmException("reached the limit of {} (max-instance-count) "
2304                                        "scaling-out operations for the "
2305                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2306
2307                 nb_scale_op += 1
2308                 vdu_scaling_info["scaling_direction"] = "OUT"
2309                 vdu_scaling_info["vdu-create"] = {}
2310                 for vdu_scale_info in scaling_descriptor["vdu"]:
2311                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2312                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2313                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2314
2315             elif scaling_type == "SCALE_IN":
2316                 # count if min-instance-count is reached
2317                 min_instance_count = 0
2318                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2319                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2320                 if nb_scale_op <= min_instance_count:
2321                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2322                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2323                 nb_scale_op -= 1
2324                 vdu_scaling_info["scaling_direction"] = "IN"
2325                 vdu_scaling_info["vdu-delete"] = {}
2326                 for vdu_scale_info in scaling_descriptor["vdu"]:
2327                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2328                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2329                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2330
2331             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2332             vdu_create = vdu_scaling_info.get("vdu-create")
2333             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2334             if vdu_scaling_info["scaling_direction"] == "IN":
2335                 for vdur in reversed(db_vnfr["vdur"]):
2336                     if vdu_delete.get(vdur["vdu-id-ref"]):
2337                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2338                         vdu_scaling_info["vdu"].append({
2339                             "name": vdur["name"],
2340                             "vdu_id": vdur["vdu-id-ref"],
2341                             "interface": []
2342                         })
2343                         for interface in vdur["interfaces"]:
2344                             vdu_scaling_info["vdu"][-1]["interface"].append({
2345                                 "name": interface["name"],
2346                                 "ip_address": interface["ip-address"],
2347                                 "mac_address": interface.get("mac-address"),
2348                             })
2349                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2350
2351             # execute primitive service PRE-SCALING
2352             step = "Executing pre-scale vnf-config-primitive"
2353             if scaling_descriptor.get("scaling-config-action"):
2354                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2355                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2356                             and scaling_type == "SCALE_IN":
2357                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2358                         step = db_nslcmop_update["detailed-status"] = \
2359                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2360
2361                         # look for primitive
2362                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2363                             if config_primitive["name"] == vnf_config_primitive:
2364                                 break
2365                         else:
2366                             raise LcmException(
2367                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2368                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2369                                 "primitive".format(scaling_group, config_primitive))
2370
2371                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2372                         if db_vnfr.get("additionalParamsForVnf"):
2373                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2374
2375                         scale_process = "VCA"
2376                         db_nsr_update["config-status"] = "configuring pre-scaling"
2377                         result, result_detail = await self._ns_execute_primitive(
2378                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2379                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2380                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2381                             vnf_config_primitive, result, result_detail))
2382                         if result == "FAILED":
2383                             raise LcmException(result_detail)
2384                         db_nsr_update["config-status"] = old_config_status
2385                         scale_process = None
2386
2387             if RO_scaling_info:
2388                 scale_process = "RO"
2389                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2390                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2391                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2392                 # wait until ready
2393                 RO_nslcmop_id = RO_desc["instance_action_id"]
2394                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2395
2396                 RO_task_done = False
2397                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2398                 detailed_status_old = None
2399                 self.logger.debug(logging_text + step)
2400
2401                 deployment_timeout = 1 * 3600   # One hour
2402                 while deployment_timeout > 0:
2403                     if not RO_task_done:
2404                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2405                                                   extra_item_id=RO_nslcmop_id)
2406                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2407                         if ns_status == "ERROR":
2408                             raise ROclient.ROClientException(ns_status_info)
2409                         elif ns_status == "BUILD":
2410                             detailed_status = step + "; {}".format(ns_status_info)
2411                         elif ns_status == "ACTIVE":
2412                             RO_task_done = True
2413                             step = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
2414                             self.logger.debug(logging_text + step)
2415                         else:
2416                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2417                     else:
2418                         desc = await self.RO.show("ns", RO_nsr_id)
2419                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2420                         if ns_status == "ERROR":
2421                             raise ROclient.ROClientException(ns_status_info)
2422                         elif ns_status == "BUILD":
2423                             detailed_status = step + "; {}".format(ns_status_info)
2424                         elif ns_status == "ACTIVE":
2425                             step = detailed_status = \
2426                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2427                             if not vnfr_scaled:
2428                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2429                                 vnfr_scaled = True
2430                             try:
2431                                 desc = await self.RO.show("ns", RO_nsr_id)
2432                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2433                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2434                                 break
2435                             except LcmExceptionNoMgmtIP:
2436                                 pass
2437                         else:
2438                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2439                     if detailed_status != detailed_status_old:
2440                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2441                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2442
2443                     await asyncio.sleep(5, loop=self.loop)
2444                     deployment_timeout -= 5
2445                 if deployment_timeout <= 0:
2446                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2447
2448                 # update VDU_SCALING_INFO with the obtained ip_addresses
2449                 if vdu_scaling_info["scaling_direction"] == "OUT":
2450                     for vdur in reversed(db_vnfr["vdur"]):
2451                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2452                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2453                             vdu_scaling_info["vdu"].append({
2454                                 "name": vdur["name"],
2455                                 "vdu_id": vdur["vdu-id-ref"],
2456                                 "interface": []
2457                             })
2458                             for interface in vdur["interfaces"]:
2459                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2460                                     "name": interface["name"],
2461                                     "ip_address": interface["ip-address"],
2462                                     "mac_address": interface.get("mac-address"),
2463                                 })
2464                     del vdu_scaling_info["vdu-create"]
2465
2466             scale_process = None
2467             if db_nsr_update:
2468                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2469
2470             # execute primitive service POST-SCALING
2471             step = "Executing post-scale vnf-config-primitive"
2472             if scaling_descriptor.get("scaling-config-action"):
2473                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2474                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2475                             and scaling_type == "SCALE_OUT":
2476                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2477                         step = db_nslcmop_update["detailed-status"] = \
2478                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2479
2480                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2481                         if db_vnfr.get("additionalParamsForVnf"):
2482                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2483
2484                         # look for primitive
2485                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2486                             if config_primitive["name"] == vnf_config_primitive:
2487                                 break
2488                         else:
2489                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2490                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2491                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2492                                                                                                      config_primitive))
2493                         scale_process = "VCA"
2494                         db_nsr_update["config-status"] = "configuring post-scaling"
2495
2496                         result, result_detail = await self._ns_execute_primitive(
2497                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2498                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2499                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2500                             vnf_config_primitive, result, result_detail))
2501                         if result == "FAILED":
2502                             raise LcmException(result_detail)
2503                         db_nsr_update["config-status"] = old_config_status
2504                         scale_process = None
2505
2506             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2507             db_nslcmop_update["statusEnteredTime"] = time()
2508             db_nslcmop_update["detailed-status"] = "done"
2509             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2510             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2511                 else old_operational_status
2512             db_nsr_update["config-status"] = old_config_status
2513             return
2514         except (ROclient.ROClientException, DbException, LcmException) as e:
2515             self.logger.error(logging_text + "Exit Exception {}".format(e))
2516             exc = e
2517         except asyncio.CancelledError:
2518             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2519             exc = "Operation was cancelled"
2520         except Exception as e:
2521             exc = traceback.format_exc()
2522             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2523         finally:
2524             if exc:
2525                 if db_nslcmop:
2526                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2527                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2528                     db_nslcmop_update["statusEnteredTime"] = time()
2529                 if db_nsr:
2530                     db_nsr_update["operational-status"] = old_operational_status
2531                     db_nsr_update["config-status"] = old_config_status
2532                     db_nsr_update["detailed-status"] = ""
2533                     db_nsr_update["_admin.nslcmop"] = None
2534                     if scale_process:
2535                         if "VCA" in scale_process:
2536                             db_nsr_update["config-status"] = "failed"
2537                         if "RO" in scale_process:
2538                             db_nsr_update["operational-status"] = "failed"
2539                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2540                                                                                                      exc)
2541             try:
2542                 if db_nslcmop and db_nslcmop_update:
2543                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2544                 if db_nsr:
2545                     db_nsr_update["_admin.nslcmop"] = None
2546                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2547             except DbException as e:
2548                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2549             if nslcmop_operation_state:
2550                 try:
2551                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2552                                                              "operationState": nslcmop_operation_state},
2553                                             loop=self.loop)
2554                     # if cooldown_time:
2555                     #     await asyncio.sleep(cooldown_time)
2556                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2557                 except Exception as e:
2558                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2559             self.logger.debug(logging_text + "Exit")
2560             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")