osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105
 106     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 107         """
 108         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 109         :param vnfd: input vnfd
 110         :param new_id: overrides vnf id if provided
 111         :param additionalParams: Instantiation params for VNFs provided
 112         :param nsrId: Id of the NSR
 113         :return: copy of vnfd
 114         """
 115         try:
 116             vnfd_RO = deepcopy(vnfd)
 117             # remove unused by RO configuration, monitoring, scaling and internal keys
 118             vnfd_RO.pop("_id", None)
 119             vnfd_RO.pop("_admin", None)
 120             vnfd_RO.pop("vnf-configuration", None)
 121             vnfd_RO.pop("monitoring-param", None)
 122             vnfd_RO.pop("scaling-group-descriptor", None)
 123             if new_id:
 124                 vnfd_RO["id"] = new_id
 125
 126             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 127             for vdu in get_iterable(vnfd_RO, "vdu"):
 128                 cloud_init_file = None
 129                 if vdu.get("cloud-init-file"):
 130                     base_folder = vnfd["_admin"]["storage"]
 131                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 132                                                                    vdu["cloud-init-file"])
 133                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 134                         cloud_init_content = ci_file.read()
 135                     vdu.pop("cloud-init-file", None)
 136                 elif vdu.get("cloud-init"):
 137                     cloud_init_content = vdu["cloud-init"]
 138                 else:
 139                     continue
 140
 141                 env = Environment()
 142                 ast = env.parse(cloud_init_content)
 143                 mandatory_vars = meta.find_undeclared_variables(ast)
 144                 if mandatory_vars:
 145                     for var in mandatory_vars:
 146                         if not additionalParams or var not in additionalParams.keys():
 147                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 148                                                "file, must be provided in the instantiation parameters inside the "
 149                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 150                 template = Template(cloud_init_content)
 151                 cloud_init_content = template.render(additionalParams or {})
 152                 vdu["cloud-init"] = cloud_init_content
 153
 154             return vnfd_RO
 155         except FsException as e:
 156             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 157                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 158         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 159             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 160                                format(vnfd["id"], vdu["id"], e))
 161
 162     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 163         """
 164         Callback both for charm status change and task completion
 165         :param model_name: Charm model name
 166         :param application_name: Charm application name
 167         :param status: Can be
 168             - blocked: The unit needs manual intervention
 169             - maintenance: The unit is actively deploying/configuring
 170             - waiting: The unit is waiting for another charm to be ready
 171             - active: The unit is deployed, configured, and ready
 172             - error: The charm has failed and needs attention.
 173             - terminated: The charm has been destroyed
 174             - removing,
 175             - removed
 176         :param message: detailed message error
 177         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 178             nsr_id:
 179             nslcmop_id:
 180             lcmOperationType: currently "instantiate"
 181             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 182             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 183             n2vc_event: event used to notify instantiation task that some change has been produced
 184         :param task: None for charm status change, or task for completion task callback
 185         :return:
 186         """
 187         try:
 188             nsr_id = n2vc_info["nsr_id"]
 189             deployed = n2vc_info["deployed"]
 190             db_nsr_update = n2vc_info["db_update"]
 191             nslcmop_id = n2vc_info["nslcmop_id"]
 192             ns_operation = n2vc_info["lcmOperationType"]
 193             n2vc_event = n2vc_info["n2vc_event"]
 194             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 195                                                                                     application_name)
 196             for vca_index, vca_deployed in enumerate(deployed):
 197                 if not vca_deployed:
 198                     continue
 199                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 200                     break
 201             else:
 202                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 203                                   format(model_name))
 204                 return
 205             if task:
 206                 if task.cancelled():
 207                     self.logger.debug(logging_text + " task Cancelled")
 208                     vca_deployed['operational-status'] = "error"
 209                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 210                     vca_deployed['detailed-status'] = "Task Cancelled"
 211                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 212
 213                 elif task.done():
 214                     exc = task.exception()
 215                     if exc:
 216                         self.logger.error(logging_text + " task Exception={}".format(exc))
 217                         vca_deployed['operational-status'] = "error"
 218                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 219                         vca_deployed['detailed-status'] = str(exc)
 220                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 221                     else:
 222                         self.logger.debug(logging_text + " task Done")
 223                         # task is Done, but callback is still ongoing. So ignore
 224                         return
 225             elif status:
 226                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 227                 if vca_deployed['operational-status'] == status:
 228                     return  # same status, ignore
 229                 vca_deployed['operational-status'] = status
 230                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 231                 vca_deployed['detailed-status'] = str(message)
 232                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 233             else:
 234                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 235                 return
 236             # wake up instantiate task
 237             n2vc_event.set()
 238         except Exception as e:
 239             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 240
 241     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 242         """
 243         Creates a RO ns descriptor from OSM ns_instantiate params
 244         :param ns_params: OSM instantiate params
 245         :return: The RO ns descriptor
 246         """
 247         vim_2_RO = {}
 248         wim_2_RO = {}
 249         # TODO feature 1417: Check that no instantiation is set over PDU
 250         # check if PDU forces a concrete vim-network-id and add it
 251         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 252
 253         def vim_account_2_RO(vim_account):
 254             if vim_account in vim_2_RO:
 255                 return vim_2_RO[vim_account]
 256
 257             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 258             if db_vim["_admin"]["operationalState"] != "ENABLED":
 259                 raise LcmException("VIM={} is not available. operationalState={}".format(
 260                     vim_account, db_vim["_admin"]["operationalState"]))
 261             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 262             vim_2_RO[vim_account] = RO_vim_id
 263             return RO_vim_id
 264
 265         def wim_account_2_RO(wim_account):
 266             if isinstance(wim_account, str):
 267                 if wim_account in wim_2_RO:
 268                     return wim_2_RO[wim_account]
 269
 270                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 271                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 272                     raise LcmException("WIM={} is not available. operationalState={}".format(
 273                         wim_account, db_wim["_admin"]["operationalState"]))
 274                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 275                 wim_2_RO[wim_account] = RO_wim_id
 276                 return RO_wim_id
 277             else:
 278                 return wim_account
 279
 280         def ip_profile_2_RO(ip_profile):
 281             RO_ip_profile = deepcopy((ip_profile))
 282             if "dns-server" in RO_ip_profile:
 283                 if isinstance(RO_ip_profile["dns-server"], list):
 284                     RO_ip_profile["dns-address"] = []
 285                     for ds in RO_ip_profile.pop("dns-server"):
 286                         RO_ip_profile["dns-address"].append(ds['address'])
 287                 else:
 288                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 289             if RO_ip_profile.get("ip-version") == "ipv4":
 290                 RO_ip_profile["ip-version"] = "IPv4"
 291             if RO_ip_profile.get("ip-version") == "ipv6":
 292                 RO_ip_profile["ip-version"] = "IPv6"
 293             if "dhcp-params" in RO_ip_profile:
 294                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 295             return RO_ip_profile
 296
 297         if not ns_params:
 298             return None
 299         RO_ns_params = {
 300             # "name": ns_params["nsName"],
 301             # "description": ns_params.get("nsDescription"),
 302             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 303             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 304             # "scenario": ns_params["nsdId"],
 305         }
 306         if n2vc_key_list:
 307             for vnfd_ref, vnfd in vnfd_dict.items():
 308                 vdu_needed_access = []
 309                 mgmt_cp = None
 310                 if vnfd.get("vnf-configuration"):
 311                     if vnfd.get("mgmt-interface"):
 312                         if vnfd["mgmt-interface"].get("vdu-id"):
 313                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 314                         elif vnfd["mgmt-interface"].get("cp"):
 315                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 316
 317                 for vdu in vnfd.get("vdu", ()):
 318                     if vdu.get("vdu-configuration"):
 319                         vdu_needed_access.append(vdu["id"])
 320                     elif mgmt_cp:
 321                         for vdu_interface in vdu.get("interface"):
 322                             if vdu_interface.get("external-connection-point-ref") and \
 323                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 324                                 vdu_needed_access.append(vdu["id"])
 325                                 mgmt_cp = None
 326                                 break
 327
 328                 if vdu_needed_access:
 329                     for vnf_member in nsd.get("constituent-vnfd"):
 330                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 331                             continue
 332                         for vdu in vdu_needed_access:
 333                             populate_dict(RO_ns_params,
 334                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 335                                           n2vc_key_list)
 336
 337         if ns_params.get("vduImage"):
 338             RO_ns_params["vduImage"] = ns_params["vduImage"]
 339
 340         if ns_params.get("ssh_keys"):
 341             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 342         for vnf_params in get_iterable(ns_params, "vnf"):
 343             for constituent_vnfd in nsd["constituent-vnfd"]:
 344                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 345                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 346                     break
 347             else:
 348                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 349                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 350             if vnf_params.get("vimAccountId"):
 351                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 352                               vim_account_2_RO(vnf_params["vimAccountId"]))
 353
 354             for vdu_params in get_iterable(vnf_params, "vdu"):
 355                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 356                 if vdu_params.get("volume"):
 357                     for volume_params in vdu_params["volume"]:
 358                         if volume_params.get("vim-volume-id"):
 359                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 360                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 361                                           volume_params["vim-volume-id"])
 362                 if vdu_params.get("interface"):
 363                     for interface_params in vdu_params["interface"]:
 364                         if interface_params.get("ip-address"):
 365                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 366                                                          vdu_params["id"], "interfaces", interface_params["name"],
 367                                                          "ip_address"),
 368                                           interface_params["ip-address"])
 369                         if interface_params.get("mac-address"):
 370                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 371                                                          vdu_params["id"], "interfaces", interface_params["name"],
 372                                                          "mac_address"),
 373                                           interface_params["mac-address"])
 374                         if interface_params.get("floating-ip-required"):
 375                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 376                                                          vdu_params["id"], "interfaces", interface_params["name"],
 377                                                          "floating-ip"),
 378                                           interface_params["floating-ip-required"])
 379
 380             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 381                 if internal_vld_params.get("vim-network-name"):
 382                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 383                                                  internal_vld_params["name"], "vim-network-name"),
 384                                   internal_vld_params["vim-network-name"])
 385                 if internal_vld_params.get("vim-network-id"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-id"),
 388                                   internal_vld_params["vim-network-id"])
 389                 if internal_vld_params.get("ip-profile"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "ip-profile"),
 392                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 393
 394                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 395                     # look for interface
 396                     iface_found = False
 397                     for vdu_descriptor in vnf_descriptor["vdu"]:
 398                         for vdu_interface in vdu_descriptor["interface"]:
 399                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 400                                 if icp_params.get("ip-address"):
 401                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 402                                                                  vdu_descriptor["id"], "interfaces",
 403                                                                  vdu_interface["name"], "ip_address"),
 404                                                   icp_params["ip-address"])
 405
 406                                 if icp_params.get("mac-address"):
 407                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 408                                                                  vdu_descriptor["id"], "interfaces",
 409                                                                  vdu_interface["name"], "mac_address"),
 410                                                   icp_params["mac-address"])
 411                                 iface_found = True
 412                                 break
 413                         if iface_found:
 414                             break
 415                     else:
 416                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 417                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 418                                            "connection-point".format(vnf_params["member-vnf-index"],
 419                                                                      icp_params["id-ref"]))
 420
 421         for vld_params in get_iterable(ns_params, "vld"):
 422             if "ip-profile" in vld_params:
 423                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 424                               ip_profile_2_RO(vld_params["ip-profile"]))
 425
 426             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 428                               wim_account_2_RO(vld_params["wimAccountId"])),
 429             if vld_params.get("vim-network-name"):
 430                 RO_vld_sites = []
 431                 if isinstance(vld_params["vim-network-name"], dict):
 432                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 433                         RO_vld_sites.append({
 434                             "netmap-use": vim_net,
 435                             "datacenter": vim_account_2_RO(vim_account)
 436                         })
 437                 else:  # isinstance str
 438                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 439                 if RO_vld_sites:
 440                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 441             if vld_params.get("vim-network-id"):
 442                 RO_vld_sites = []
 443                 if isinstance(vld_params["vim-network-id"], dict):
 444                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 445                         RO_vld_sites.append({
 446                             "netmap-use": vim_net,
 447                             "datacenter": vim_account_2_RO(vim_account)
 448                         })
 449                 else:  # isinstance str
 450                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 451                 if RO_vld_sites:
 452                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 453             if vld_params.get("ns-net"):
 454                 if isinstance(vld_params["ns-net"], dict):
 455                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 456                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 457                 if RO_vld_ns_net:
 458                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 459             if "vnfd-connection-point-ref" in vld_params:
 460                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 461                     # look for interface
 462                     for constituent_vnfd in nsd["constituent-vnfd"]:
 463                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 464                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 465                             break
 466                     else:
 467                         raise LcmException(
 468                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 469                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 470                     match_cp = False
 471                     for vdu_descriptor in vnf_descriptor["vdu"]:
 472                         for interface_descriptor in vdu_descriptor["interface"]:
 473                             if interface_descriptor.get("external-connection-point-ref") == \
 474                                     cp_params["vnfd-connection-point-ref"]:
 475                                 match_cp = True
 476                                 break
 477                         if match_cp:
 478                             break
 479                     else:
 480                         raise LcmException(
 481                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 482                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 483                                 cp_params["member-vnf-index-ref"],
 484                                 cp_params["vnfd-connection-point-ref"],
 485                                 vnf_descriptor["id"]))
 486                     if cp_params.get("ip-address"):
 487                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 488                                                      vdu_descriptor["id"], "interfaces",
 489                                                      interface_descriptor["name"], "ip_address"),
 490                                       cp_params["ip-address"])
 491                     if cp_params.get("mac-address"):
 492                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 493                                                      vdu_descriptor["id"], "interfaces",
 494                                                      interface_descriptor["name"], "mac_address"),
 495                                       cp_params["mac-address"])
 496         return RO_ns_params
 497
 498     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 499         # make a copy to do not change
 500         vdu_create = copy(vdu_create)
 501         vdu_delete = copy(vdu_delete)
 502
 503         vdurs = db_vnfr.get("vdur")
 504         if vdurs is None:
 505             vdurs = []
 506         vdu_index = len(vdurs)
 507         while vdu_index:
 508             vdu_index -= 1
 509             vdur = vdurs[vdu_index]
 510             if vdur.get("pdu-type"):
 511                 continue
 512             vdu_id_ref = vdur["vdu-id-ref"]
 513             if vdu_create and vdu_create.get(vdu_id_ref):
 514                 for index in range(0, vdu_create[vdu_id_ref]):
 515                     vdur = deepcopy(vdur)
 516                     vdur["_id"] = str(uuid4())
 517                     vdur["count-index"] += 1
 518                     vdurs.insert(vdu_index+1+index, vdur)
 519                 del vdu_create[vdu_id_ref]
 520             if vdu_delete and vdu_delete.get(vdu_id_ref):
 521                 del vdurs[vdu_index]
 522                 vdu_delete[vdu_id_ref] -= 1
 523                 if not vdu_delete[vdu_id_ref]:
 524                     del vdu_delete[vdu_id_ref]
 525         # check all operations are done
 526         if vdu_create or vdu_delete:
 527             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 528                 vdu_create))
 529         if vdu_delete:
 530             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 531                 vdu_delete))
 532
 533         vnfr_update = {"vdur": vdurs}
 534         db_vnfr["vdur"] = vdurs
 535         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 536
 537     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 538         """
 539         Updates database nsr with the RO info for the created vld
 540         :param ns_update_nsr: dictionary to be filled with the updated info
 541         :param db_nsr: content of db_nsr. This is also modified
 542         :param nsr_desc_RO: nsr descriptor from RO
 543         :return: Nothing, LcmException is raised on errors
 544         """
 545
 546         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 547             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 548                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 549                     continue
 550                 vld["vim-id"] = net_RO.get("vim_net_id")
 551                 vld["name"] = net_RO.get("vim_name")
 552                 vld["status"] = net_RO.get("status")
 553                 vld["status-detailed"] = net_RO.get("error_msg")
 554                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 555                 break
 556             else:
 557                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 558
 559     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 560         """
 561         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 562         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 563         :param nsr_desc_RO: nsr descriptor from RO
 564         :return: Nothing, LcmException is raised on errors
 565         """
 566         for vnf_index, db_vnfr in db_vnfrs.items():
 567             for vnf_RO in nsr_desc_RO["vnfs"]:
 568                 if vnf_RO["member_vnf_index"] != vnf_index:
 569                     continue
 570                 vnfr_update = {}
 571                 if vnf_RO.get("ip_address"):
 572                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 573                 elif not db_vnfr.get("ip-address"):
 574                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 575
 576                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 577                     vdur_RO_count_index = 0
 578                     if vdur.get("pdu-type"):
 579                         continue
 580                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 581                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 582                             continue
 583                         if vdur["count-index"] != vdur_RO_count_index:
 584                             vdur_RO_count_index += 1
 585                             continue
 586                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 587                         if vdur_RO.get("ip_address"):
 588                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 589                         else:
 590                             vdur["ip-address"] = None
 591                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 592                         vdur["name"] = vdur_RO.get("vim_name")
 593                         vdur["status"] = vdur_RO.get("status")
 594                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 595                         for ifacer in get_iterable(vdur, "interfaces"):
 596                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 597                                 if ifacer["name"] == interface_RO.get("internal_name"):
 598                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 599                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 600                                     break
 601                             else:
 602                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 603                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 604                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 605                         break
 606                     else:
 607                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 608                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 609
 610                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 611                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 612                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 613                             continue
 614                         vld["vim-id"] = net_RO.get("vim_net_id")
 615                         vld["name"] = net_RO.get("vim_name")
 616                         vld["status"] = net_RO.get("status")
 617                         vld["status-detailed"] = net_RO.get("error_msg")
 618                         vnfr_update["vld.{}".format(vld_index)] = vld
 619                         break
 620                     else:
 621                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 622                             vnf_index, vld["id"]))
 623
 624                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 625                 break
 626
 627             else:
 628                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 629
 630     async def instantiate(self, nsr_id, nslcmop_id):
 631
 632         # Try to lock HA task here
 633         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 634         if not task_is_locked_by_me:
 635             return
 636
 637         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 638         self.logger.debug(logging_text + "Enter")
 639         # get all needed from database
 640         start_deploy = time()
 641         db_nsr = None
 642         db_nslcmop = None
 643         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 644         db_nslcmop_update = {}
 645         nslcmop_operation_state = None
 646         db_vnfrs = {}
 647         RO_descriptor_number = 0   # number of descriptors created at RO
 648         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 649         n2vc_info = {}
 650         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 651         exc = None
 652         try:
 653             # wait for any previous tasks in process
 654             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 655
 656             step = "Getting nslcmop={} from db".format(nslcmop_id)
 657             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 658             step = "Getting nsr={} from db".format(nsr_id)
 659             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 660             ns_params = db_nslcmop.get("operationParams")
 661             nsd = db_nsr["nsd"]
 662             nsr_name = db_nsr["name"]   # TODO short-name??
 663
 664             step = "Getting vnfrs from db"
 665             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 666             db_vnfds_ref = {}
 667             db_vnfds = {}
 668             db_vnfds_index = {}
 669             for vnfr in db_vnfrs_list:
 670                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 671                 vnfd_id = vnfr["vnfd-id"]
 672                 vnfd_ref = vnfr["vnfd-ref"]
 673                 if vnfd_id not in db_vnfds:
 674                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 675                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 676                     db_vnfds_ref[vnfd_ref] = vnfd
 677                     db_vnfds[vnfd_id] = vnfd
 678                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 679
 680             # Get or generates the _admin.deployed,VCA list
 681             vca_deployed_list = None
 682             vca_model_name = None
 683             if db_nsr["_admin"].get("deployed"):
 684                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 685                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 686             if vca_deployed_list is None:
 687                 vca_deployed_list = []
 688                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 689                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 690             elif isinstance(vca_deployed_list, dict):
 691                 # maintain backward compatibility. Change a dict to list at database
 692                 vca_deployed_list = list(vca_deployed_list.values())
 693                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 694                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 695
 696             db_nsr_update["detailed-status"] = "creating"
 697             db_nsr_update["operational-status"] = "init"
 698             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 699                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 700                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 701                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 702
 703             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 704             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 705             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 706
 707             # Deploy charms
 708             # The parameters we'll need to deploy a charm
 709             number_to_configure = 0
 710
 711             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 712                 """An inner function to deploy the charm from either ns, vnf or vdu
 713                 For ns both vnf_index and vdu_id are None.
 714                 For vnf only vdu_id is None
 715                 For vdu both vnf_index and vdu_id contain a value
 716                 """
 717                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 718                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 719
 720                 machine_spec = {}
 721                 if native_charm:
 722                     machine_spec["username"] = charm_params.get("username"),
 723                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 724
 725                 # Note: The charm needs to exist on disk at the location
 726                 # specified by charm_path.
 727                 descriptor = vnfd if vnf_index else nsd
 728                 base_folder = descriptor["_admin"]["storage"]
 729                 storage_params = self.fs.get_params()
 730                 charm_path = "{}{}/{}/charms/{}".format(
 731                     storage_params["path"],
 732                     base_folder["folder"],
 733                     base_folder["pkg-dir"],
 734                     proxy_charm
 735                 )
 736
 737                 # ns_name will be ignored in the current version of N2VC
 738                 # but will be implemented for the next point release.
 739                 model_name = nsr_id
 740                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 741                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 742                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 743
 744                 vca_index = len(vca_deployed_list)
 745                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 746                 # 26*26 charm in the same NS
 747                 application_name = application_name[0:48]
 748                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 749                 vca_deployed_ = {
 750                     "member-vnf-index": vnf_index,
 751                     "vdu_id": vdu_id,
 752                     "model": model_name,
 753                     "application": application_name,
 754                     "operational-status": "init",
 755                     "detailed-status": "",
 756                     "step": "initial-deploy",
 757                     "vnfd_id": vnfd_id,
 758                     "vdu_name": vdu_name,
 759                     "vdu_count_index": vdu_count_index,
 760                 }
 761                 vca_deployed_list.append(vca_deployed_)
 762                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 763                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 764
 765                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 766                                                                                                 proxy_charm))
 767                 if not n2vc_info:
 768                     n2vc_info["nsr_id"] = nsr_id
 769                     n2vc_info["nslcmop_id"] = nslcmop_id
 770                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 771                     n2vc_info["lcmOperationType"] = "instantiate"
 772                     n2vc_info["deployed"] = vca_deployed_list
 773                     n2vc_info["db_update"] = db_nsr_update
 774                 task = asyncio.ensure_future(
 775                     self.n2vc.DeployCharms(
 776                         model_name,          # The network service name
 777                         application_name,    # The application name
 778                         descriptor,          # The vnf/nsd descriptor
 779                         charm_path,          # Path to charm
 780                         charm_params,        # Runtime params, like mgmt ip
 781                         machine_spec,        # for native charms only
 782                         self.n2vc_callback,  # Callback for status changes
 783                         n2vc_info,           # Callback parameter
 784                         None,                # Callback parameter (task)
 785                     )
 786                 )
 787                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 788                                                          n2vc_info))
 789                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 790
 791             step = "Looking for needed vnfd to configure with proxy charm"
 792             self.logger.debug(logging_text + step)
 793
 794             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 795                 vnfd_id = c_vnf["vnfd-id-ref"]
 796                 vnf_index = str(c_vnf["member-vnf-index"])
 797                 vnfd = db_vnfds_ref[vnfd_id]
 798
 799                 # Get additional parameters
 800                 vnfr_params = {}
 801                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 802                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 803                 for k, v in vnfr_params.items():
 804                     if isinstance(v, str) and v.startswith("!!yaml "):
 805                         vnfr_params[k] = yaml.safe_load(v[7:])
 806
 807                 step = "deploying proxy charms for configuration"
 808                 # Check if this VNF has a charm configuration
 809                 vnf_config = vnfd.get("vnf-configuration")
 810                 if vnf_config and vnf_config.get("juju"):
 811                     proxy_charm = vnf_config["juju"]["charm"]
 812                     if vnf_config["juju"].get("proxy") is False:
 813                         # native_charm, will be deployed after VM. Skip
 814                         proxy_charm = None
 815
 816                     if proxy_charm:
 817                         if not vca_model_name:
 818                             step = "creating VCA model name '{}'".format(nsr_id)
 819                             self.logger.debug(logging_text + step)
 820                             await self.n2vc.CreateNetworkService(nsr_id)
 821                             vca_model_name = nsr_id
 822                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 823                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 824                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 825                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 826                         charm_params = {
 827                             "user_values": vnfr_params,
 828                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 829                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 830                         }
 831
 832                         # Login to the VCA. If there are multiple calls to login(),
 833                         # subsequent calls will be a nop and return immediately.
 834                         await self.n2vc.login()
 835
 836                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 837                         number_to_configure += 1
 838
 839                 # Deploy charms for each VDU that supports one.
 840                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 841                     vdu_config = vdu.get('vdu-configuration')
 842                     proxy_charm = None
 843
 844                     if vdu_config and vdu_config.get("juju"):
 845                         proxy_charm = vdu_config["juju"]["charm"]
 846                         if vdu_config["juju"].get("proxy") is False:
 847                             # native_charm, will be deployed after VM. Skip
 848                             proxy_charm = None
 849
 850                         if proxy_charm:
 851                             if not vca_model_name:
 852                                 step = "creating VCA model name"
 853                                 await self.n2vc.CreateNetworkService(nsr_id)
 854                                 vca_model_name = nsr_id
 855                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 856                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 857                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 858                                                                                                           vdu["id"])
 859                             await self.n2vc.login()
 860                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 861                             # TODO for the moment only first vdu_id contains a charm deployed
 862                             if vdur["vdu-id-ref"] != vdu["id"]:
 863                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 864                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 865                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 866                             charm_params = {
 867                                 "user_values": vnfr_params,
 868                                 "rw_mgmt_ip": vdur["ip-address"],
 869                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 870                             }
 871                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 872                                          charm_params, n2vc_info)
 873                             number_to_configure += 1
 874
 875             # Check if this NS has a charm configuration
 876
 877             ns_config = nsd.get("ns-configuration")
 878             if ns_config and ns_config.get("juju"):
 879                 proxy_charm = ns_config["juju"]["charm"]
 880                 if ns_config["juju"].get("proxy") is False:
 881                     # native_charm, will be deployed after VM. Skip
 882                     proxy_charm = None
 883
 884                 if proxy_charm:
 885                     step = "deploying proxy charm to configure ns"
 886                     # TODO is NS magmt IP address needed?
 887
 888                     # Get additional parameters
 889                     additional_params = {}
 890                     if db_nsr.get("additionalParamsForNs"):
 891                         additional_params = db_nsr["additionalParamsForNs"].copy()
 892                     for k, v in additional_params.items():
 893                         if isinstance(v, str) and v.startswith("!!yaml "):
 894                             additional_params[k] = yaml.safe_load(v[7:])
 895
 896                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 897                     charm_params = {
 898                         "user_values": additional_params,
 899                         # "rw_mgmt_ip": db_nsr["ip-address"],
 900                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 901                     }
 902
 903                     # Login to the VCA. If there are multiple calls to login(),
 904                     # subsequent calls will be a nop and return immediately.
 905                     await self.n2vc.login()
 906                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 907                     number_to_configure += 1
 908
 909             db_nsr_update["operational-status"] = "running"
 910
 911             # Wait until all charms has reached blocked or active status
 912             step = "waiting proxy charms to be ready"
 913             if number_to_configure:
 914                 # wait until all charms are configured.
 915                 # steps are:
 916                 #       initial-deploy
 917                 #       get-ssh-public-key
 918                 #       generate-ssh-key
 919                 #       retry-get-ssh-public-key
 920                 #       ssh-public-key-obtained
 921                 while time() <= start_deploy + self.total_deploy_timeout:
 922                     if db_nsr_update:
 923                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 924                     if db_nslcmop_update:
 925                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 926
 927                     all_active = True
 928                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 929                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 930                         if vca_deployed["step"] == "initial-deploy":
 931                             if vca_deployed["operational-status"] in ("active", "blocked"):
 932                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 933                                        .format(vca_deployed["member-vnf-index"],
 934                                                vca_deployed["vdu_id"])
 935                                 self.logger.debug(logging_text + step)
 936                                 primitive_id = await self.n2vc.ExecutePrimitive(
 937                                     vca_deployed["model"],
 938                                     vca_deployed["application"],
 939                                     "get-ssh-public-key",
 940                                     None,
 941                                 )
 942                                 vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 943                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 944                                     primitive_id
 945                                 db_nsr_update[database_entry + "operational-status"] =\
 946                                     vca_deployed["operational-status"]
 947                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 948                             primitive_id = vca_deployed["primitive_id"]
 949                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 950                                                                                   primitive_id)
 951                             if primitive_status in ("completed", "failed"):
 952                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 953                                                                                       primitive_id)
 954                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 955                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 956                                         primitive_result.get("pubkey"):
 957                                     ssh_public_key = primitive_result.get("pubkey")
 958                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 959                                         "ssh-public-key-obtained"
 960                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 961                                         ssh_public_key
 962                                     n2vc_key_list.append(ssh_public_key)
 963                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 964                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 965                                     self.logger.debug(logging_text + step)
 966                                 else:  # primitive_status == "failed":
 967                                     if vca_deployed["step"] == "get-ssh-public-key":
 968                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 969                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 970                                                                      vca_deployed["vdu_id"])
 971                                         self.logger.debug(logging_text + step)
 972                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 973                                             "generate-ssh-key"
 974                                         primitive_id = await self.n2vc.ExecutePrimitive(
 975                                             vca_deployed["model"],
 976                                             vca_deployed["application"],
 977                                             "generate-ssh-key",
 978                                             None,
 979                                         )
 980                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 981                                             primitive_id
 982                                     else:  # failed for second time
 983                                         raise LcmException(
 984                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 985
 986                         elif vca_deployed["step"] == "generate-ssh-key":
 987                             primitive_id = vca_deployed["primitive_id"]
 988                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 989                                                                                   primitive_id)
 990                             if primitive_status in ("completed", "failed"):
 991                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 992                                                                                       primitive_id)
 993                                 vca_deployed["primitive_id"] = db_nsr_update[
 994                                     database_entry + "primitive_id"] = None
 995                                 if primitive_status == "completed":
 996                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
 997                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
 998                                                               vca_deployed["vdu_id"])
 999                                     self.logger.debug(logging_text + step)
1000                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1001                                         "retry-get-ssh-public-key"
1002                                     primitive_id = await self.n2vc.ExecutePrimitive(
1003                                         vca_deployed["model"],
1004                                         vca_deployed["application"],
1005                                         "get-ssh-public-key",
1006                                         None,
1007                                     )
1008                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1009                                         primitive_id
1010
1011                                 else:  # primitive_status == "failed":
1012                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1013                                                        .format(primitive_result))
1014
1015                         if vca_deployed["step"] != "ssh-public-key-obtained":
1016                             all_active = False
1017
1018                     if all_active:
1019                         break
1020                     await asyncio.sleep(5)
1021                 else:   # total_deploy_timeout
1022                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1023                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1024
1025             # deploy RO
1026             RO = ROclient.ROClient(self.loop, **self.ro_config)
1027             # get vnfds, instantiate at RO
1028             for c_vnf in nsd.get("constituent-vnfd", ()):
1029                 member_vnf_index = c_vnf["member-vnf-index"]
1030                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1031                 vnfd_ref = vnfd["id"]
1032                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1033                     vnfd_ref, member_vnf_index)
1034                 # self.logger.debug(logging_text + step)
1035                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1036                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1037                 RO_descriptor_number += 1
1038
1039                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1040                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1041                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1042                         break
1043                 else:
1044                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1045                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1046
1047                 # look if present
1048                 RO_update = {"member-vnf-index": member_vnf_index}
1049                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1050                 if vnfd_list:
1051                     RO_update["id"] = vnfd_list[0]["uuid"]
1052                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1053                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1054                 else:
1055                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1056                                            get("additionalParamsForVnf"), nsr_id)
1057                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
1058                     RO_update["id"] = desc["uuid"]
1059                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1060                         vnfd_ref, member_vnf_index, desc["uuid"]))
1061                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1062                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1063                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1064
1065             # create nsd at RO
1066             nsd_ref = nsd["id"]
1067             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1068             # self.logger.debug(logging_text + step)
1069
1070             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1071             RO_descriptor_number += 1
1072             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1073             if nsd_list:
1074                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1075                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1076                     nsd_ref, RO_nsd_uuid))
1077             else:
1078                 nsd_RO = deepcopy(nsd)
1079                 nsd_RO["id"] = RO_osm_nsd_id
1080                 nsd_RO.pop("_id", None)
1081                 nsd_RO.pop("_admin", None)
1082                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1083                     member_vnf_index = c_vnf["member-vnf-index"]
1084                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1085                 for c_vld in nsd_RO.get("vld", ()):
1086                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1087                         member_vnf_index = cp["member-vnf-index-ref"]
1088                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1089
1090                 desc = await RO.create("nsd", descriptor=nsd_RO)
1091                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1092                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1093                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1094             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1095
1096             # Crate ns at RO
1097             # if present use it unless in error status
1098             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1099             if RO_nsr_id:
1100                 try:
1101                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1102                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1103                     desc = await RO.show("ns", RO_nsr_id)
1104                 except ROclient.ROClientException as e:
1105                     if e.http_code != HTTPStatus.NOT_FOUND:
1106                         raise
1107                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1108                 if RO_nsr_id:
1109                     ns_status, ns_status_info = RO.check_ns_status(desc)
1110                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1111                     if ns_status == "ERROR":
1112                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1113                         self.logger.debug(logging_text + step)
1114                         await RO.delete("ns", RO_nsr_id)
1115                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1116             if not RO_nsr_id:
1117                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1118                 # self.logger.debug(logging_text + step)
1119
1120                 # check if VIM is creating and wait  look if previous tasks in process
1121                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1122                 if task_dependency:
1123                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1124                     self.logger.debug(logging_text + step)
1125                     await asyncio.wait(task_dependency, timeout=3600)
1126                 if ns_params.get("vnf"):
1127                     for vnf in ns_params["vnf"]:
1128                         if "vimAccountId" in vnf:
1129                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1130                                                                                         vnf["vimAccountId"])
1131                         if task_dependency:
1132                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1133                             self.logger.debug(logging_text + step)
1134                             await asyncio.wait(task_dependency, timeout=3600)
1135
1136                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1137
1138                 # feature 1429. Add n2vc public key to needed VMs
1139                 n2vc_key = await self.n2vc.GetPublicKey()
1140                 n2vc_key_list.append(n2vc_key)
1141                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1142
1143                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
1144                 desc = await RO.create("ns", descriptor=RO_ns_params,
1145                                        name=db_nsr["name"],
1146                                        scenario=RO_nsd_uuid)
1147                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1148                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1149                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1150                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1151             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1152
1153             # wait until NS is ready
1154             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1155             detailed_status_old = None
1156             self.logger.debug(logging_text + step)
1157
1158             while time() <= start_deploy + self.total_deploy_timeout:
1159                 desc = await RO.show("ns", RO_nsr_id)
1160                 ns_status, ns_status_info = RO.check_ns_status(desc)
1161                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1162                 if ns_status == "ERROR":
1163                     raise ROclient.ROClientException(ns_status_info)
1164                 elif ns_status == "BUILD":
1165                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1166                 elif ns_status == "ACTIVE":
1167                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1168                     try:
1169                         self.ns_update_vnfr(db_vnfrs, desc)
1170                         break
1171                     except LcmExceptionNoMgmtIP:
1172                         pass
1173                 else:
1174                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1175                 if detailed_status != detailed_status_old:
1176                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1177                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1178                 await asyncio.sleep(5, loop=self.loop)
1179             else:   # total_deploy_timeout
1180                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1181
1182             step = "Updating NSR"
1183             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1184
1185             db_nsr_update["operational-status"] = "running"
1186             db_nsr["detailed-status"] = "Configuring vnfr"
1187             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1188
1189             # Configure proxy charms once VMs are up
1190             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1191                 vnf_index = vca_deployed.get("member-vnf-index")
1192                 vdu_id = vca_deployed.get("vdu_id")
1193                 vdu_name = None
1194                 vdu_count_index = None
1195
1196                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1197                                                                                                            vdu_id)
1198                 add_params = {}
1199                 initial_config_primitive_list = []
1200                 if vnf_index:
1201                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1202                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1203                     vnfd = db_vnfds_index[vnf_index]
1204
1205                     if vdu_id:
1206                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1207                             if vdu["id"] == vdu_id:
1208                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1209                                     'initial-config-primitive', [])
1210                                 break
1211                         else:
1212                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1213                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1214                         # TODO for the moment only first vdu_id contains a charm deployed
1215                         if vdur["vdu-id-ref"] != vdu["id"]:
1216                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1217                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1218                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1219                     else:
1220                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1221                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1222                 else:
1223                     if db_nsr.get("additionalParamsForNs"):
1224                         add_params = db_nsr["additionalParamsForNs"].copy()
1225                     for k, v in add_params.items():
1226                         if isinstance(v, str) and v.startswith("!!yaml "):
1227                             add_params[k] = yaml.safe_load(v[7:])
1228                     add_params["rw_mgmt_ip"] = None
1229                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1230
1231                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1232                 initial_config_primitive_list = initial_config_primitive_list.copy()
1233                 if initial_config_primitive_list and vnf_index:
1234                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1235
1236                 for initial_config_primitive in initial_config_primitive_list:
1237                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1238                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1239                                       .format(initial_config_primitive["name"], primitive_params_))
1240                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1241                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1242                         initial_config_primitive["name"],
1243                         primitive_params_,
1244                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1245                         retries_interval=30)
1246                     if primitive_result != "COMPLETED":
1247                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1248                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1249                                                    vca_deployed["vdu_id"], primitive_detail))
1250
1251             # Deploy native charms
1252             step = "Looking for needed vnfd to configure with native charm"
1253             self.logger.debug(logging_text + step)
1254
1255             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1256                 vnfd_id = c_vnf["vnfd-id-ref"]
1257                 vnf_index = str(c_vnf["member-vnf-index"])
1258                 vnfd = db_vnfds_ref[vnfd_id]
1259
1260                 # Get additional parameters
1261                 vnfr_params = {}
1262                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1263                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1264                 for k, v in vnfr_params.items():
1265                     if isinstance(v, str) and v.startswith("!!yaml "):
1266                         vnfr_params[k] = yaml.safe_load(v[7:])
1267
1268                 # Check if this VNF has a charm configuration
1269                 vnf_config = vnfd.get("vnf-configuration")
1270                 if vnf_config and vnf_config.get("juju"):
1271                     native_charm = vnf_config["juju"].get("proxy") is False
1272
1273                     if native_charm:
1274                         if not vca_model_name:
1275                             step = "creating VCA model name '{}'".format(nsr_id)
1276                             self.logger.debug(logging_text + step)
1277                             await self.n2vc.CreateNetworkService(nsr_id)
1278                             vca_model_name = nsr_id
1279                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1280                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1281                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1282                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1283                         charm_params = {
1284                             "user_values": vnfr_params,
1285                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1286                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1287                         }
1288
1289                         # get username
1290                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1291                         #  merged. Meanwhile let's get username from initial-config-primitive
1292                         if vnf_config.get("initial-config-primitive"):
1293                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1294                                 if param["name"] == "ssh-username":
1295                                     charm_params["username"] = param["value"]
1296                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1297                             if vnf_config["config-access"]["ssh-access"].get("required"):
1298                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1299
1300                         # Login to the VCA. If there are multiple calls to login(),
1301                         # subsequent calls will be a nop and return immediately.
1302                         await self.n2vc.login()
1303
1304                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1305                         number_to_configure += 1
1306
1307                 # Deploy charms for each VDU that supports one.
1308                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1309                     vdu_config = vdu.get('vdu-configuration')
1310                     native_charm = False
1311
1312                     if vdu_config and vdu_config.get("juju"):
1313                         native_charm = vdu_config["juju"].get("proxy") is False
1314
1315                         if native_charm:
1316                             if not vca_model_name:
1317                                 step = "creating VCA model name"
1318                                 await self.n2vc.CreateNetworkService(nsr_id)
1319                                 vca_model_name = nsr_id
1320                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1321                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1322                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1323                                                                                                      vdu["id"])
1324                             await self.n2vc.login()
1325                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1326                             # TODO for the moment only first vdu_id contains a charm deployed
1327                             if vdur["vdu-id-ref"] != vdu["id"]:
1328                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1329                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1330                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1331                             charm_params = {
1332                                 "user_values": vnfr_params,
1333                                 "rw_mgmt_ip": vdur["ip-address"],
1334                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1335                             }
1336
1337                             # get username
1338                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1339                             #  merged. Meanwhile let's get username from initial-config-primitive
1340                             if vdu_config.get("initial-config-primitive"):
1341                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1342                                     if param["name"] == "ssh-username":
1343                                         charm_params["username"] = param["value"]
1344                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1345                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1346                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1347                                         "default-user")
1348
1349                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1350                                          charm_params, n2vc_info, native_charm)
1351                             number_to_configure += 1
1352
1353             # Check if this NS has a charm configuration
1354
1355             ns_config = nsd.get("ns-configuration")
1356             if ns_config and ns_config.get("juju"):
1357                 native_charm = ns_config["juju"].get("proxy") is False
1358
1359                 if native_charm:
1360                     step = "deploying native charm to configure ns"
1361                     # TODO is NS magmt IP address needed?
1362
1363                     # Get additional parameters
1364                     additional_params = {}
1365                     if db_nsr.get("additionalParamsForNs"):
1366                         additional_params = db_nsr["additionalParamsForNs"].copy()
1367                     for k, v in additional_params.items():
1368                         if isinstance(v, str) and v.startswith("!!yaml "):
1369                             additional_params[k] = yaml.safe_load(v[7:])
1370
1371                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1372                     charm_params = {
1373                         "user_values": additional_params,
1374                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1375                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1376                     }
1377
1378                     # get username
1379                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1380                     #  merged. Meanwhile let's get username from initial-config-primitive
1381                     if ns_config.get("initial-config-primitive"):
1382                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1383                             if param["name"] == "ssh-username":
1384                                 charm_params["username"] = param["value"]
1385                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1386                         if ns_config["config-access"]["ssh-access"].get("required"):
1387                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1388
1389                     # Login to the VCA. If there are multiple calls to login(),
1390                     # subsequent calls will be a nop and return immediately.
1391                     await self.n2vc.login()
1392                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1393                     number_to_configure += 1
1394
1395             # waiting all charms are ok
1396             configuration_failed = False
1397             if number_to_configure:
1398                 step = "Waiting all charms are active"
1399                 old_status = "configuring: init: {}".format(number_to_configure)
1400                 db_nsr_update["config-status"] = old_status
1401                 db_nsr_update["detailed-status"] = old_status
1402                 db_nslcmop_update["detailed-status"] = old_status
1403
1404                 # wait until all are configured.
1405                 while time() <= start_deploy + self.total_deploy_timeout:
1406                     if db_nsr_update:
1407                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1408                     if db_nslcmop_update:
1409                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1410                     # TODO add a fake task that set n2vc_event after some time
1411                     await n2vc_info["n2vc_event"].wait()
1412                     n2vc_info["n2vc_event"].clear()
1413                     all_active = True
1414                     status_map = {}
1415                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1416                     now = time()
1417                     for vca_deployed in vca_deployed_list:
1418                         vca_status = vca_deployed["operational-status"]
1419                         if vca_status not in status_map:
1420                             # Initialize it
1421                             status_map[vca_status] = 0
1422                         status_map[vca_status] += 1
1423
1424                         if vca_status == "active":
1425                             vca_deployed.pop("time_first_error", None)
1426                             vca_deployed.pop("status_first_error", None)
1427                             continue
1428
1429                         all_active = False
1430                         if vca_status in ("error", "blocked"):
1431                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1432                             # if not first time in this status error
1433                             if not vca_deployed.get("time_first_error"):
1434                                 vca_deployed["time_first_error"] = now
1435                                 continue
1436                         if vca_deployed.get("time_first_error") and \
1437                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1438                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1439                                                    .format(vca_deployed["member-vnf-index"],
1440                                                            vca_deployed["vdu_id"], vca_status,
1441                                                            vca_deployed["detailed-status-error"]))
1442
1443                     if all_active:
1444                         break
1445                     elif n2vc_error_text:
1446                         db_nsr_update["config-status"] = "failed"
1447                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1448                         db_nsr_update["detailed-status"] = error_text
1449                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1450                         db_nslcmop_update["detailed-status"] = error_text
1451                         db_nslcmop_update["statusEnteredTime"] = time()
1452                         configuration_failed = True
1453                         break
1454                     else:
1455                         cs = "configuring: "
1456                         separator = ""
1457                         for status, num in status_map.items():
1458                             cs += separator + "{}: {}".format(status, num)
1459                             separator = ", "
1460                         if old_status != cs:
1461                             db_nsr_update["config-status"] = cs
1462                             db_nsr_update["detailed-status"] = cs
1463                             db_nslcmop_update["detailed-status"] = cs
1464                             old_status = cs
1465                 else:   # total_deploy_timeout
1466                     raise LcmException("Timeout waiting ns to be configured")
1467
1468             if not configuration_failed:
1469                 # all is done
1470                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1471                 db_nslcmop_update["statusEnteredTime"] = time()
1472                 db_nslcmop_update["detailed-status"] = "done"
1473                 db_nsr_update["config-status"] = "configured"
1474                 db_nsr_update["detailed-status"] = "done"
1475
1476             return
1477
1478         except (ROclient.ROClientException, DbException, LcmException) as e:
1479             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1480             exc = e
1481         except asyncio.CancelledError:
1482             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1483             exc = "Operation was cancelled"
1484         except Exception as e:
1485             exc = traceback.format_exc()
1486             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1487                                  exc_info=True)
1488         finally:
1489             if exc:
1490                 if db_nsr:
1491                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1492                     db_nsr_update["operational-status"] = "failed"
1493                 if db_nslcmop:
1494                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1495                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1496                     db_nslcmop_update["statusEnteredTime"] = time()
1497             try:
1498                 if db_nsr:
1499                     db_nsr_update["_admin.nslcmop"] = None
1500                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1501                 if db_nslcmop_update:
1502                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1503             except DbException as e:
1504                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1505             if nslcmop_operation_state:
1506                 try:
1507                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1508                                                                    "operationState": nslcmop_operation_state},
1509                                             loop=self.loop)
1510                 except Exception as e:
1511                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1512
1513             self.logger.debug(logging_text + "Exit")
1514             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1515
1516     async def _destroy_charm(self, model, application):
1517         """
1518         Order N2VC destroy a charm
1519         :param model:
1520         :param application:
1521         :return: True if charm does not exist. False if it exist
1522         """
1523         if not await self.n2vc.HasApplication(model, application):
1524             return True  # Already removed
1525         await self.n2vc.RemoveCharms(model, application)
1526         return False
1527
1528     async def _wait_charm_destroyed(self, model, application, timeout):
1529         """
1530         Wait until charm does not exist
1531         :param model:
1532         :param application:
1533         :param timeout:
1534         :return: True if not exist, False if timeout
1535         """
1536         while True:
1537             if not await self.n2vc.HasApplication(model, application):
1538                 return True
1539             if timeout < 0:
1540                 return False
1541             await asyncio.sleep(10)
1542             timeout -= 10
1543
1544     # Check if this VNFD has a configured terminate action
1545     def _has_terminate_config_primitive(self, vnfd):
1546         vnf_config = vnfd.get("vnf-configuration")
1547         if vnf_config and vnf_config.get("terminate-config-primitive"):
1548             return True
1549         else:
1550             return False
1551
1552     # Get a numerically sorted list of the sequences for this VNFD's terminate action
1553     def _get_terminate_config_primitive_seq_list(self, vnfd):
1554         # No need to check for existing primitive twice, already done before
1555         vnf_config = vnfd.get("vnf-configuration")
1556         seq_list = vnf_config.get("terminate-config-primitive")
1557         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1558         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1559         return seq_list_sorted
1560
1561     @staticmethod
1562     def _create_nslcmop(nsr_id, operation, params):
1563         """
1564         Creates a ns-lcm-opp content to be stored at database.
1565         :param nsr_id: internal id of the instance
1566         :param operation: instantiate, terminate, scale, action, ...
1567         :param params: user parameters for the operation
1568         :return: dictionary following SOL005 format
1569         """
1570         # Raise exception if invalid arguments
1571         if not (nsr_id and operation and params):
1572             raise LcmException(
1573                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1574         now = time()
1575         _id = str(uuid4())
1576         nslcmop = {
1577             "id": _id,
1578             "_id": _id,
1579             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1580             "operationState": "PROCESSING",
1581             "statusEnteredTime": now,
1582             "nsInstanceId": nsr_id,
1583             "lcmOperationType": operation,
1584             "startTime": now,
1585             "isAutomaticInvocation": False,
1586             "operationParams": params,
1587             "isCancelPending": False,
1588             "links": {
1589                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1590                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1591             }
1592         }
1593         return nslcmop
1594
1595     # Create a primitive with params from VNFD
1596     # - Called from terminate() before deleting instance
1597     # - Calls action() to execute the primitive
1598     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1599         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1600         db_vnfds = {}
1601         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1602         # Loop over VNFRs
1603         for vnfr in db_vnfrs_list:
1604             vnfd_id = vnfr["vnfd-id"]
1605             vnf_index = vnfr["member-vnf-index-ref"]
1606             if vnfd_id not in db_vnfds:
1607                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1608                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1609                 db_vnfds[vnfd_id] = vnfd
1610             vnfd = db_vnfds[vnfd_id]
1611             if not self._has_terminate_config_primitive(vnfd):
1612                 continue
1613             # Get the primitive's sorted sequence list
1614             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1615             for seq in seq_list:
1616                 # For each sequence in list, call terminate action
1617                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1618                     vnf_index, seq.get("name"))
1619                 self.logger.debug(logging_text + step)
1620                 # Create the primitive for each sequence
1621                 operation = "action"
1622                 # primitive, i.e. "primitive": "touch"
1623                 primitive = seq.get('name')
1624                 primitive_params = {}
1625                 params = {
1626                     "member_vnf_index": vnf_index,
1627                     "primitive": primitive,
1628                     "primitive_params": primitive_params,
1629                 }
1630                 nslcmop_primitive = self._create_nslcmop(nsr_id, operation, params)
1631                 # Get a copy of db_nslcmop 'admin' part
1632                 db_nslcmop_action = {"_admin": deepcopy(db_nslcmop["_admin"])}
1633                 # Update db_nslcmop with the primitive data
1634                 db_nslcmop_action.update(nslcmop_primitive)
1635                 # Create a new db entry for the created primitive, returns the new ID.
1636                 # (The ID is normally obtained from Kafka.)
1637                 nslcmop_terminate_action_id = self.db.create(
1638                     "nslcmops", db_nslcmop_action)
1639                 # Execute the primitive
1640                 nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1641                     nsr_id, nslcmop_terminate_action_id)
1642                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1643                 nslcmop_operation_states_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1644                 if nslcmop_operation_state not in nslcmop_operation_states_ok:
1645                     raise LcmException(
1646                         "terminate_primitive_action for vnf_member_index={}",
1647                         " primitive={} fails with error {}".format(
1648                             vnf_index, seq.get("name"), nslcmop_operation_state_detail))
1649
1650     async def terminate(self, nsr_id, nslcmop_id):
1651
1652         # Try to lock HA task here
1653         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1654         if not task_is_locked_by_me:
1655             return
1656
1657         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1658         self.logger.debug(logging_text + "Enter")
1659         db_nsr = None
1660         db_nslcmop = None
1661         exc = None
1662         failed_detail = []   # annotates all failed error messages
1663         vca_time_destroy = None   # time of where destroy charm order
1664         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1665         db_nslcmop_update = {}
1666         nslcmop_operation_state = None
1667         autoremove = False  # autoremove after terminated
1668         try:
1669             # wait for any previous tasks in process
1670             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1671
1672             step = "Getting nslcmop={} from db".format(nslcmop_id)
1673             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1674             step = "Getting nsr={} from db".format(nsr_id)
1675             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1676             # nsd = db_nsr["nsd"]
1677             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1678             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1679                 return
1680             # #TODO check if VIM is creating and wait
1681             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1682             # Call internal terminate action
1683             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1684
1685             db_nsr_update["operational-status"] = "terminating"
1686             db_nsr_update["config-status"] = "terminating"
1687
1688             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1689                 vca_model_name = nsr_deployed["VCA-model-name"]
1690                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1691                 self.logger.debug(logging_text + step)
1692                 try:
1693                     await self.n2vc.DestroyNetworkService(vca_model_name)
1694                 except NetworkServiceDoesNotExist:
1695                     pass
1696                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1697                 if nsr_deployed.get("VCA"):
1698                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1699                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1700                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1701             # for backward compatibility if charm have been created with "default"  model name delete one by one
1702             elif nsr_deployed and nsr_deployed.get("VCA"):
1703                 try:
1704                     step = "Scheduling configuration charms removing"
1705                     db_nsr_update["detailed-status"] = "Deleting charms"
1706                     self.logger.debug(logging_text + step)
1707                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1708                     # for backward compatibility
1709                     if isinstance(nsr_deployed["VCA"], dict):
1710                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1711                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1712                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1713
1714                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1715                         if vca_deployed:
1716                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1717                                 vca_deployed.clear()
1718                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1719                             else:
1720                                 vca_time_destroy = time()
1721                 except Exception as e:
1722                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1723
1724             # remove from RO
1725             RO_fail = False
1726             RO = ROclient.ROClient(self.loop, **self.ro_config)
1727
1728             # Delete ns
1729             RO_nsr_id = RO_delete_action = None
1730             if nsr_deployed and nsr_deployed.get("RO"):
1731                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1732                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1733             try:
1734                 if RO_nsr_id:
1735                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1736                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1737                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1738                     self.logger.debug(logging_text + step)
1739                     desc = await RO.delete("ns", RO_nsr_id)
1740                     RO_delete_action = desc["action_id"]
1741                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1742                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1743                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1744                 if RO_delete_action:
1745                     # wait until NS is deleted from VIM
1746                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1747                         format(RO_nsr_id, RO_delete_action)
1748                     detailed_status_old = None
1749                     self.logger.debug(logging_text + step)
1750
1751                     delete_timeout = 20 * 60   # 20 minutes
1752                     while delete_timeout > 0:
1753                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1754                                              extra_item_id=RO_delete_action)
1755                         ns_status, ns_status_info = RO.check_action_status(desc)
1756                         if ns_status == "ERROR":
1757                             raise ROclient.ROClientException(ns_status_info)
1758                         elif ns_status == "BUILD":
1759                             detailed_status = step + "; {}".format(ns_status_info)
1760                         elif ns_status == "ACTIVE":
1761                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1762                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1763                             break
1764                         else:
1765                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1766                         if detailed_status != detailed_status_old:
1767                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1768                                 db_nsr_update["detailed-status"] = detailed_status
1769                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1770                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1771                         await asyncio.sleep(5, loop=self.loop)
1772                         delete_timeout -= 5
1773                     else:  # delete_timeout <= 0:
1774                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1775
1776             except ROclient.ROClientException as e:
1777                 if e.http_code == 404:  # not found
1778                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1779                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1780                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1781                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1782                 elif e.http_code == 409:   # conflict
1783                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1784                     self.logger.debug(logging_text + failed_detail[-1])
1785                     RO_fail = True
1786                 else:
1787                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1788                     self.logger.error(logging_text + failed_detail[-1])
1789                     RO_fail = True
1790
1791             # Delete nsd
1792             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1793                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1794                 try:
1795                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1796                         "Deleting nsd at RO"
1797                     await RO.delete("nsd", RO_nsd_id)
1798                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1799                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1800                 except ROclient.ROClientException as e:
1801                     if e.http_code == 404:  # not found
1802                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1803                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1804                     elif e.http_code == 409:   # conflict
1805                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1806                         self.logger.debug(logging_text + failed_detail[-1])
1807                         RO_fail = True
1808                     else:
1809                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1810                         self.logger.error(logging_text + failed_detail[-1])
1811                         RO_fail = True
1812
1813             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1814                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1815                     if not vnf_deployed or not vnf_deployed["id"]:
1816                         continue
1817                     try:
1818                         RO_vnfd_id = vnf_deployed["id"]
1819                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1820                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1821                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1822                         await RO.delete("vnfd", RO_vnfd_id)
1823                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1824                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1825                     except ROclient.ROClientException as e:
1826                         if e.http_code == 404:  # not found
1827                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1828                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1829                         elif e.http_code == 409:   # conflict
1830                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1831                             self.logger.debug(logging_text + failed_detail[-1])
1832                         else:
1833                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1834                             self.logger.error(logging_text + failed_detail[-1])
1835
1836             # wait until charm deleted
1837             if vca_time_destroy:
1838                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1839                     "Waiting for deletion of configuration charms"
1840                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1841                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1842                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1843                     if not vca_deployed:
1844                         continue
1845                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1846                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1847                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1848                                                             timeout):
1849                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1850                             vca_deployed["application"]))
1851                     else:
1852                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1853
1854             if failed_detail:
1855                 self.logger.error(logging_text + " ;".join(failed_detail))
1856                 db_nsr_update["operational-status"] = "failed"
1857                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1858                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1859                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1860                 db_nslcmop_update["statusEnteredTime"] = time()
1861             else:
1862                 db_nsr_update["operational-status"] = "terminated"
1863                 db_nsr_update["detailed-status"] = "Done"
1864                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1865                 db_nslcmop_update["detailed-status"] = "Done"
1866                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1867                 db_nslcmop_update["statusEnteredTime"] = time()
1868                 if db_nslcmop["operationParams"].get("autoremove"):
1869                     autoremove = True
1870
1871         except (ROclient.ROClientException, DbException, LcmException) as e:
1872             self.logger.error(logging_text + "Exit Exception {}".format(e))
1873             exc = e
1874         except asyncio.CancelledError:
1875             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1876             exc = "Operation was cancelled"
1877         except Exception as e:
1878             exc = traceback.format_exc()
1879             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1880         finally:
1881             if exc and db_nslcmop:
1882                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1883                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1884                 db_nslcmop_update["statusEnteredTime"] = time()
1885             try:
1886                 if db_nslcmop and db_nslcmop_update:
1887                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1888                 if db_nsr:
1889                     db_nsr_update["_admin.nslcmop"] = None
1890                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1891             except DbException as e:
1892                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1893             if nslcmop_operation_state:
1894                 try:
1895                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1896                                                                  "operationState": nslcmop_operation_state,
1897                                                                  "autoremove": autoremove},
1898                                             loop=self.loop)
1899                 except Exception as e:
1900                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1901             self.logger.debug(logging_text + "Exit")
1902             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1903
1904     @staticmethod
1905     def _map_primitive_params(primitive_desc, params, instantiation_params):
1906         """
1907         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1908         The default-value is used. If it is between < > it look for a value at instantiation_params
1909         :param primitive_desc: portion of VNFD/NSD that describes primitive
1910         :param params: Params provided by user
1911         :param instantiation_params: Instantiation params provided by user
1912         :return: a dictionary with the calculated params
1913         """
1914         calculated_params = {}
1915         for parameter in primitive_desc.get("parameter", ()):
1916             param_name = parameter["name"]
1917             if param_name in params:
1918                 calculated_params[param_name] = params[param_name]
1919             elif "default-value" in parameter or "value" in parameter:
1920                 if "value" in parameter:
1921                     calculated_params[param_name] = parameter["value"]
1922                 else:
1923                     calculated_params[param_name] = parameter["default-value"]
1924                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1925                         and calculated_params[param_name].endswith(">"):
1926                     if calculated_params[param_name][1:-1] in instantiation_params:
1927                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1928                     else:
1929                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1930                                            format(parameter["default-value"], primitive_desc["name"]))
1931             else:
1932                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1933                                    format(param_name, primitive_desc["name"]))
1934
1935             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1936                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1937                                                                width=256)
1938             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1939                 calculated_params[param_name] = calculated_params[param_name][7:]
1940         return calculated_params
1941
1942     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1943                                     primitive, primitive_params, retries=0, retries_interval=30):
1944         start_primitive_time = time()
1945         try:
1946             for vca_deployed in db_deployed["VCA"]:
1947                 if not vca_deployed:
1948                     continue
1949                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1950                     continue
1951                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1952                     continue
1953                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1954                     continue
1955                 break
1956             else:
1957                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1958                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1959             model_name = vca_deployed.get("model")
1960             application_name = vca_deployed.get("application")
1961             if not model_name or not application_name:
1962                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1963                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1964                                                                        vdu_count_index))
1965             # if vca_deployed["operational-status"] != "active":
1966             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1967             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1968             callback = None  # self.n2vc_callback
1969             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1970             await self.n2vc.login()
1971             if primitive == "config":
1972                 primitive_params = {"params": primitive_params}
1973             while retries >= 0:
1974                 primitive_id = await self.n2vc.ExecutePrimitive(
1975                     model_name,
1976                     application_name,
1977                     primitive,
1978                     callback,
1979                     *callback_args,
1980                     **primitive_params
1981                 )
1982                 while time() - start_primitive_time < self.timeout_primitive:
1983                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1984                     if primitive_result_ in ("completed", "failed"):
1985                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1986                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1987                         break
1988                     elif primitive_result_ is None and primitive == "config":
1989                         primitive_result = "COMPLETED"
1990                         detailed_result = None
1991                         break
1992                     else:  # ("running", "pending", None):
1993                         pass
1994                     await asyncio.sleep(5)
1995                 else:
1996                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1997                 if primitive_result == "COMPLETED":
1998                     break
1999                 retries -= 1
2000                 if retries >= 0:
2001                     await asyncio.sleep(retries_interval)
2002
2003             return primitive_result, detailed_result
2004         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2005             return "FAILED", str(e)
2006
2007     async def action(self, nsr_id, nslcmop_id):
2008
2009         # Try to lock HA task here
2010         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2011         if not task_is_locked_by_me:
2012             return
2013
2014         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2015         self.logger.debug(logging_text + "Enter")
2016         # get all needed from database
2017         db_nsr = None
2018         db_nslcmop = None
2019         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2020         db_nslcmop_update = {}
2021         nslcmop_operation_state = None
2022         nslcmop_operation_state_detail = None
2023         exc = None
2024         try:
2025             # wait for any previous tasks in process
2026             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2027
2028             step = "Getting information from database"
2029             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2030             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2031
2032             nsr_deployed = db_nsr["_admin"].get("deployed")
2033             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2034             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2035             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2036             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2037
2038             if vnf_index:
2039                 step = "Getting vnfr from database"
2040                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2041                 step = "Getting vnfd from database"
2042                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2043             else:
2044                 if db_nsr.get("nsd"):
2045                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2046                 else:
2047                     step = "Getting nsd from database"
2048                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2049
2050             # for backward compatibility
2051             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2052                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2053                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2054                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2055
2056             primitive = db_nslcmop["operationParams"]["primitive"]
2057             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2058
2059             # look for primitive
2060             config_primitive_desc = None
2061             if vdu_id:
2062                 for vdu in get_iterable(db_vnfd, "vdu"):
2063                     if vdu_id == vdu["id"]:
2064                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2065                             if config_primitive["name"] == primitive:
2066                                 config_primitive_desc = config_primitive
2067                                 break
2068             elif vnf_index:
2069                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2070                     if config_primitive["name"] == primitive:
2071                         config_primitive_desc = config_primitive
2072                         break
2073             else:
2074                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2075                     if config_primitive["name"] == primitive:
2076                         config_primitive_desc = config_primitive
2077                         break
2078
2079             if not config_primitive_desc:
2080                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2081                                    format(primitive))
2082
2083             desc_params = {}
2084             if vnf_index:
2085                 if db_vnfr.get("additionalParamsForVnf"):
2086                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2087             else:
2088                 if db_nsr.get("additionalParamsForVnf"):
2089                     desc_params.update(db_nsr["additionalParamsForNs"])
2090
2091             # TODO check if ns is in a proper status
2092             result, result_detail = await self._ns_execute_primitive(
2093                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2094                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2095             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2096             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2097             db_nslcmop_update["statusEnteredTime"] = time()
2098             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2099             return  # database update is called inside finally
2100
2101         except (DbException, LcmException) as e:
2102             self.logger.error(logging_text + "Exit Exception {}".format(e))
2103             exc = e
2104         except asyncio.CancelledError:
2105             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2106             exc = "Operation was cancelled"
2107         except Exception as e:
2108             exc = traceback.format_exc()
2109             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2110         finally:
2111             if exc and db_nslcmop:
2112                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2113                     "FAILED {}: {}".format(step, exc)
2114                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2115                 db_nslcmop_update["statusEnteredTime"] = time()
2116             try:
2117                 if db_nslcmop_update:
2118                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2119                 if db_nsr:
2120                     db_nsr_update["_admin.nslcmop"] = None
2121                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2122             except DbException as e:
2123                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2124             self.logger.debug(logging_text + "Exit")
2125             if nslcmop_operation_state:
2126                 try:
2127                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2128                                                                "operationState": nslcmop_operation_state},
2129                                             loop=self.loop)
2130                 except Exception as e:
2131                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2132             self.logger.debug(logging_text + "Exit")
2133             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2134             return nslcmop_operation_state, nslcmop_operation_state_detail
2135
2136     async def scale(self, nsr_id, nslcmop_id):
2137
2138         # Try to lock HA task here
2139         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2140         if not task_is_locked_by_me:
2141             return
2142
2143         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2144         self.logger.debug(logging_text + "Enter")
2145         # get all needed from database
2146         db_nsr = None
2147         db_nslcmop = None
2148         db_nslcmop_update = {}
2149         nslcmop_operation_state = None
2150         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2151         exc = None
2152         # in case of error, indicates what part of scale was failed to put nsr at error status
2153         scale_process = None
2154         old_operational_status = ""
2155         old_config_status = ""
2156         vnfr_scaled = False
2157         try:
2158             # wait for any previous tasks in process
2159             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2160
2161             step = "Getting nslcmop from database"
2162             self.logger.debug(step + " after having waited for previous tasks to be completed")
2163             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2164             step = "Getting nsr from database"
2165             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2166
2167             old_operational_status = db_nsr["operational-status"]
2168             old_config_status = db_nsr["config-status"]
2169             step = "Parsing scaling parameters"
2170             # self.logger.debug(step)
2171             db_nsr_update["operational-status"] = "scaling"
2172             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2173             nsr_deployed = db_nsr["_admin"].get("deployed")
2174             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2175             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2176             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2177             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2178             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2179
2180             # for backward compatibility
2181             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2182                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2183                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2184                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2185
2186             step = "Getting vnfr from database"
2187             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2188             step = "Getting vnfd from database"
2189             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2190
2191             step = "Getting scaling-group-descriptor"
2192             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2193                 if scaling_descriptor["name"] == scaling_group:
2194                     break
2195             else:
2196                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2197                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2198
2199             # cooldown_time = 0
2200             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2201             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2202             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2203             #         break
2204
2205             # TODO check if ns is in a proper status
2206             step = "Sending scale order to RO"
2207             nb_scale_op = 0
2208             if not db_nsr["_admin"].get("scaling-group"):
2209                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2210                 admin_scale_index = 0
2211             else:
2212                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2213                     if admin_scale_info["name"] == scaling_group:
2214                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2215                         break
2216                 else:  # not found, set index one plus last element and add new entry with the name
2217                     admin_scale_index += 1
2218                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2219             RO_scaling_info = []
2220             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2221             if scaling_type == "SCALE_OUT":
2222                 # count if max-instance-count is reached
2223                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
2224                     max_instance_count = int(scaling_descriptor["max-instance-count"])
2225
2226                     # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(scaling_descriptor["max-instance-count"]))
2227                     if nb_scale_op >= max_instance_count:
2228                         raise LcmException("reached the limit of {} (max-instance-count) "
2229                                            "scaling-out operations for the "
2230                                            "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2231
2232                 nb_scale_op += 1
2233                 vdu_scaling_info["scaling_direction"] = "OUT"
2234                 vdu_scaling_info["vdu-create"] = {}
2235                 for vdu_scale_info in scaling_descriptor["vdu"]:
2236                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2237                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2238                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2239
2240             elif scaling_type == "SCALE_IN":
2241                 # count if min-instance-count is reached
2242                 min_instance_count = 0
2243                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2244                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2245                 if nb_scale_op <= min_instance_count:
2246                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2247                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2248                 nb_scale_op -= 1
2249                 vdu_scaling_info["scaling_direction"] = "IN"
2250                 vdu_scaling_info["vdu-delete"] = {}
2251                 for vdu_scale_info in scaling_descriptor["vdu"]:
2252                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2253                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2254                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2255
2256             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2257             vdu_create = vdu_scaling_info.get("vdu-create")
2258             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2259             if vdu_scaling_info["scaling_direction"] == "IN":
2260                 for vdur in reversed(db_vnfr["vdur"]):
2261                     if vdu_delete.get(vdur["vdu-id-ref"]):
2262                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2263                         vdu_scaling_info["vdu"].append({
2264                             "name": vdur["name"],
2265                             "vdu_id": vdur["vdu-id-ref"],
2266                             "interface": []
2267                         })
2268                         for interface in vdur["interfaces"]:
2269                             vdu_scaling_info["vdu"][-1]["interface"].append({
2270                                 "name": interface["name"],
2271                                 "ip_address": interface["ip-address"],
2272                                 "mac_address": interface.get("mac-address"),
2273                             })
2274                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2275
2276             # execute primitive service PRE-SCALING
2277             step = "Executing pre-scale vnf-config-primitive"
2278             if scaling_descriptor.get("scaling-config-action"):
2279                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2280                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2281                             and scaling_type == "SCALE_IN":
2282                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2283                         step = db_nslcmop_update["detailed-status"] = \
2284                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2285
2286                         # look for primitive
2287                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2288                             if config_primitive["name"] == vnf_config_primitive:
2289                                 break
2290                         else:
2291                             raise LcmException(
2292                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2293                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2294                                 "primitive".format(scaling_group, config_primitive))
2295
2296                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2297                         if db_vnfr.get("additionalParamsForVnf"):
2298                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2299
2300                         scale_process = "VCA"
2301                         db_nsr_update["config-status"] = "configuring pre-scaling"
2302                         result, result_detail = await self._ns_execute_primitive(
2303                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2304                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2305                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2306                             vnf_config_primitive, result, result_detail))
2307                         if result == "FAILED":
2308                             raise LcmException(result_detail)
2309                         db_nsr_update["config-status"] = old_config_status
2310                         scale_process = None
2311
2312             if RO_scaling_info:
2313                 scale_process = "RO"
2314                 RO = ROclient.ROClient(self.loop, **self.ro_config)
2315                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2316                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2317                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2318                 # wait until ready
2319                 RO_nslcmop_id = RO_desc["instance_action_id"]
2320                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2321
2322                 RO_task_done = False
2323                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2324                 detailed_status_old = None
2325                 self.logger.debug(logging_text + step)
2326
2327                 deployment_timeout = 1 * 3600   # One hour
2328                 while deployment_timeout > 0:
2329                     if not RO_task_done:
2330                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2331                                              extra_item_id=RO_nslcmop_id)
2332                         ns_status, ns_status_info = RO.check_action_status(desc)
2333                         if ns_status == "ERROR":
2334                             raise ROclient.ROClientException(ns_status_info)
2335                         elif ns_status == "BUILD":
2336                             detailed_status = step + "; {}".format(ns_status_info)
2337                         elif ns_status == "ACTIVE":
2338                             RO_task_done = True
2339                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2340                             self.logger.debug(logging_text + step)
2341                         else:
2342                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2343                     else:
2344                         desc = await RO.show("ns", RO_nsr_id)
2345                         ns_status, ns_status_info = RO.check_ns_status(desc)
2346                         if ns_status == "ERROR":
2347                             raise ROclient.ROClientException(ns_status_info)
2348                         elif ns_status == "BUILD":
2349                             detailed_status = step + "; {}".format(ns_status_info)
2350                         elif ns_status == "ACTIVE":
2351                             step = detailed_status = \
2352                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2353                             if not vnfr_scaled:
2354                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2355                                 vnfr_scaled = True
2356                             try:
2357                                 desc = await RO.show("ns", RO_nsr_id)
2358                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2359                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2360                                 break
2361                             except LcmExceptionNoMgmtIP:
2362                                 pass
2363                         else:
2364                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2365                     if detailed_status != detailed_status_old:
2366                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2367                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2368
2369                     await asyncio.sleep(5, loop=self.loop)
2370                     deployment_timeout -= 5
2371                 if deployment_timeout <= 0:
2372                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2373
2374                 # update VDU_SCALING_INFO with the obtained ip_addresses
2375                 if vdu_scaling_info["scaling_direction"] == "OUT":
2376                     for vdur in reversed(db_vnfr["vdur"]):
2377                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2378                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2379                             vdu_scaling_info["vdu"].append({
2380                                 "name": vdur["name"],
2381                                 "vdu_id": vdur["vdu-id-ref"],
2382                                 "interface": []
2383                             })
2384                             for interface in vdur["interfaces"]:
2385                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2386                                     "name": interface["name"],
2387                                     "ip_address": interface["ip-address"],
2388                                     "mac_address": interface.get("mac-address"),
2389                                 })
2390                     del vdu_scaling_info["vdu-create"]
2391
2392             scale_process = None
2393             if db_nsr_update:
2394                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2395
2396             # execute primitive service POST-SCALING
2397             step = "Executing post-scale vnf-config-primitive"
2398             if scaling_descriptor.get("scaling-config-action"):
2399                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2400                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2401                             and scaling_type == "SCALE_OUT":
2402                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2403                         step = db_nslcmop_update["detailed-status"] = \
2404                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2405
2406                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2407                         if db_vnfr.get("additionalParamsForVnf"):
2408                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2409
2410                         # look for primitive
2411                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2412                             if config_primitive["name"] == vnf_config_primitive:
2413                                 break
2414                         else:
2415                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2416                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2417                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2418                                                                                                      config_primitive))
2419                         scale_process = "VCA"
2420                         db_nsr_update["config-status"] = "configuring post-scaling"
2421
2422                         result, result_detail = await self._ns_execute_primitive(
2423                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2424                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2425                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2426                             vnf_config_primitive, result, result_detail))
2427                         if result == "FAILED":
2428                             raise LcmException(result_detail)
2429                         db_nsr_update["config-status"] = old_config_status
2430                         scale_process = None
2431
2432             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2433             db_nslcmop_update["statusEnteredTime"] = time()
2434             db_nslcmop_update["detailed-status"] = "done"
2435             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2436             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2437                 else old_operational_status
2438             db_nsr_update["config-status"] = old_config_status
2439             return
2440         except (ROclient.ROClientException, DbException, LcmException) as e:
2441             self.logger.error(logging_text + "Exit Exception {}".format(e))
2442             exc = e
2443         except asyncio.CancelledError:
2444             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2445             exc = "Operation was cancelled"
2446         except Exception as e:
2447             exc = traceback.format_exc()
2448             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2449         finally:
2450             if exc:
2451                 if db_nslcmop:
2452                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2453                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2454                     db_nslcmop_update["statusEnteredTime"] = time()
2455                 if db_nsr:
2456                     db_nsr_update["operational-status"] = old_operational_status
2457                     db_nsr_update["config-status"] = old_config_status
2458                     db_nsr_update["detailed-status"] = ""
2459                     db_nsr_update["_admin.nslcmop"] = None
2460                     if scale_process:
2461                         if "VCA" in scale_process:
2462                             db_nsr_update["config-status"] = "failed"
2463                         if "RO" in scale_process:
2464                             db_nsr_update["operational-status"] = "failed"
2465                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2466                                                                                                      exc)
2467             try:
2468                 if db_nslcmop and db_nslcmop_update:
2469                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2470                 if db_nsr:
2471                     db_nsr_update["_admin.nslcmop"] = None
2472                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2473             except DbException as e:
2474                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2475             if nslcmop_operation_state:
2476                 try:
2477                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2478                                                              "operationState": nslcmop_operation_state},
2479                                             loop=self.loop)
2480                     # if cooldown_time:
2481                     #     await asyncio.sleep(cooldown_time)
2482                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2483                 except Exception as e:
2484                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2485             self.logger.debug(logging_text + "Exit")
2486             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")