osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105
 106     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 107         """
 108         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 109         :param vnfd: input vnfd
 110         :param new_id: overrides vnf id if provided
 111         :param additionalParams: Instantiation params for VNFs provided
 112         :param nsrId: Id of the NSR
 113         :return: copy of vnfd
 114         """
 115         try:
 116             vnfd_RO = deepcopy(vnfd)
 117             # remove unused by RO configuration, monitoring, scaling and internal keys
 118             vnfd_RO.pop("_id", None)
 119             vnfd_RO.pop("_admin", None)
 120             vnfd_RO.pop("vnf-configuration", None)
 121             vnfd_RO.pop("monitoring-param", None)
 122             vnfd_RO.pop("scaling-group-descriptor", None)
 123             if new_id:
 124                 vnfd_RO["id"] = new_id
 125
 126             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 127             for vdu in get_iterable(vnfd_RO, "vdu"):
 128                 cloud_init_file = None
 129                 if vdu.get("cloud-init-file"):
 130                     base_folder = vnfd["_admin"]["storage"]
 131                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 132                                                                    vdu["cloud-init-file"])
 133                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 134                         cloud_init_content = ci_file.read()
 135                     vdu.pop("cloud-init-file", None)
 136                 elif vdu.get("cloud-init"):
 137                     cloud_init_content = vdu["cloud-init"]
 138                 else:
 139                     continue
 140
 141                 env = Environment()
 142                 ast = env.parse(cloud_init_content)
 143                 mandatory_vars = meta.find_undeclared_variables(ast)
 144                 if mandatory_vars:
 145                     for var in mandatory_vars:
 146                         if not additionalParams or var not in additionalParams.keys():
 147                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 148                                                "file, must be provided in the instantiation parameters inside the "
 149                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 150                 template = Template(cloud_init_content)
 151                 cloud_init_content = template.render(additionalParams or {})
 152                 vdu["cloud-init"] = cloud_init_content
 153
 154             return vnfd_RO
 155         except FsException as e:
 156             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 157                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 158         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 159             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 160                                format(vnfd["id"], vdu["id"], e))
 161
 162     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 163         """
 164         Callback both for charm status change and task completion
 165         :param model_name: Charm model name
 166         :param application_name: Charm application name
 167         :param status: Can be
 168             - blocked: The unit needs manual intervention
 169             - maintenance: The unit is actively deploying/configuring
 170             - waiting: The unit is waiting for another charm to be ready
 171             - active: The unit is deployed, configured, and ready
 172             - error: The charm has failed and needs attention.
 173             - terminated: The charm has been destroyed
 174             - removing,
 175             - removed
 176         :param message: detailed message error
 177         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 178             nsr_id:
 179             nslcmop_id:
 180             lcmOperationType: currently "instantiate"
 181             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 182             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 183             n2vc_event: event used to notify instantiation task that some change has been produced
 184         :param task: None for charm status change, or task for completion task callback
 185         :return:
 186         """
 187         try:
 188             nsr_id = n2vc_info["nsr_id"]
 189             deployed = n2vc_info["deployed"]
 190             db_nsr_update = n2vc_info["db_update"]
 191             nslcmop_id = n2vc_info["nslcmop_id"]
 192             ns_operation = n2vc_info["lcmOperationType"]
 193             n2vc_event = n2vc_info["n2vc_event"]
 194             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 195                                                                                     application_name)
 196             for vca_index, vca_deployed in enumerate(deployed):
 197                 if not vca_deployed:
 198                     continue
 199                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 200                     break
 201             else:
 202                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 203                                   format(model_name))
 204                 return
 205             if task:
 206                 if task.cancelled():
 207                     self.logger.debug(logging_text + " task Cancelled")
 208                     vca_deployed['operational-status'] = "error"
 209                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 210                     vca_deployed['detailed-status'] = "Task Cancelled"
 211                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 212
 213                 elif task.done():
 214                     exc = task.exception()
 215                     if exc:
 216                         self.logger.error(logging_text + " task Exception={}".format(exc))
 217                         vca_deployed['operational-status'] = "error"
 218                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 219                         vca_deployed['detailed-status'] = str(exc)
 220                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 221                     else:
 222                         self.logger.debug(logging_text + " task Done")
 223                         # task is Done, but callback is still ongoing. So ignore
 224                         return
 225             elif status:
 226                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 227                 if vca_deployed['operational-status'] == status:
 228                     return  # same status, ignore
 229                 vca_deployed['operational-status'] = status
 230                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 231                 vca_deployed['detailed-status'] = str(message)
 232                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 233             else:
 234                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 235                 return
 236             # wake up instantiate task
 237             n2vc_event.set()
 238         except Exception as e:
 239             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 240
 241     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 242         """
 243         Creates a RO ns descriptor from OSM ns_instantiate params
 244         :param ns_params: OSM instantiate params
 245         :return: The RO ns descriptor
 246         """
 247         vim_2_RO = {}
 248         wim_2_RO = {}
 249         # TODO feature 1417: Check that no instantiation is set over PDU
 250         # check if PDU forces a concrete vim-network-id and add it
 251         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 252
 253         def vim_account_2_RO(vim_account):
 254             if vim_account in vim_2_RO:
 255                 return vim_2_RO[vim_account]
 256
 257             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 258             if db_vim["_admin"]["operationalState"] != "ENABLED":
 259                 raise LcmException("VIM={} is not available. operationalState={}".format(
 260                     vim_account, db_vim["_admin"]["operationalState"]))
 261             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 262             vim_2_RO[vim_account] = RO_vim_id
 263             return RO_vim_id
 264
 265         def wim_account_2_RO(wim_account):
 266             if isinstance(wim_account, str):
 267                 if wim_account in wim_2_RO:
 268                     return wim_2_RO[wim_account]
 269
 270                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 271                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 272                     raise LcmException("WIM={} is not available. operationalState={}".format(
 273                         wim_account, db_wim["_admin"]["operationalState"]))
 274                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 275                 wim_2_RO[wim_account] = RO_wim_id
 276                 return RO_wim_id
 277             else:
 278                 return wim_account
 279
 280         def ip_profile_2_RO(ip_profile):
 281             RO_ip_profile = deepcopy((ip_profile))
 282             if "dns-server" in RO_ip_profile:
 283                 if isinstance(RO_ip_profile["dns-server"], list):
 284                     RO_ip_profile["dns-address"] = []
 285                     for ds in RO_ip_profile.pop("dns-server"):
 286                         RO_ip_profile["dns-address"].append(ds['address'])
 287                 else:
 288                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 289             if RO_ip_profile.get("ip-version") == "ipv4":
 290                 RO_ip_profile["ip-version"] = "IPv4"
 291             if RO_ip_profile.get("ip-version") == "ipv6":
 292                 RO_ip_profile["ip-version"] = "IPv6"
 293             if "dhcp-params" in RO_ip_profile:
 294                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 295             return RO_ip_profile
 296
 297         if not ns_params:
 298             return None
 299         RO_ns_params = {
 300             # "name": ns_params["nsName"],
 301             # "description": ns_params.get("nsDescription"),
 302             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 303             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 304             # "scenario": ns_params["nsdId"],
 305         }
 306         if n2vc_key_list:
 307             for vnfd_ref, vnfd in vnfd_dict.items():
 308                 vdu_needed_access = []
 309                 mgmt_cp = None
 310                 if vnfd.get("vnf-configuration"):
 311                     if vnfd.get("mgmt-interface"):
 312                         if vnfd["mgmt-interface"].get("vdu-id"):
 313                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 314                         elif vnfd["mgmt-interface"].get("cp"):
 315                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 316
 317                 for vdu in vnfd.get("vdu", ()):
 318                     if vdu.get("vdu-configuration"):
 319                         vdu_needed_access.append(vdu["id"])
 320                     elif mgmt_cp:
 321                         for vdu_interface in vdu.get("interface"):
 322                             if vdu_interface.get("external-connection-point-ref") and \
 323                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 324                                 vdu_needed_access.append(vdu["id"])
 325                                 mgmt_cp = None
 326                                 break
 327
 328                 if vdu_needed_access:
 329                     for vnf_member in nsd.get("constituent-vnfd"):
 330                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 331                             continue
 332                         for vdu in vdu_needed_access:
 333                             populate_dict(RO_ns_params,
 334                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 335                                           n2vc_key_list)
 336
 337         if ns_params.get("vduImage"):
 338             RO_ns_params["vduImage"] = ns_params["vduImage"]
 339
 340         if ns_params.get("ssh_keys"):
 341             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 342         for vnf_params in get_iterable(ns_params, "vnf"):
 343             for constituent_vnfd in nsd["constituent-vnfd"]:
 344                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 345                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 346                     break
 347             else:
 348                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 349                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 350             if vnf_params.get("vimAccountId"):
 351                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 352                               vim_account_2_RO(vnf_params["vimAccountId"]))
 353
 354             for vdu_params in get_iterable(vnf_params, "vdu"):
 355                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 356                 if vdu_params.get("volume"):
 357                     for volume_params in vdu_params["volume"]:
 358                         if volume_params.get("vim-volume-id"):
 359                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 360                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 361                                           volume_params["vim-volume-id"])
 362                 if vdu_params.get("interface"):
 363                     for interface_params in vdu_params["interface"]:
 364                         if interface_params.get("ip-address"):
 365                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 366                                                          vdu_params["id"], "interfaces", interface_params["name"],
 367                                                          "ip_address"),
 368                                           interface_params["ip-address"])
 369                         if interface_params.get("mac-address"):
 370                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 371                                                          vdu_params["id"], "interfaces", interface_params["name"],
 372                                                          "mac_address"),
 373                                           interface_params["mac-address"])
 374                         if interface_params.get("floating-ip-required"):
 375                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 376                                                          vdu_params["id"], "interfaces", interface_params["name"],
 377                                                          "floating-ip"),
 378                                           interface_params["floating-ip-required"])
 379
 380             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 381                 if internal_vld_params.get("vim-network-name"):
 382                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 383                                                  internal_vld_params["name"], "vim-network-name"),
 384                                   internal_vld_params["vim-network-name"])
 385                 if internal_vld_params.get("vim-network-id"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-id"),
 388                                   internal_vld_params["vim-network-id"])
 389                 if internal_vld_params.get("ip-profile"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "ip-profile"),
 392                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 393
 394                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 395                     # look for interface
 396                     iface_found = False
 397                     for vdu_descriptor in vnf_descriptor["vdu"]:
 398                         for vdu_interface in vdu_descriptor["interface"]:
 399                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 400                                 if icp_params.get("ip-address"):
 401                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 402                                                                  vdu_descriptor["id"], "interfaces",
 403                                                                  vdu_interface["name"], "ip_address"),
 404                                                   icp_params["ip-address"])
 405
 406                                 if icp_params.get("mac-address"):
 407                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 408                                                                  vdu_descriptor["id"], "interfaces",
 409                                                                  vdu_interface["name"], "mac_address"),
 410                                                   icp_params["mac-address"])
 411                                 iface_found = True
 412                                 break
 413                         if iface_found:
 414                             break
 415                     else:
 416                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 417                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 418                                            "connection-point".format(vnf_params["member-vnf-index"],
 419                                                                      icp_params["id-ref"]))
 420
 421         for vld_params in get_iterable(ns_params, "vld"):
 422             if "ip-profile" in vld_params:
 423                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 424                               ip_profile_2_RO(vld_params["ip-profile"]))
 425
 426             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 428                               wim_account_2_RO(vld_params["wimAccountId"])),
 429             if vld_params.get("vim-network-name"):
 430                 RO_vld_sites = []
 431                 if isinstance(vld_params["vim-network-name"], dict):
 432                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 433                         RO_vld_sites.append({
 434                             "netmap-use": vim_net,
 435                             "datacenter": vim_account_2_RO(vim_account)
 436                         })
 437                 else:  # isinstance str
 438                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 439                 if RO_vld_sites:
 440                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 441             if vld_params.get("vim-network-id"):
 442                 RO_vld_sites = []
 443                 if isinstance(vld_params["vim-network-id"], dict):
 444                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 445                         RO_vld_sites.append({
 446                             "netmap-use": vim_net,
 447                             "datacenter": vim_account_2_RO(vim_account)
 448                         })
 449                 else:  # isinstance str
 450                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 451                 if RO_vld_sites:
 452                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 453             if vld_params.get("ns-net"):
 454                 if isinstance(vld_params["ns-net"], dict):
 455                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 456                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 457                 if RO_vld_ns_net:
 458                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 459             if "vnfd-connection-point-ref" in vld_params:
 460                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 461                     # look for interface
 462                     for constituent_vnfd in nsd["constituent-vnfd"]:
 463                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 464                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 465                             break
 466                     else:
 467                         raise LcmException(
 468                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 469                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 470                     match_cp = False
 471                     for vdu_descriptor in vnf_descriptor["vdu"]:
 472                         for interface_descriptor in vdu_descriptor["interface"]:
 473                             if interface_descriptor.get("external-connection-point-ref") == \
 474                                     cp_params["vnfd-connection-point-ref"]:
 475                                 match_cp = True
 476                                 break
 477                         if match_cp:
 478                             break
 479                     else:
 480                         raise LcmException(
 481                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 482                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 483                                 cp_params["member-vnf-index-ref"],
 484                                 cp_params["vnfd-connection-point-ref"],
 485                                 vnf_descriptor["id"]))
 486                     if cp_params.get("ip-address"):
 487                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 488                                                      vdu_descriptor["id"], "interfaces",
 489                                                      interface_descriptor["name"], "ip_address"),
 490                                       cp_params["ip-address"])
 491                     if cp_params.get("mac-address"):
 492                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 493                                                      vdu_descriptor["id"], "interfaces",
 494                                                      interface_descriptor["name"], "mac_address"),
 495                                       cp_params["mac-address"])
 496         return RO_ns_params
 497
 498     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 499         # make a copy to do not change
 500         vdu_create = copy(vdu_create)
 501         vdu_delete = copy(vdu_delete)
 502
 503         vdurs = db_vnfr.get("vdur")
 504         if vdurs is None:
 505             vdurs = []
 506         vdu_index = len(vdurs)
 507         while vdu_index:
 508             vdu_index -= 1
 509             vdur = vdurs[vdu_index]
 510             if vdur.get("pdu-type"):
 511                 continue
 512             vdu_id_ref = vdur["vdu-id-ref"]
 513             if vdu_create and vdu_create.get(vdu_id_ref):
 514                 for index in range(0, vdu_create[vdu_id_ref]):
 515                     vdur = deepcopy(vdur)
 516                     vdur["_id"] = str(uuid4())
 517                     vdur["count-index"] += 1
 518                     vdurs.insert(vdu_index+1+index, vdur)
 519                 del vdu_create[vdu_id_ref]
 520             if vdu_delete and vdu_delete.get(vdu_id_ref):
 521                 del vdurs[vdu_index]
 522                 vdu_delete[vdu_id_ref] -= 1
 523                 if not vdu_delete[vdu_id_ref]:
 524                     del vdu_delete[vdu_id_ref]
 525         # check all operations are done
 526         if vdu_create or vdu_delete:
 527             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 528                 vdu_create))
 529         if vdu_delete:
 530             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 531                 vdu_delete))
 532
 533         vnfr_update = {"vdur": vdurs}
 534         db_vnfr["vdur"] = vdurs
 535         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 536
 537     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 538         """
 539         Updates database nsr with the RO info for the created vld
 540         :param ns_update_nsr: dictionary to be filled with the updated info
 541         :param db_nsr: content of db_nsr. This is also modified
 542         :param nsr_desc_RO: nsr descriptor from RO
 543         :return: Nothing, LcmException is raised on errors
 544         """
 545
 546         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 547             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 548                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 549                     continue
 550                 vld["vim-id"] = net_RO.get("vim_net_id")
 551                 vld["name"] = net_RO.get("vim_name")
 552                 vld["status"] = net_RO.get("status")
 553                 vld["status-detailed"] = net_RO.get("error_msg")
 554                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 555                 break
 556             else:
 557                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 558
 559     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 560         """
 561         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 562         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 563         :param nsr_desc_RO: nsr descriptor from RO
 564         :return: Nothing, LcmException is raised on errors
 565         """
 566         for vnf_index, db_vnfr in db_vnfrs.items():
 567             for vnf_RO in nsr_desc_RO["vnfs"]:
 568                 if vnf_RO["member_vnf_index"] != vnf_index:
 569                     continue
 570                 vnfr_update = {}
 571                 if vnf_RO.get("ip_address"):
 572                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 573                 elif not db_vnfr.get("ip-address"):
 574                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 575
 576                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 577                     vdur_RO_count_index = 0
 578                     if vdur.get("pdu-type"):
 579                         continue
 580                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 581                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 582                             continue
 583                         if vdur["count-index"] != vdur_RO_count_index:
 584                             vdur_RO_count_index += 1
 585                             continue
 586                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 587                         if vdur_RO.get("ip_address"):
 588                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 589                         else:
 590                             vdur["ip-address"] = None
 591                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 592                         vdur["name"] = vdur_RO.get("vim_name")
 593                         vdur["status"] = vdur_RO.get("status")
 594                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 595                         for ifacer in get_iterable(vdur, "interfaces"):
 596                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 597                                 if ifacer["name"] == interface_RO.get("internal_name"):
 598                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 599                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 600                                     break
 601                             else:
 602                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 603                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 604                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 605                         break
 606                     else:
 607                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 608                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 609
 610                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 611                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 612                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 613                             continue
 614                         vld["vim-id"] = net_RO.get("vim_net_id")
 615                         vld["name"] = net_RO.get("vim_name")
 616                         vld["status"] = net_RO.get("status")
 617                         vld["status-detailed"] = net_RO.get("error_msg")
 618                         vnfr_update["vld.{}".format(vld_index)] = vld
 619                         break
 620                     else:
 621                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 622                             vnf_index, vld["id"]))
 623
 624                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 625                 break
 626
 627             else:
 628                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 629
 630     async def instantiate(self, nsr_id, nslcmop_id):
 631
 632         # Try to lock HA task here
 633         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 634         if not task_is_locked_by_me:
 635             return
 636
 637         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 638         self.logger.debug(logging_text + "Enter")
 639         # get all needed from database
 640         start_deploy = time()
 641         db_nsr = None
 642         db_nslcmop = None
 643         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 644         db_nslcmop_update = {}
 645         nslcmop_operation_state = None
 646         db_vnfrs = {}
 647         RO_descriptor_number = 0   # number of descriptors created at RO
 648         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 649         n2vc_info = {}
 650         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 651         exc = None
 652         try:
 653             # wait for any previous tasks in process
 654             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 655
 656             step = "Getting nslcmop={} from db".format(nslcmop_id)
 657             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 658             step = "Getting nsr={} from db".format(nsr_id)
 659             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 660             ns_params = db_nslcmop.get("operationParams")
 661             nsd = db_nsr["nsd"]
 662             nsr_name = db_nsr["name"]   # TODO short-name??
 663
 664             step = "Getting vnfrs from db"
 665             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 666             db_vnfds_ref = {}
 667             db_vnfds = {}
 668             db_vnfds_index = {}
 669             for vnfr in db_vnfrs_list:
 670                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 671                 vnfd_id = vnfr["vnfd-id"]
 672                 vnfd_ref = vnfr["vnfd-ref"]
 673                 if vnfd_id not in db_vnfds:
 674                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 675                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 676                     db_vnfds_ref[vnfd_ref] = vnfd
 677                     db_vnfds[vnfd_id] = vnfd
 678                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 679
 680             # Get or generates the _admin.deployed,VCA list
 681             vca_deployed_list = None
 682             vca_model_name = None
 683             if db_nsr["_admin"].get("deployed"):
 684                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 685                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 686             if vca_deployed_list is None:
 687                 vca_deployed_list = []
 688                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 689                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 690             elif isinstance(vca_deployed_list, dict):
 691                 # maintain backward compatibility. Change a dict to list at database
 692                 vca_deployed_list = list(vca_deployed_list.values())
 693                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 694                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 695
 696             db_nsr_update["detailed-status"] = "creating"
 697             db_nsr_update["operational-status"] = "init"
 698             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 699                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 700                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 701                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 702
 703             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 704             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 705             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 706
 707             # Deploy charms
 708             # The parameters we'll need to deploy a charm
 709             number_to_configure = 0
 710
 711             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 712                 """An inner function to deploy the charm from either ns, vnf or vdu
 713                 For ns both vnf_index and vdu_id are None.
 714                 For vnf only vdu_id is None
 715                 For vdu both vnf_index and vdu_id contain a value
 716                 """
 717                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 718                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 719
 720                 machine_spec = {}
 721                 if native_charm:
 722                     machine_spec["username"] = charm_params.get("username"),
 723                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 724
 725                 # Note: The charm needs to exist on disk at the location
 726                 # specified by charm_path.
 727                 descriptor = vnfd if vnf_index else nsd
 728                 base_folder = descriptor["_admin"]["storage"]
 729                 storage_params = self.fs.get_params()
 730                 charm_path = "{}{}/{}/charms/{}".format(
 731                     storage_params["path"],
 732                     base_folder["folder"],
 733                     base_folder["pkg-dir"],
 734                     proxy_charm
 735                 )
 736
 737                 # ns_name will be ignored in the current version of N2VC
 738                 # but will be implemented for the next point release.
 739                 model_name = nsr_id
 740                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 741                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 742                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 743
 744                 vca_index = len(vca_deployed_list)
 745                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 746                 # 26*26 charm in the same NS
 747                 application_name = application_name[0:48]
 748                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 749                 vca_deployed_ = {
 750                     "member-vnf-index": vnf_index,
 751                     "vdu_id": vdu_id,
 752                     "model": model_name,
 753                     "application": application_name,
 754                     "operational-status": "init",
 755                     "detailed-status": "",
 756                     "step": "initial-deploy",
 757                     "vnfd_id": vnfd_id,
 758                     "vdu_name": vdu_name,
 759                     "vdu_count_index": vdu_count_index,
 760                 }
 761                 vca_deployed_list.append(vca_deployed_)
 762                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 763                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 764
 765                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 766                                                                                                 proxy_charm))
 767                 if not n2vc_info:
 768                     n2vc_info["nsr_id"] = nsr_id
 769                     n2vc_info["nslcmop_id"] = nslcmop_id
 770                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 771                     n2vc_info["lcmOperationType"] = "instantiate"
 772                     n2vc_info["deployed"] = vca_deployed_list
 773                     n2vc_info["db_update"] = db_nsr_update
 774                 task = asyncio.ensure_future(
 775                     self.n2vc.DeployCharms(
 776                         model_name,          # The network service name
 777                         application_name,    # The application name
 778                         descriptor,          # The vnf/nsd descriptor
 779                         charm_path,          # Path to charm
 780                         charm_params,        # Runtime params, like mgmt ip
 781                         machine_spec,        # for native charms only
 782                         self.n2vc_callback,  # Callback for status changes
 783                         n2vc_info,           # Callback parameter
 784                         None,                # Callback parameter (task)
 785                     )
 786                 )
 787                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 788                                                          n2vc_info))
 789                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 790
 791             step = "Looking for needed vnfd to configure with proxy charm"
 792             self.logger.debug(logging_text + step)
 793
 794             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 795                 vnfd_id = c_vnf["vnfd-id-ref"]
 796                 vnf_index = str(c_vnf["member-vnf-index"])
 797                 vnfd = db_vnfds_ref[vnfd_id]
 798
 799                 # Get additional parameters
 800                 vnfr_params = {}
 801                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 802                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 803                 for k, v in vnfr_params.items():
 804                     if isinstance(v, str) and v.startswith("!!yaml "):
 805                         vnfr_params[k] = yaml.safe_load(v[7:])
 806
 807                 step = "deploying proxy charms for configuration"
 808                 # Check if this VNF has a charm configuration
 809                 vnf_config = vnfd.get("vnf-configuration")
 810                 if vnf_config and vnf_config.get("juju"):
 811                     proxy_charm = vnf_config["juju"]["charm"]
 812                     if vnf_config["juju"].get("proxy") is False:
 813                         # native_charm, will be deployed after VM. Skip
 814                         proxy_charm = None
 815
 816                     if proxy_charm:
 817                         if not vca_model_name:
 818                             step = "creating VCA model name '{}'".format(nsr_id)
 819                             self.logger.debug(logging_text + step)
 820                             await self.n2vc.CreateNetworkService(nsr_id)
 821                             vca_model_name = nsr_id
 822                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 823                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 824                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 825                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 826                         charm_params = {
 827                             "user_values": vnfr_params,
 828                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 829                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 830                         }
 831
 832                         # Login to the VCA. If there are multiple calls to login(),
 833                         # subsequent calls will be a nop and return immediately.
 834                         await self.n2vc.login()
 835
 836                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 837                         number_to_configure += 1
 838
 839                 # Deploy charms for each VDU that supports one.
 840                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 841                     vdu_config = vdu.get('vdu-configuration')
 842                     proxy_charm = None
 843
 844                     if vdu_config and vdu_config.get("juju"):
 845                         proxy_charm = vdu_config["juju"]["charm"]
 846                         if vdu_config["juju"].get("proxy") is False:
 847                             # native_charm, will be deployed after VM. Skip
 848                             proxy_charm = None
 849
 850                         if proxy_charm:
 851                             if not vca_model_name:
 852                                 step = "creating VCA model name"
 853                                 await self.n2vc.CreateNetworkService(nsr_id)
 854                                 vca_model_name = nsr_id
 855                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 856                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 857                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 858                                                                                                           vdu["id"])
 859                             await self.n2vc.login()
 860                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 861                             # TODO for the moment only first vdu_id contains a charm deployed
 862                             if vdur["vdu-id-ref"] != vdu["id"]:
 863                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 864                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 865                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 866                             charm_params = {
 867                                 "user_values": vnfr_params,
 868                                 "rw_mgmt_ip": vdur["ip-address"],
 869                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 870                             }
 871                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 872                                          charm_params, n2vc_info)
 873                             number_to_configure += 1
 874
 875             # Check if this NS has a charm configuration
 876
 877             ns_config = nsd.get("ns-configuration")
 878             if ns_config and ns_config.get("juju"):
 879                 proxy_charm = ns_config["juju"]["charm"]
 880                 if ns_config["juju"].get("proxy") is False:
 881                     # native_charm, will be deployed after VM. Skip
 882                     proxy_charm = None
 883
 884                 if proxy_charm:
 885                     step = "deploying proxy charm to configure ns"
 886                     # TODO is NS magmt IP address needed?
 887
 888                     # Get additional parameters
 889                     additional_params = {}
 890                     if db_nsr.get("additionalParamsForNs"):
 891                         additional_params = db_nsr["additionalParamsForNs"].copy()
 892                     for k, v in additional_params.items():
 893                         if isinstance(v, str) and v.startswith("!!yaml "):
 894                             additional_params[k] = yaml.safe_load(v[7:])
 895
 896                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 897                     charm_params = {
 898                         "user_values": additional_params,
 899                         # "rw_mgmt_ip": db_nsr["ip-address"],
 900                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 901                     }
 902
 903                     # Login to the VCA. If there are multiple calls to login(),
 904                     # subsequent calls will be a nop and return immediately.
 905                     await self.n2vc.login()
 906                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 907                     number_to_configure += 1
 908
 909             db_nsr_update["operational-status"] = "running"
 910
 911             # Wait until all charms has reached blocked or active status
 912             step = "waiting proxy charms to be ready"
 913             if number_to_configure:
 914                 # wait until all charms are configured.
 915                 # steps are:
 916                 #       initial-deploy
 917                 #       get-ssh-public-key
 918                 #       generate-ssh-key
 919                 #       retry-get-ssh-public-key
 920                 #       ssh-public-key-obtained
 921                 while time() <= start_deploy + self.total_deploy_timeout:
 922                     if db_nsr_update:
 923                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 924                     if db_nslcmop_update:
 925                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 926
 927                     all_active = True
 928                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 929                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 930                         if vca_deployed["step"] == "initial-deploy":
 931                             if vca_deployed["operational-status"] in ("active", "blocked"):
 932                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 933                                        .format(vca_deployed["member-vnf-index"],
 934                                                vca_deployed["vdu_id"])
 935                                 self.logger.debug(logging_text + step)
 936                                 try:
 937                                     primitive_id = await self.n2vc.ExecutePrimitive(
 938                                         vca_deployed["model"],
 939                                         vca_deployed["application"],
 940                                         "get-ssh-public-key",
 941                                         None,
 942                                     )
 943                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 944                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 945                                         primitive_id
 946                                     db_nsr_update[database_entry + "operational-status"] =\
 947                                         vca_deployed["operational-status"]
 948                                 except PrimitiveDoesNotExist:
 949                                     ssh_public_key = None
 950                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 951                                         "ssh-public-key-obtained"
 952                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 953                                         ssh_public_key
 954                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 955                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 956                                     self.logger.debug(logging_text + step)
 957
 958                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 959                             primitive_id = vca_deployed["primitive_id"]
 960                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 961                                                                                   primitive_id)
 962                             if primitive_status in ("completed", "failed"):
 963                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 964                                                                                       primitive_id)
 965                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 966                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 967                                         primitive_result.get("pubkey"):
 968                                     ssh_public_key = primitive_result.get("pubkey")
 969                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 970                                         "ssh-public-key-obtained"
 971                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 972                                         ssh_public_key
 973                                     n2vc_key_list.append(ssh_public_key)
 974                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 975                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 976                                     self.logger.debug(logging_text + step)
 977                                 else:  # primitive_status == "failed":
 978                                     if vca_deployed["step"] == "get-ssh-public-key":
 979                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 980                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 981                                                                      vca_deployed["vdu_id"])
 982                                         self.logger.debug(logging_text + step)
 983                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 984                                             "generate-ssh-key"
 985                                         primitive_id = await self.n2vc.ExecutePrimitive(
 986                                             vca_deployed["model"],
 987                                             vca_deployed["application"],
 988                                             "generate-ssh-key",
 989                                             None,
 990                                         )
 991                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 992                                             primitive_id
 993                                     else:  # failed for second time
 994                                         raise LcmException(
 995                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 996
 997                         elif vca_deployed["step"] == "generate-ssh-key":
 998                             primitive_id = vca_deployed["primitive_id"]
 999                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1000                                                                                   primitive_id)
1001                             if primitive_status in ("completed", "failed"):
1002                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1003                                                                                       primitive_id)
1004                                 vca_deployed["primitive_id"] = db_nsr_update[
1005                                     database_entry + "primitive_id"] = None
1006                                 if primitive_status == "completed":
1007                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1008                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1009                                                               vca_deployed["vdu_id"])
1010                                     self.logger.debug(logging_text + step)
1011                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1012                                         "retry-get-ssh-public-key"
1013                                     primitive_id = await self.n2vc.ExecutePrimitive(
1014                                         vca_deployed["model"],
1015                                         vca_deployed["application"],
1016                                         "get-ssh-public-key",
1017                                         None,
1018                                     )
1019                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1020                                         primitive_id
1021
1022                                 else:  # primitive_status == "failed":
1023                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1024                                                        .format(primitive_result))
1025
1026                         if vca_deployed["step"] != "ssh-public-key-obtained":
1027                             all_active = False
1028
1029                     if all_active:
1030                         break
1031                     await asyncio.sleep(5)
1032                 else:   # total_deploy_timeout
1033                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1034                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1035
1036             # deploy RO
1037             RO = ROclient.ROClient(self.loop, **self.ro_config)
1038             # get vnfds, instantiate at RO
1039             for c_vnf in nsd.get("constituent-vnfd", ()):
1040                 member_vnf_index = c_vnf["member-vnf-index"]
1041                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1042                 vnfd_ref = vnfd["id"]
1043                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1044                     vnfd_ref, member_vnf_index)
1045                 # self.logger.debug(logging_text + step)
1046                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1047                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1048                 RO_descriptor_number += 1
1049
1050                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1051                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1052                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1053                         break
1054                 else:
1055                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1056                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1057
1058                 # look if present
1059                 RO_update = {"member-vnf-index": member_vnf_index}
1060                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1061                 if vnfd_list:
1062                     RO_update["id"] = vnfd_list[0]["uuid"]
1063                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1064                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1065                 else:
1066                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1067                                            get("additionalParamsForVnf"), nsr_id)
1068                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
1069                     RO_update["id"] = desc["uuid"]
1070                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1071                         vnfd_ref, member_vnf_index, desc["uuid"]))
1072                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1073                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1074                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1075
1076             # create nsd at RO
1077             nsd_ref = nsd["id"]
1078             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1079             # self.logger.debug(logging_text + step)
1080
1081             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1082             RO_descriptor_number += 1
1083             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1084             if nsd_list:
1085                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1086                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1087                     nsd_ref, RO_nsd_uuid))
1088             else:
1089                 nsd_RO = deepcopy(nsd)
1090                 nsd_RO["id"] = RO_osm_nsd_id
1091                 nsd_RO.pop("_id", None)
1092                 nsd_RO.pop("_admin", None)
1093                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1094                     member_vnf_index = c_vnf["member-vnf-index"]
1095                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1096                 for c_vld in nsd_RO.get("vld", ()):
1097                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1098                         member_vnf_index = cp["member-vnf-index-ref"]
1099                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1100
1101                 desc = await RO.create("nsd", descriptor=nsd_RO)
1102                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1103                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1104                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1105             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1106
1107             # Crate ns at RO
1108             # if present use it unless in error status
1109             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1110             if RO_nsr_id:
1111                 try:
1112                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1113                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1114                     desc = await RO.show("ns", RO_nsr_id)
1115                 except ROclient.ROClientException as e:
1116                     if e.http_code != HTTPStatus.NOT_FOUND:
1117                         raise
1118                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1119                 if RO_nsr_id:
1120                     ns_status, ns_status_info = RO.check_ns_status(desc)
1121                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1122                     if ns_status == "ERROR":
1123                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1124                         self.logger.debug(logging_text + step)
1125                         await RO.delete("ns", RO_nsr_id)
1126                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1127             if not RO_nsr_id:
1128                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1129                 # self.logger.debug(logging_text + step)
1130
1131                 # check if VIM is creating and wait  look if previous tasks in process
1132                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1133                 if task_dependency:
1134                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1135                     self.logger.debug(logging_text + step)
1136                     await asyncio.wait(task_dependency, timeout=3600)
1137                 if ns_params.get("vnf"):
1138                     for vnf in ns_params["vnf"]:
1139                         if "vimAccountId" in vnf:
1140                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1141                                                                                         vnf["vimAccountId"])
1142                         if task_dependency:
1143                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1144                             self.logger.debug(logging_text + step)
1145                             await asyncio.wait(task_dependency, timeout=3600)
1146
1147                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1148
1149                 # feature 1429. Add n2vc public key to needed VMs
1150                 n2vc_key = await self.n2vc.GetPublicKey()
1151                 n2vc_key_list.append(n2vc_key)
1152                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1153
1154                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
1155                 desc = await RO.create("ns", descriptor=RO_ns_params,
1156                                        name=db_nsr["name"],
1157                                        scenario=RO_nsd_uuid)
1158                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1159                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1160                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1161                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1162             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1163
1164             # wait until NS is ready
1165             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1166             detailed_status_old = None
1167             self.logger.debug(logging_text + step)
1168
1169             while time() <= start_deploy + self.total_deploy_timeout:
1170                 desc = await RO.show("ns", RO_nsr_id)
1171                 ns_status, ns_status_info = RO.check_ns_status(desc)
1172                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1173                 if ns_status == "ERROR":
1174                     raise ROclient.ROClientException(ns_status_info)
1175                 elif ns_status == "BUILD":
1176                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1177                 elif ns_status == "ACTIVE":
1178                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1179                     try:
1180                         self.ns_update_vnfr(db_vnfrs, desc)
1181                         break
1182                     except LcmExceptionNoMgmtIP:
1183                         pass
1184                 else:
1185                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1186                 if detailed_status != detailed_status_old:
1187                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1188                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1189                 await asyncio.sleep(5, loop=self.loop)
1190             else:   # total_deploy_timeout
1191                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1192
1193             step = "Updating NSR"
1194             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1195
1196             db_nsr_update["operational-status"] = "running"
1197             db_nsr["detailed-status"] = "Configuring vnfr"
1198             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1199
1200             # Configure proxy charms once VMs are up
1201             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1202                 vnf_index = vca_deployed.get("member-vnf-index")
1203                 vdu_id = vca_deployed.get("vdu_id")
1204                 vdu_name = None
1205                 vdu_count_index = None
1206
1207                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1208                                                                                                            vdu_id)
1209                 add_params = {}
1210                 initial_config_primitive_list = []
1211                 if vnf_index:
1212                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1213                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1214                     vnfd = db_vnfds_index[vnf_index]
1215
1216                     if vdu_id:
1217                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1218                             if vdu["id"] == vdu_id:
1219                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1220                                     'initial-config-primitive', [])
1221                                 break
1222                         else:
1223                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1224                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1225                         # TODO for the moment only first vdu_id contains a charm deployed
1226                         if vdur["vdu-id-ref"] != vdu["id"]:
1227                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1228                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1229                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1230                     else:
1231                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1232                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1233                 else:
1234                     if db_nsr.get("additionalParamsForNs"):
1235                         add_params = db_nsr["additionalParamsForNs"].copy()
1236                     for k, v in add_params.items():
1237                         if isinstance(v, str) and v.startswith("!!yaml "):
1238                             add_params[k] = yaml.safe_load(v[7:])
1239                     add_params["rw_mgmt_ip"] = None
1240                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1241
1242                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1243                 initial_config_primitive_list = initial_config_primitive_list.copy()
1244                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1245                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1246
1247                 for initial_config_primitive in initial_config_primitive_list:
1248                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1249                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1250                                       .format(initial_config_primitive["name"], primitive_params_))
1251                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1252                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1253                         initial_config_primitive["name"],
1254                         primitive_params_,
1255                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1256                         retries_interval=30)
1257                     if primitive_result != "COMPLETED":
1258                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1259                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1260                                                    vca_deployed["vdu_id"], primitive_detail))
1261
1262             # Deploy native charms
1263             step = "Looking for needed vnfd to configure with native charm"
1264             self.logger.debug(logging_text + step)
1265
1266             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1267                 vnfd_id = c_vnf["vnfd-id-ref"]
1268                 vnf_index = str(c_vnf["member-vnf-index"])
1269                 vnfd = db_vnfds_ref[vnfd_id]
1270
1271                 # Get additional parameters
1272                 vnfr_params = {}
1273                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1274                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1275                 for k, v in vnfr_params.items():
1276                     if isinstance(v, str) and v.startswith("!!yaml "):
1277                         vnfr_params[k] = yaml.safe_load(v[7:])
1278
1279                 # Check if this VNF has a charm configuration
1280                 vnf_config = vnfd.get("vnf-configuration")
1281                 if vnf_config and vnf_config.get("juju"):
1282                     native_charm = vnf_config["juju"].get("proxy") is False
1283
1284                     if native_charm:
1285                         if not vca_model_name:
1286                             step = "creating VCA model name '{}'".format(nsr_id)
1287                             self.logger.debug(logging_text + step)
1288                             await self.n2vc.CreateNetworkService(nsr_id)
1289                             vca_model_name = nsr_id
1290                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1291                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1292                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1293                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1294                         charm_params = {
1295                             "user_values": vnfr_params,
1296                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1297                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1298                         }
1299
1300                         # get username
1301                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1302                         #  merged. Meanwhile let's get username from initial-config-primitive
1303                         if vnf_config.get("initial-config-primitive"):
1304                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1305                                 if param["name"] == "ssh-username":
1306                                     charm_params["username"] = param["value"]
1307                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1308                             if vnf_config["config-access"]["ssh-access"].get("required"):
1309                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1310
1311                         # Login to the VCA. If there are multiple calls to login(),
1312                         # subsequent calls will be a nop and return immediately.
1313                         await self.n2vc.login()
1314
1315                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1316                         number_to_configure += 1
1317
1318                 # Deploy charms for each VDU that supports one.
1319                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1320                     vdu_config = vdu.get('vdu-configuration')
1321                     native_charm = False
1322
1323                     if vdu_config and vdu_config.get("juju"):
1324                         native_charm = vdu_config["juju"].get("proxy") is False
1325
1326                         if native_charm:
1327                             if not vca_model_name:
1328                                 step = "creating VCA model name"
1329                                 await self.n2vc.CreateNetworkService(nsr_id)
1330                                 vca_model_name = nsr_id
1331                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1332                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1333                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1334                                                                                                      vdu["id"])
1335                             await self.n2vc.login()
1336                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1337                             # TODO for the moment only first vdu_id contains a charm deployed
1338                             if vdur["vdu-id-ref"] != vdu["id"]:
1339                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1340                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1341                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1342                             charm_params = {
1343                                 "user_values": vnfr_params,
1344                                 "rw_mgmt_ip": vdur["ip-address"],
1345                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1346                             }
1347
1348                             # get username
1349                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1350                             #  merged. Meanwhile let's get username from initial-config-primitive
1351                             if vdu_config.get("initial-config-primitive"):
1352                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1353                                     if param["name"] == "ssh-username":
1354                                         charm_params["username"] = param["value"]
1355                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1356                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1357                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1358                                         "default-user")
1359
1360                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1361                                          charm_params, n2vc_info, native_charm)
1362                             number_to_configure += 1
1363
1364             # Check if this NS has a charm configuration
1365
1366             ns_config = nsd.get("ns-configuration")
1367             if ns_config and ns_config.get("juju"):
1368                 native_charm = ns_config["juju"].get("proxy") is False
1369
1370                 if native_charm:
1371                     step = "deploying native charm to configure ns"
1372                     # TODO is NS magmt IP address needed?
1373
1374                     # Get additional parameters
1375                     additional_params = {}
1376                     if db_nsr.get("additionalParamsForNs"):
1377                         additional_params = db_nsr["additionalParamsForNs"].copy()
1378                     for k, v in additional_params.items():
1379                         if isinstance(v, str) and v.startswith("!!yaml "):
1380                             additional_params[k] = yaml.safe_load(v[7:])
1381
1382                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1383                     charm_params = {
1384                         "user_values": additional_params,
1385                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1386                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1387                     }
1388
1389                     # get username
1390                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1391                     #  merged. Meanwhile let's get username from initial-config-primitive
1392                     if ns_config.get("initial-config-primitive"):
1393                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1394                             if param["name"] == "ssh-username":
1395                                 charm_params["username"] = param["value"]
1396                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1397                         if ns_config["config-access"]["ssh-access"].get("required"):
1398                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1399
1400                     # Login to the VCA. If there are multiple calls to login(),
1401                     # subsequent calls will be a nop and return immediately.
1402                     await self.n2vc.login()
1403                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1404                     number_to_configure += 1
1405
1406             # waiting all charms are ok
1407             configuration_failed = False
1408             if number_to_configure:
1409                 step = "Waiting all charms are active"
1410                 old_status = "configuring: init: {}".format(number_to_configure)
1411                 db_nsr_update["config-status"] = old_status
1412                 db_nsr_update["detailed-status"] = old_status
1413                 db_nslcmop_update["detailed-status"] = old_status
1414
1415                 # wait until all are configured.
1416                 while time() <= start_deploy + self.total_deploy_timeout:
1417                     if db_nsr_update:
1418                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1419                     if db_nslcmop_update:
1420                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1421                     # TODO add a fake task that set n2vc_event after some time
1422                     await n2vc_info["n2vc_event"].wait()
1423                     n2vc_info["n2vc_event"].clear()
1424                     all_active = True
1425                     status_map = {}
1426                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1427                     now = time()
1428                     for vca_deployed in vca_deployed_list:
1429                         vca_status = vca_deployed["operational-status"]
1430                         if vca_status not in status_map:
1431                             # Initialize it
1432                             status_map[vca_status] = 0
1433                         status_map[vca_status] += 1
1434
1435                         if vca_status == "active":
1436                             vca_deployed.pop("time_first_error", None)
1437                             vca_deployed.pop("status_first_error", None)
1438                             continue
1439
1440                         all_active = False
1441                         if vca_status in ("error", "blocked"):
1442                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1443                             # if not first time in this status error
1444                             if not vca_deployed.get("time_first_error"):
1445                                 vca_deployed["time_first_error"] = now
1446                                 continue
1447                         if vca_deployed.get("time_first_error") and \
1448                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1449                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1450                                                    .format(vca_deployed["member-vnf-index"],
1451                                                            vca_deployed["vdu_id"], vca_status,
1452                                                            vca_deployed["detailed-status-error"]))
1453
1454                     if all_active:
1455                         break
1456                     elif n2vc_error_text:
1457                         db_nsr_update["config-status"] = "failed"
1458                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1459                         db_nsr_update["detailed-status"] = error_text
1460                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1461                         db_nslcmop_update["detailed-status"] = error_text
1462                         db_nslcmop_update["statusEnteredTime"] = time()
1463                         configuration_failed = True
1464                         break
1465                     else:
1466                         cs = "configuring: "
1467                         separator = ""
1468                         for status, num in status_map.items():
1469                             cs += separator + "{}: {}".format(status, num)
1470                             separator = ", "
1471                         if old_status != cs:
1472                             db_nsr_update["config-status"] = cs
1473                             db_nsr_update["detailed-status"] = cs
1474                             db_nslcmop_update["detailed-status"] = cs
1475                             old_status = cs
1476                 else:   # total_deploy_timeout
1477                     raise LcmException("Timeout waiting ns to be configured")
1478
1479             if not configuration_failed:
1480                 # all is done
1481                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1482                 db_nslcmop_update["statusEnteredTime"] = time()
1483                 db_nslcmop_update["detailed-status"] = "done"
1484                 db_nsr_update["config-status"] = "configured"
1485                 db_nsr_update["detailed-status"] = "done"
1486
1487             return
1488
1489         except (ROclient.ROClientException, DbException, LcmException) as e:
1490             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1491             exc = e
1492         except asyncio.CancelledError:
1493             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1494             exc = "Operation was cancelled"
1495         except Exception as e:
1496             exc = traceback.format_exc()
1497             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1498                                  exc_info=True)
1499         finally:
1500             if exc:
1501                 if db_nsr:
1502                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1503                     db_nsr_update["operational-status"] = "failed"
1504                 if db_nslcmop:
1505                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1506                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1507                     db_nslcmop_update["statusEnteredTime"] = time()
1508             try:
1509                 if db_nsr:
1510                     db_nsr_update["_admin.nslcmop"] = None
1511                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1512                 if db_nslcmop_update:
1513                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1514             except DbException as e:
1515                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1516             if nslcmop_operation_state:
1517                 try:
1518                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1519                                                                    "operationState": nslcmop_operation_state},
1520                                             loop=self.loop)
1521                 except Exception as e:
1522                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1523
1524             self.logger.debug(logging_text + "Exit")
1525             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1526
1527     async def _destroy_charm(self, model, application):
1528         """
1529         Order N2VC destroy a charm
1530         :param model:
1531         :param application:
1532         :return: True if charm does not exist. False if it exist
1533         """
1534         if not await self.n2vc.HasApplication(model, application):
1535             return True  # Already removed
1536         await self.n2vc.RemoveCharms(model, application)
1537         return False
1538
1539     async def _wait_charm_destroyed(self, model, application, timeout):
1540         """
1541         Wait until charm does not exist
1542         :param model:
1543         :param application:
1544         :param timeout:
1545         :return: True if not exist, False if timeout
1546         """
1547         while True:
1548             if not await self.n2vc.HasApplication(model, application):
1549                 return True
1550             if timeout < 0:
1551                 return False
1552             await asyncio.sleep(10)
1553             timeout -= 10
1554
1555     # Check if this VNFD has a configured terminate action
1556     def _has_terminate_config_primitive(self, vnfd):
1557         vnf_config = vnfd.get("vnf-configuration")
1558         if vnf_config and vnf_config.get("terminate-config-primitive"):
1559             return True
1560         else:
1561             return False
1562
1563     @staticmethod
1564     def _get_terminate_config_primitive_seq_list(vnfd):
1565         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1566         # No need to check for existing primitive twice, already done before
1567         vnf_config = vnfd.get("vnf-configuration")
1568         seq_list = vnf_config.get("terminate-config-primitive")
1569         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1570         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1571         return seq_list_sorted
1572
1573     @staticmethod
1574     def _create_nslcmop(nsr_id, operation, params):
1575         """
1576         Creates a ns-lcm-opp content to be stored at database.
1577         :param nsr_id: internal id of the instance
1578         :param operation: instantiate, terminate, scale, action, ...
1579         :param params: user parameters for the operation
1580         :return: dictionary following SOL005 format
1581         """
1582         # Raise exception if invalid arguments
1583         if not (nsr_id and operation and params):
1584             raise LcmException(
1585                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1586         now = time()
1587         _id = str(uuid4())
1588         nslcmop = {
1589             "id": _id,
1590             "_id": _id,
1591             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1592             "operationState": "PROCESSING",
1593             "statusEnteredTime": now,
1594             "nsInstanceId": nsr_id,
1595             "lcmOperationType": operation,
1596             "startTime": now,
1597             "isAutomaticInvocation": False,
1598             "operationParams": params,
1599             "isCancelPending": False,
1600             "links": {
1601                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1602                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1603             }
1604         }
1605         return nslcmop
1606
1607     def _get_terminate_primitive_params(self, seq, vnf_index):
1608         primitive = seq.get('name')
1609         primitive_params = {}
1610         params = {
1611             "member_vnf_index": vnf_index,
1612             "primitive": primitive,
1613             "primitive_params": primitive_params,
1614         }
1615         desc_params = {}
1616         return self._map_primitive_params(seq, params, desc_params)
1617
1618     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1619                           vdu_name, primitive, mapped_primitive_params):
1620         # Create the "_admin.operations" array, or append operation if array already exists
1621         key_admin = '_admin'
1622         key_operations = 'operations'
1623         db_nslcmop_update = {}
1624         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1625         op_list = db_nslcmop_admin.get(key_operations)
1626         new_op = {'member_vnf_index': vnf_index,
1627                   'vdu_id': vdu_id,
1628                   'vdu_count_index': vdu_count_index,
1629                   'primitive': primitive,
1630                   'primitive_params': mapped_primitive_params}
1631         if db_nslcmop_admin:
1632             if not op_list:
1633                 # First operation, create key 'operations' with current operation as first list element
1634                 db_nslcmop_admin.update({key_operations: [new_op]})
1635                 op_list = db_nslcmop_admin.get(key_operations)
1636             else:
1637                 # Not first operation, append operation to existing list
1638                 op_list.append(new_op)
1639
1640         db_nslcmop_update['_admin.operations'] = op_list
1641         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1642
1643     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1644         """ Create a primitive with params from VNFD
1645             Called from terminate() before deleting instance
1646             Calls action() to execute the primitive """
1647         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1648         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1649         db_vnfds = {}
1650         # Loop over VNFRs
1651         for vnfr in db_vnfrs_list:
1652             vnfd_id = vnfr["vnfd-id"]
1653             vnf_index = vnfr["member-vnf-index-ref"]
1654             if vnfd_id not in db_vnfds:
1655                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1656                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1657                 db_vnfds[vnfd_id] = vnfd
1658             vnfd = db_vnfds[vnfd_id]
1659             if not self._has_terminate_config_primitive(vnfd):
1660                 continue
1661             # Get the primitive's sorted sequence list
1662             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1663             for seq in seq_list:
1664                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1665                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1666                     vnf_index, seq.get("name"))
1667                 self.logger.debug(logging_text + step)
1668                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1669                 primitive = seq.get('name')
1670                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1671                 # The following 3 parameters are currently set to None for 'terminate':
1672                 # vdu_id, vdu_count_index, vdu_name
1673                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1674                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1675                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1676                 # Add suboperation
1677                 self._add_suboperation(db_nslcmop,
1678                                        nslcmop_id,
1679                                        vnf_index,
1680                                        vdu_id,
1681                                        vdu_count_index,
1682                                        vdu_name,
1683                                        primitive,
1684                                        mapped_primitive_params)
1685                 # Suboperations: Call _ns_execute_primitive() instead of action()
1686                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1687                 nsr_deployed = db_nsr["_admin"]["deployed"]
1688                 result, result_detail = await self._ns_execute_primitive(
1689                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1690                     mapped_primitive_params)
1691
1692                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1693                 #    nsr_id, nslcmop_terminate_action_id)
1694                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1695                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1696                 if result not in result_ok:
1697                     raise LcmException(
1698                         "terminate_primitive_action for vnf_member_index={}",
1699                         " primitive={} fails with error {}".format(
1700                             vnf_index, seq.get("name"), result_detail))
1701
1702     async def terminate(self, nsr_id, nslcmop_id):
1703
1704         # Try to lock HA task here
1705         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1706         if not task_is_locked_by_me:
1707             return
1708
1709         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1710         self.logger.debug(logging_text + "Enter")
1711         db_nsr = None
1712         db_nslcmop = None
1713         exc = None
1714         failed_detail = []   # annotates all failed error messages
1715         vca_time_destroy = None   # time of where destroy charm order
1716         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1717         db_nslcmop_update = {}
1718         nslcmop_operation_state = None
1719         autoremove = False  # autoremove after terminated
1720         try:
1721             # wait for any previous tasks in process
1722             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1723
1724             step = "Getting nslcmop={} from db".format(nslcmop_id)
1725             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1726             step = "Getting nsr={} from db".format(nsr_id)
1727             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1728             # nsd = db_nsr["nsd"]
1729             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1730             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1731                 return
1732             # #TODO check if VIM is creating and wait
1733             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1734             # Call internal terminate action
1735             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1736
1737             db_nsr_update["operational-status"] = "terminating"
1738             db_nsr_update["config-status"] = "terminating"
1739
1740             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1741                 vca_model_name = nsr_deployed["VCA-model-name"]
1742                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1743                 self.logger.debug(logging_text + step)
1744                 try:
1745                     await self.n2vc.DestroyNetworkService(vca_model_name)
1746                 except NetworkServiceDoesNotExist:
1747                     pass
1748                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1749                 if nsr_deployed.get("VCA"):
1750                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1751                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1752                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1753             # for backward compatibility if charm have been created with "default"  model name delete one by one
1754             elif nsr_deployed and nsr_deployed.get("VCA"):
1755                 try:
1756                     step = "Scheduling configuration charms removing"
1757                     db_nsr_update["detailed-status"] = "Deleting charms"
1758                     self.logger.debug(logging_text + step)
1759                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1760                     # for backward compatibility
1761                     if isinstance(nsr_deployed["VCA"], dict):
1762                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1763                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1764                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1765
1766                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1767                         if vca_deployed:
1768                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1769                                 vca_deployed.clear()
1770                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1771                             else:
1772                                 vca_time_destroy = time()
1773                 except Exception as e:
1774                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1775
1776             # remove from RO
1777             RO_fail = False
1778             RO = ROclient.ROClient(self.loop, **self.ro_config)
1779
1780             # Delete ns
1781             RO_nsr_id = RO_delete_action = None
1782             if nsr_deployed and nsr_deployed.get("RO"):
1783                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1784                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1785             try:
1786                 if RO_nsr_id:
1787                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1788                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1789                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1790                     self.logger.debug(logging_text + step)
1791                     desc = await RO.delete("ns", RO_nsr_id)
1792                     RO_delete_action = desc["action_id"]
1793                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1794                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1795                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1796                 if RO_delete_action:
1797                     # wait until NS is deleted from VIM
1798                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1799                         format(RO_nsr_id, RO_delete_action)
1800                     detailed_status_old = None
1801                     self.logger.debug(logging_text + step)
1802
1803                     delete_timeout = 20 * 60   # 20 minutes
1804                     while delete_timeout > 0:
1805                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1806                                              extra_item_id=RO_delete_action)
1807                         ns_status, ns_status_info = RO.check_action_status(desc)
1808                         if ns_status == "ERROR":
1809                             raise ROclient.ROClientException(ns_status_info)
1810                         elif ns_status == "BUILD":
1811                             detailed_status = step + "; {}".format(ns_status_info)
1812                         elif ns_status == "ACTIVE":
1813                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1814                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1815                             break
1816                         else:
1817                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1818                         if detailed_status != detailed_status_old:
1819                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1820                                 db_nsr_update["detailed-status"] = detailed_status
1821                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1822                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1823                         await asyncio.sleep(5, loop=self.loop)
1824                         delete_timeout -= 5
1825                     else:  # delete_timeout <= 0:
1826                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1827
1828             except ROclient.ROClientException as e:
1829                 if e.http_code == 404:  # not found
1830                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1831                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1832                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1833                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1834                 elif e.http_code == 409:   # conflict
1835                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1836                     self.logger.debug(logging_text + failed_detail[-1])
1837                     RO_fail = True
1838                 else:
1839                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1840                     self.logger.error(logging_text + failed_detail[-1])
1841                     RO_fail = True
1842
1843             # Delete nsd
1844             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1845                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1846                 try:
1847                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1848                         "Deleting nsd at RO"
1849                     await RO.delete("nsd", RO_nsd_id)
1850                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1851                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1852                 except ROclient.ROClientException as e:
1853                     if e.http_code == 404:  # not found
1854                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1855                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1856                     elif e.http_code == 409:   # conflict
1857                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1858                         self.logger.debug(logging_text + failed_detail[-1])
1859                         RO_fail = True
1860                     else:
1861                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1862                         self.logger.error(logging_text + failed_detail[-1])
1863                         RO_fail = True
1864
1865             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1866                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1867                     if not vnf_deployed or not vnf_deployed["id"]:
1868                         continue
1869                     try:
1870                         RO_vnfd_id = vnf_deployed["id"]
1871                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1872                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1873                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1874                         await RO.delete("vnfd", RO_vnfd_id)
1875                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1876                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1877                     except ROclient.ROClientException as e:
1878                         if e.http_code == 404:  # not found
1879                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1880                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1881                         elif e.http_code == 409:   # conflict
1882                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1883                             self.logger.debug(logging_text + failed_detail[-1])
1884                         else:
1885                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1886                             self.logger.error(logging_text + failed_detail[-1])
1887
1888             # wait until charm deleted
1889             if vca_time_destroy:
1890                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1891                     "Waiting for deletion of configuration charms"
1892                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1893                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1894                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1895                     if not vca_deployed:
1896                         continue
1897                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1898                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1899                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1900                                                             timeout):
1901                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1902                             vca_deployed["application"]))
1903                     else:
1904                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1905
1906             if failed_detail:
1907                 self.logger.error(logging_text + " ;".join(failed_detail))
1908                 db_nsr_update["operational-status"] = "failed"
1909                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1910                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1911                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1912                 db_nslcmop_update["statusEnteredTime"] = time()
1913             else:
1914                 db_nsr_update["operational-status"] = "terminated"
1915                 db_nsr_update["detailed-status"] = "Done"
1916                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1917                 db_nslcmop_update["detailed-status"] = "Done"
1918                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1919                 db_nslcmop_update["statusEnteredTime"] = time()
1920                 if db_nslcmop["operationParams"].get("autoremove"):
1921                     autoremove = True
1922
1923         except (ROclient.ROClientException, DbException, LcmException) as e:
1924             self.logger.error(logging_text + "Exit Exception {}".format(e))
1925             exc = e
1926         except asyncio.CancelledError:
1927             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1928             exc = "Operation was cancelled"
1929         except Exception as e:
1930             exc = traceback.format_exc()
1931             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1932         finally:
1933             if exc and db_nslcmop:
1934                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1935                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1936                 db_nslcmop_update["statusEnteredTime"] = time()
1937             try:
1938                 if db_nslcmop and db_nslcmop_update:
1939                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1940                 if db_nsr:
1941                     db_nsr_update["_admin.nslcmop"] = None
1942                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1943             except DbException as e:
1944                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1945             if nslcmop_operation_state:
1946                 try:
1947                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1948                                                                  "operationState": nslcmop_operation_state,
1949                                                                  "autoremove": autoremove},
1950                                             loop=self.loop)
1951                 except Exception as e:
1952                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1953             self.logger.debug(logging_text + "Exit")
1954             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1955
1956     @staticmethod
1957     def _map_primitive_params(primitive_desc, params, instantiation_params):
1958         """
1959         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1960         The default-value is used. If it is between < > it look for a value at instantiation_params
1961         :param primitive_desc: portion of VNFD/NSD that describes primitive
1962         :param params: Params provided by user
1963         :param instantiation_params: Instantiation params provided by user
1964         :return: a dictionary with the calculated params
1965         """
1966         calculated_params = {}
1967         for parameter in primitive_desc.get("parameter", ()):
1968             param_name = parameter["name"]
1969             if param_name in params:
1970                 calculated_params[param_name] = params[param_name]
1971             elif "default-value" in parameter or "value" in parameter:
1972                 if "value" in parameter:
1973                     calculated_params[param_name] = parameter["value"]
1974                 else:
1975                     calculated_params[param_name] = parameter["default-value"]
1976                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1977                         and calculated_params[param_name].endswith(">"):
1978                     if calculated_params[param_name][1:-1] in instantiation_params:
1979                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1980                     else:
1981                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1982                                            format(parameter["default-value"], primitive_desc["name"]))
1983             else:
1984                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1985                                    format(param_name, primitive_desc["name"]))
1986
1987             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1988                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1989                                                                width=256)
1990             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1991                 calculated_params[param_name] = calculated_params[param_name][7:]
1992         return calculated_params
1993
1994     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1995                                     primitive, primitive_params, retries=0, retries_interval=30):
1996         start_primitive_time = time()
1997         try:
1998             for vca_deployed in db_deployed["VCA"]:
1999                 if not vca_deployed:
2000                     continue
2001                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2002                     continue
2003                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2004                     continue
2005                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2006                     continue
2007                 break
2008             else:
2009                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2010                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2011             model_name = vca_deployed.get("model")
2012             application_name = vca_deployed.get("application")
2013             if not model_name or not application_name:
2014                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2015                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2016                                                                        vdu_count_index))
2017             # if vca_deployed["operational-status"] != "active":
2018             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2019             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2020             callback = None  # self.n2vc_callback
2021             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2022             await self.n2vc.login()
2023             if primitive == "config":
2024                 primitive_params = {"params": primitive_params}
2025             while retries >= 0:
2026                 primitive_id = await self.n2vc.ExecutePrimitive(
2027                     model_name,
2028                     application_name,
2029                     primitive,
2030                     callback,
2031                     *callback_args,
2032                     **primitive_params
2033                 )
2034                 while time() - start_primitive_time < self.timeout_primitive:
2035                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2036                     if primitive_result_ in ("completed", "failed"):
2037                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2038                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2039                         break
2040                     elif primitive_result_ is None and primitive == "config":
2041                         primitive_result = "COMPLETED"
2042                         detailed_result = None
2043                         break
2044                     else:  # ("running", "pending", None):
2045                         pass
2046                     await asyncio.sleep(5)
2047                 else:
2048                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2049                 if primitive_result == "COMPLETED":
2050                     break
2051                 retries -= 1
2052                 if retries >= 0:
2053                     await asyncio.sleep(retries_interval)
2054
2055             return primitive_result, detailed_result
2056         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2057             return "FAILED", str(e)
2058
2059     async def action(self, nsr_id, nslcmop_id):
2060
2061         # Try to lock HA task here
2062         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2063         if not task_is_locked_by_me:
2064             return
2065
2066         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2067         self.logger.debug(logging_text + "Enter")
2068         # get all needed from database
2069         db_nsr = None
2070         db_nslcmop = None
2071         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2072         db_nslcmop_update = {}
2073         nslcmop_operation_state = None
2074         nslcmop_operation_state_detail = None
2075         exc = None
2076         try:
2077             # wait for any previous tasks in process
2078             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2079
2080             step = "Getting information from database"
2081             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2082             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2083
2084             nsr_deployed = db_nsr["_admin"].get("deployed")
2085             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2086             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2087             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2088             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2089
2090             if vnf_index:
2091                 step = "Getting vnfr from database"
2092                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2093                 step = "Getting vnfd from database"
2094                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2095             else:
2096                 if db_nsr.get("nsd"):
2097                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2098                 else:
2099                     step = "Getting nsd from database"
2100                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2101
2102             # for backward compatibility
2103             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2104                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2105                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2106                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2107
2108             primitive = db_nslcmop["operationParams"]["primitive"]
2109             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2110
2111             # look for primitive
2112             config_primitive_desc = None
2113             if vdu_id:
2114                 for vdu in get_iterable(db_vnfd, "vdu"):
2115                     if vdu_id == vdu["id"]:
2116                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2117                             if config_primitive["name"] == primitive:
2118                                 config_primitive_desc = config_primitive
2119                                 break
2120             elif vnf_index:
2121                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2122                     if config_primitive["name"] == primitive:
2123                         config_primitive_desc = config_primitive
2124                         break
2125             else:
2126                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2127                     if config_primitive["name"] == primitive:
2128                         config_primitive_desc = config_primitive
2129                         break
2130
2131             if not config_primitive_desc:
2132                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2133                                    format(primitive))
2134
2135             desc_params = {}
2136             if vnf_index:
2137                 if db_vnfr.get("additionalParamsForVnf"):
2138                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2139             else:
2140                 if db_nsr.get("additionalParamsForVnf"):
2141                     desc_params.update(db_nsr["additionalParamsForNs"])
2142
2143             # TODO check if ns is in a proper status
2144             result, result_detail = await self._ns_execute_primitive(
2145                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2146                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2147             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2148             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2149             db_nslcmop_update["statusEnteredTime"] = time()
2150             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2151             return  # database update is called inside finally
2152
2153         except (DbException, LcmException) as e:
2154             self.logger.error(logging_text + "Exit Exception {}".format(e))
2155             exc = e
2156         except asyncio.CancelledError:
2157             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2158             exc = "Operation was cancelled"
2159         except Exception as e:
2160             exc = traceback.format_exc()
2161             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2162         finally:
2163             if exc and db_nslcmop:
2164                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2165                     "FAILED {}: {}".format(step, exc)
2166                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2167                 db_nslcmop_update["statusEnteredTime"] = time()
2168             try:
2169                 if db_nslcmop_update:
2170                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2171                 if db_nsr:
2172                     db_nsr_update["_admin.nslcmop"] = None
2173                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2174             except DbException as e:
2175                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2176             self.logger.debug(logging_text + "Exit")
2177             if nslcmop_operation_state:
2178                 try:
2179                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2180                                                                "operationState": nslcmop_operation_state},
2181                                             loop=self.loop)
2182                 except Exception as e:
2183                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2184             self.logger.debug(logging_text + "Exit")
2185             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2186             return nslcmop_operation_state, nslcmop_operation_state_detail
2187
2188     async def scale(self, nsr_id, nslcmop_id):
2189
2190         # Try to lock HA task here
2191         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2192         if not task_is_locked_by_me:
2193             return
2194
2195         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2196         self.logger.debug(logging_text + "Enter")
2197         # get all needed from database
2198         db_nsr = None
2199         db_nslcmop = None
2200         db_nslcmop_update = {}
2201         nslcmop_operation_state = None
2202         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2203         exc = None
2204         # in case of error, indicates what part of scale was failed to put nsr at error status
2205         scale_process = None
2206         old_operational_status = ""
2207         old_config_status = ""
2208         vnfr_scaled = False
2209         try:
2210             # wait for any previous tasks in process
2211             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2212
2213             step = "Getting nslcmop from database"
2214             self.logger.debug(step + " after having waited for previous tasks to be completed")
2215             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2216             step = "Getting nsr from database"
2217             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2218
2219             old_operational_status = db_nsr["operational-status"]
2220             old_config_status = db_nsr["config-status"]
2221             step = "Parsing scaling parameters"
2222             # self.logger.debug(step)
2223             db_nsr_update["operational-status"] = "scaling"
2224             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2225             nsr_deployed = db_nsr["_admin"].get("deployed")
2226             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2227             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2228             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2229             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2230             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2231
2232             # for backward compatibility
2233             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2234                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2235                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2236                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2237
2238             step = "Getting vnfr from database"
2239             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2240             step = "Getting vnfd from database"
2241             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2242
2243             step = "Getting scaling-group-descriptor"
2244             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2245                 if scaling_descriptor["name"] == scaling_group:
2246                     break
2247             else:
2248                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2249                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2250
2251             # cooldown_time = 0
2252             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2253             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2254             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2255             #         break
2256
2257             # TODO check if ns is in a proper status
2258             step = "Sending scale order to RO"
2259             nb_scale_op = 0
2260             if not db_nsr["_admin"].get("scaling-group"):
2261                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2262                 admin_scale_index = 0
2263             else:
2264                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2265                     if admin_scale_info["name"] == scaling_group:
2266                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2267                         break
2268                 else:  # not found, set index one plus last element and add new entry with the name
2269                     admin_scale_index += 1
2270                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2271             RO_scaling_info = []
2272             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2273             if scaling_type == "SCALE_OUT":
2274                 # count if max-instance-count is reached
2275                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
2276                     max_instance_count = int(scaling_descriptor["max-instance-count"])
2277
2278                     # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(scaling_descriptor["max-instance-count"]))
2279                     if nb_scale_op >= max_instance_count:
2280                         raise LcmException("reached the limit of {} (max-instance-count) "
2281                                            "scaling-out operations for the "
2282                                            "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2283
2284                 nb_scale_op += 1
2285                 vdu_scaling_info["scaling_direction"] = "OUT"
2286                 vdu_scaling_info["vdu-create"] = {}
2287                 for vdu_scale_info in scaling_descriptor["vdu"]:
2288                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2289                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2290                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2291
2292             elif scaling_type == "SCALE_IN":
2293                 # count if min-instance-count is reached
2294                 min_instance_count = 0
2295                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2296                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2297                 if nb_scale_op <= min_instance_count:
2298                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2299                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2300                 nb_scale_op -= 1
2301                 vdu_scaling_info["scaling_direction"] = "IN"
2302                 vdu_scaling_info["vdu-delete"] = {}
2303                 for vdu_scale_info in scaling_descriptor["vdu"]:
2304                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2305                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2306                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2307
2308             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2309             vdu_create = vdu_scaling_info.get("vdu-create")
2310             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2311             if vdu_scaling_info["scaling_direction"] == "IN":
2312                 for vdur in reversed(db_vnfr["vdur"]):
2313                     if vdu_delete.get(vdur["vdu-id-ref"]):
2314                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2315                         vdu_scaling_info["vdu"].append({
2316                             "name": vdur["name"],
2317                             "vdu_id": vdur["vdu-id-ref"],
2318                             "interface": []
2319                         })
2320                         for interface in vdur["interfaces"]:
2321                             vdu_scaling_info["vdu"][-1]["interface"].append({
2322                                 "name": interface["name"],
2323                                 "ip_address": interface["ip-address"],
2324                                 "mac_address": interface.get("mac-address"),
2325                             })
2326                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2327
2328             # execute primitive service PRE-SCALING
2329             step = "Executing pre-scale vnf-config-primitive"
2330             if scaling_descriptor.get("scaling-config-action"):
2331                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2332                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2333                             and scaling_type == "SCALE_IN":
2334                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2335                         step = db_nslcmop_update["detailed-status"] = \
2336                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2337
2338                         # look for primitive
2339                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2340                             if config_primitive["name"] == vnf_config_primitive:
2341                                 break
2342                         else:
2343                             raise LcmException(
2344                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2345                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2346                                 "primitive".format(scaling_group, config_primitive))
2347
2348                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2349                         if db_vnfr.get("additionalParamsForVnf"):
2350                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2351
2352                         scale_process = "VCA"
2353                         db_nsr_update["config-status"] = "configuring pre-scaling"
2354                         result, result_detail = await self._ns_execute_primitive(
2355                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2356                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2357                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2358                             vnf_config_primitive, result, result_detail))
2359                         if result == "FAILED":
2360                             raise LcmException(result_detail)
2361                         db_nsr_update["config-status"] = old_config_status
2362                         scale_process = None
2363
2364             if RO_scaling_info:
2365                 scale_process = "RO"
2366                 RO = ROclient.ROClient(self.loop, **self.ro_config)
2367                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2368                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2369                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2370                 # wait until ready
2371                 RO_nslcmop_id = RO_desc["instance_action_id"]
2372                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2373
2374                 RO_task_done = False
2375                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2376                 detailed_status_old = None
2377                 self.logger.debug(logging_text + step)
2378
2379                 deployment_timeout = 1 * 3600   # One hour
2380                 while deployment_timeout > 0:
2381                     if not RO_task_done:
2382                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2383                                              extra_item_id=RO_nslcmop_id)
2384                         ns_status, ns_status_info = RO.check_action_status(desc)
2385                         if ns_status == "ERROR":
2386                             raise ROclient.ROClientException(ns_status_info)
2387                         elif ns_status == "BUILD":
2388                             detailed_status = step + "; {}".format(ns_status_info)
2389                         elif ns_status == "ACTIVE":
2390                             RO_task_done = True
2391                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2392                             self.logger.debug(logging_text + step)
2393                         else:
2394                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2395                     else:
2396                         desc = await RO.show("ns", RO_nsr_id)
2397                         ns_status, ns_status_info = RO.check_ns_status(desc)
2398                         if ns_status == "ERROR":
2399                             raise ROclient.ROClientException(ns_status_info)
2400                         elif ns_status == "BUILD":
2401                             detailed_status = step + "; {}".format(ns_status_info)
2402                         elif ns_status == "ACTIVE":
2403                             step = detailed_status = \
2404                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2405                             if not vnfr_scaled:
2406                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2407                                 vnfr_scaled = True
2408                             try:
2409                                 desc = await RO.show("ns", RO_nsr_id)
2410                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2411                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2412                                 break
2413                             except LcmExceptionNoMgmtIP:
2414                                 pass
2415                         else:
2416                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2417                     if detailed_status != detailed_status_old:
2418                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2419                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2420
2421                     await asyncio.sleep(5, loop=self.loop)
2422                     deployment_timeout -= 5
2423                 if deployment_timeout <= 0:
2424                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2425
2426                 # update VDU_SCALING_INFO with the obtained ip_addresses
2427                 if vdu_scaling_info["scaling_direction"] == "OUT":
2428                     for vdur in reversed(db_vnfr["vdur"]):
2429                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2430                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2431                             vdu_scaling_info["vdu"].append({
2432                                 "name": vdur["name"],
2433                                 "vdu_id": vdur["vdu-id-ref"],
2434                                 "interface": []
2435                             })
2436                             for interface in vdur["interfaces"]:
2437                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2438                                     "name": interface["name"],
2439                                     "ip_address": interface["ip-address"],
2440                                     "mac_address": interface.get("mac-address"),
2441                                 })
2442                     del vdu_scaling_info["vdu-create"]
2443
2444             scale_process = None
2445             if db_nsr_update:
2446                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2447
2448             # execute primitive service POST-SCALING
2449             step = "Executing post-scale vnf-config-primitive"
2450             if scaling_descriptor.get("scaling-config-action"):
2451                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2452                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2453                             and scaling_type == "SCALE_OUT":
2454                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2455                         step = db_nslcmop_update["detailed-status"] = \
2456                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2457
2458                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2459                         if db_vnfr.get("additionalParamsForVnf"):
2460                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2461
2462                         # look for primitive
2463                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2464                             if config_primitive["name"] == vnf_config_primitive:
2465                                 break
2466                         else:
2467                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2468                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2469                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2470                                                                                                      config_primitive))
2471                         scale_process = "VCA"
2472                         db_nsr_update["config-status"] = "configuring post-scaling"
2473
2474                         result, result_detail = await self._ns_execute_primitive(
2475                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2476                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2477                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2478                             vnf_config_primitive, result, result_detail))
2479                         if result == "FAILED":
2480                             raise LcmException(result_detail)
2481                         db_nsr_update["config-status"] = old_config_status
2482                         scale_process = None
2483
2484             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2485             db_nslcmop_update["statusEnteredTime"] = time()
2486             db_nslcmop_update["detailed-status"] = "done"
2487             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2488             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2489                 else old_operational_status
2490             db_nsr_update["config-status"] = old_config_status
2491             return
2492         except (ROclient.ROClientException, DbException, LcmException) as e:
2493             self.logger.error(logging_text + "Exit Exception {}".format(e))
2494             exc = e
2495         except asyncio.CancelledError:
2496             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2497             exc = "Operation was cancelled"
2498         except Exception as e:
2499             exc = traceback.format_exc()
2500             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2501         finally:
2502             if exc:
2503                 if db_nslcmop:
2504                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2505                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2506                     db_nslcmop_update["statusEnteredTime"] = time()
2507                 if db_nsr:
2508                     db_nsr_update["operational-status"] = old_operational_status
2509                     db_nsr_update["config-status"] = old_config_status
2510                     db_nsr_update["detailed-status"] = ""
2511                     db_nsr_update["_admin.nslcmop"] = None
2512                     if scale_process:
2513                         if "VCA" in scale_process:
2514                             db_nsr_update["config-status"] = "failed"
2515                         if "RO" in scale_process:
2516                             db_nsr_update["operational-status"] = "failed"
2517                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2518                                                                                                      exc)
2519             try:
2520                 if db_nslcmop and db_nslcmop_update:
2521                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2522                 if db_nsr:
2523                     db_nsr_update["_admin.nslcmop"] = None
2524                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2525             except DbException as e:
2526                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2527             if nslcmop_operation_state:
2528                 try:
2529                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2530                                                              "operationState": nslcmop_operation_state},
2531                                             loop=self.loop)
2532                     # if cooldown_time:
2533                     #     await asyncio.sleep(cooldown_time)
2534                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2535                 except Exception as e:
2536                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2537             self.logger.debug(logging_text + "Exit")
2538             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")