osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 106
 107     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 108         """
 109         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 110         :param vnfd: input vnfd
 111         :param new_id: overrides vnf id if provided
 112         :param additionalParams: Instantiation params for VNFs provided
 113         :param nsrId: Id of the NSR
 114         :return: copy of vnfd
 115         """
 116         try:
 117             vnfd_RO = deepcopy(vnfd)
 118             # remove unused by RO configuration, monitoring, scaling and internal keys
 119             vnfd_RO.pop("_id", None)
 120             vnfd_RO.pop("_admin", None)
 121             vnfd_RO.pop("vnf-configuration", None)
 122             vnfd_RO.pop("monitoring-param", None)
 123             vnfd_RO.pop("scaling-group-descriptor", None)
 124             if new_id:
 125                 vnfd_RO["id"] = new_id
 126
 127             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 128             for vdu in get_iterable(vnfd_RO, "vdu"):
 129                 cloud_init_file = None
 130                 if vdu.get("cloud-init-file"):
 131                     base_folder = vnfd["_admin"]["storage"]
 132                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 133                                                                    vdu["cloud-init-file"])
 134                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 135                         cloud_init_content = ci_file.read()
 136                     vdu.pop("cloud-init-file", None)
 137                 elif vdu.get("cloud-init"):
 138                     cloud_init_content = vdu["cloud-init"]
 139                 else:
 140                     continue
 141
 142                 env = Environment()
 143                 ast = env.parse(cloud_init_content)
 144                 mandatory_vars = meta.find_undeclared_variables(ast)
 145                 if mandatory_vars:
 146                     for var in mandatory_vars:
 147                         if not additionalParams or var not in additionalParams.keys():
 148                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 149                                                "file, must be provided in the instantiation parameters inside the "
 150                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 151                 template = Template(cloud_init_content)
 152                 cloud_init_content = template.render(additionalParams or {})
 153                 vdu["cloud-init"] = cloud_init_content
 154
 155             return vnfd_RO
 156         except FsException as e:
 157             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 158                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 159         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 160             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 161                                format(vnfd["id"], vdu["id"], e))
 162
 163     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 164         """
 165         Callback both for charm status change and task completion
 166         :param model_name: Charm model name
 167         :param application_name: Charm application name
 168         :param status: Can be
 169             - blocked: The unit needs manual intervention
 170             - maintenance: The unit is actively deploying/configuring
 171             - waiting: The unit is waiting for another charm to be ready
 172             - active: The unit is deployed, configured, and ready
 173             - error: The charm has failed and needs attention.
 174             - terminated: The charm has been destroyed
 175             - removing,
 176             - removed
 177         :param message: detailed message error
 178         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 179             nsr_id:
 180             nslcmop_id:
 181             lcmOperationType: currently "instantiate"
 182             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 183             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 184             n2vc_event: event used to notify instantiation task that some change has been produced
 185         :param task: None for charm status change, or task for completion task callback
 186         :return:
 187         """
 188         try:
 189             nsr_id = n2vc_info["nsr_id"]
 190             deployed = n2vc_info["deployed"]
 191             db_nsr_update = n2vc_info["db_update"]
 192             nslcmop_id = n2vc_info["nslcmop_id"]
 193             ns_operation = n2vc_info["lcmOperationType"]
 194             n2vc_event = n2vc_info["n2vc_event"]
 195             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 196                                                                                     application_name)
 197             for vca_index, vca_deployed in enumerate(deployed):
 198                 if not vca_deployed:
 199                     continue
 200                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 201                     break
 202             else:
 203                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 204                                   format(model_name))
 205                 return
 206             if task:
 207                 if task.cancelled():
 208                     self.logger.debug(logging_text + " task Cancelled")
 209                     vca_deployed['operational-status'] = "error"
 210                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 211                     vca_deployed['detailed-status'] = "Task Cancelled"
 212                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 213
 214                 elif task.done():
 215                     exc = task.exception()
 216                     if exc:
 217                         self.logger.error(logging_text + " task Exception={}".format(exc))
 218                         vca_deployed['operational-status'] = "error"
 219                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 220                         vca_deployed['detailed-status'] = str(exc)
 221                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 222                     else:
 223                         self.logger.debug(logging_text + " task Done")
 224                         # task is Done, but callback is still ongoing. So ignore
 225                         return
 226             elif status:
 227                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 228                 if vca_deployed['operational-status'] == status:
 229                     return  # same status, ignore
 230                 vca_deployed['operational-status'] = status
 231                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 232                 vca_deployed['detailed-status'] = str(message)
 233                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 234             else:
 235                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 236                 return
 237             # wake up instantiate task
 238             n2vc_event.set()
 239         except Exception as e:
 240             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 241
 242     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 243         """
 244         Creates a RO ns descriptor from OSM ns_instantiate params
 245         :param ns_params: OSM instantiate params
 246         :return: The RO ns descriptor
 247         """
 248         vim_2_RO = {}
 249         wim_2_RO = {}
 250         # TODO feature 1417: Check that no instantiation is set over PDU
 251         # check if PDU forces a concrete vim-network-id and add it
 252         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 253
 254         def vim_account_2_RO(vim_account):
 255             if vim_account in vim_2_RO:
 256                 return vim_2_RO[vim_account]
 257
 258             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 259             if db_vim["_admin"]["operationalState"] != "ENABLED":
 260                 raise LcmException("VIM={} is not available. operationalState={}".format(
 261                     vim_account, db_vim["_admin"]["operationalState"]))
 262             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 263             vim_2_RO[vim_account] = RO_vim_id
 264             return RO_vim_id
 265
 266         def wim_account_2_RO(wim_account):
 267             if isinstance(wim_account, str):
 268                 if wim_account in wim_2_RO:
 269                     return wim_2_RO[wim_account]
 270
 271                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 272                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 273                     raise LcmException("WIM={} is not available. operationalState={}".format(
 274                         wim_account, db_wim["_admin"]["operationalState"]))
 275                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 276                 wim_2_RO[wim_account] = RO_wim_id
 277                 return RO_wim_id
 278             else:
 279                 return wim_account
 280
 281         def ip_profile_2_RO(ip_profile):
 282             RO_ip_profile = deepcopy((ip_profile))
 283             if "dns-server" in RO_ip_profile:
 284                 if isinstance(RO_ip_profile["dns-server"], list):
 285                     RO_ip_profile["dns-address"] = []
 286                     for ds in RO_ip_profile.pop("dns-server"):
 287                         RO_ip_profile["dns-address"].append(ds['address'])
 288                 else:
 289                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 290             if RO_ip_profile.get("ip-version") == "ipv4":
 291                 RO_ip_profile["ip-version"] = "IPv4"
 292             if RO_ip_profile.get("ip-version") == "ipv6":
 293                 RO_ip_profile["ip-version"] = "IPv6"
 294             if "dhcp-params" in RO_ip_profile:
 295                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 296             return RO_ip_profile
 297
 298         if not ns_params:
 299             return None
 300         RO_ns_params = {
 301             # "name": ns_params["nsName"],
 302             # "description": ns_params.get("nsDescription"),
 303             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 304             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 305             # "scenario": ns_params["nsdId"],
 306         }
 307         n2vc_key_list = n2vc_key_list or []
 308         for vnfd_ref, vnfd in vnfd_dict.items():
 309             vdu_needed_access = []
 310             mgmt_cp = None
 311             if vnfd.get("vnf-configuration"):
 312                 ssh_required = vnfd["vnf-configuration"].get("config-access", {}).get("ssh-access").get("required")
 313                 if ssh_required and vnfd.get("mgmt-interface"):
 314                     if vnfd["mgmt-interface"].get("vdu-id"):
 315                         vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 316                     elif vnfd["mgmt-interface"].get("cp"):
 317                         mgmt_cp = vnfd["mgmt-interface"]["cp"]
 318
 319             for vdu in vnfd.get("vdu", ()):
 320                 if vdu.get("vdu-configuration"):
 321                     ssh_required = vdu["vdu-configuration"].get("config-access", {}).get("ssh-access").get("required")
 322                     if ssh_required:
 323                         vdu_needed_access.append(vdu["id"])
 324                 elif mgmt_cp:
 325                     for vdu_interface in vdu.get("interface"):
 326                         if vdu_interface.get("external-connection-point-ref") and \
 327                                 vdu_interface["external-connection-point-ref"] == mgmt_cp:
 328                             vdu_needed_access.append(vdu["id"])
 329                             mgmt_cp = None
 330                             break
 331
 332             if vdu_needed_access:
 333                 for vnf_member in nsd.get("constituent-vnfd"):
 334                     if vnf_member["vnfd-id-ref"] != vnfd_ref:
 335                         continue
 336                     for vdu in vdu_needed_access:
 337                         populate_dict(RO_ns_params,
 338                                       ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 339                                       n2vc_key_list)
 340
 341         if ns_params.get("vduImage"):
 342             RO_ns_params["vduImage"] = ns_params["vduImage"]
 343
 344         if ns_params.get("ssh_keys"):
 345             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 346         for vnf_params in get_iterable(ns_params, "vnf"):
 347             for constituent_vnfd in nsd["constituent-vnfd"]:
 348                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 349                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 350                     break
 351             else:
 352                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 353                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 354             if vnf_params.get("vimAccountId"):
 355                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 356                               vim_account_2_RO(vnf_params["vimAccountId"]))
 357
 358             for vdu_params in get_iterable(vnf_params, "vdu"):
 359                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 360                 if vdu_params.get("volume"):
 361                     for volume_params in vdu_params["volume"]:
 362                         if volume_params.get("vim-volume-id"):
 363                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 364                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 365                                           volume_params["vim-volume-id"])
 366                 if vdu_params.get("interface"):
 367                     for interface_params in vdu_params["interface"]:
 368                         if interface_params.get("ip-address"):
 369                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 370                                                          vdu_params["id"], "interfaces", interface_params["name"],
 371                                                          "ip_address"),
 372                                           interface_params["ip-address"])
 373                         if interface_params.get("mac-address"):
 374                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 375                                                          vdu_params["id"], "interfaces", interface_params["name"],
 376                                                          "mac_address"),
 377                                           interface_params["mac-address"])
 378                         if interface_params.get("floating-ip-required"):
 379                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 380                                                          vdu_params["id"], "interfaces", interface_params["name"],
 381                                                          "floating-ip"),
 382                                           interface_params["floating-ip-required"])
 383
 384             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 385                 if internal_vld_params.get("vim-network-name"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-name"),
 388                                   internal_vld_params["vim-network-name"])
 389                 if internal_vld_params.get("vim-network-id"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "vim-network-id"),
 392                                   internal_vld_params["vim-network-id"])
 393                 if internal_vld_params.get("ip-profile"):
 394                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 395                                                  internal_vld_params["name"], "ip-profile"),
 396                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 397
 398                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 399                     # look for interface
 400                     iface_found = False
 401                     for vdu_descriptor in vnf_descriptor["vdu"]:
 402                         for vdu_interface in vdu_descriptor["interface"]:
 403                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 404                                 if icp_params.get("ip-address"):
 405                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 406                                                                  vdu_descriptor["id"], "interfaces",
 407                                                                  vdu_interface["name"], "ip_address"),
 408                                                   icp_params["ip-address"])
 409
 410                                 if icp_params.get("mac-address"):
 411                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 412                                                                  vdu_descriptor["id"], "interfaces",
 413                                                                  vdu_interface["name"], "mac_address"),
 414                                                   icp_params["mac-address"])
 415                                 iface_found = True
 416                                 break
 417                         if iface_found:
 418                             break
 419                     else:
 420                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 421                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 422                                            "connection-point".format(vnf_params["member-vnf-index"],
 423                                                                      icp_params["id-ref"]))
 424
 425         for vld_params in get_iterable(ns_params, "vld"):
 426             if "ip-profile" in vld_params:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 428                               ip_profile_2_RO(vld_params["ip-profile"]))
 429
 430             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 431                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 432                               wim_account_2_RO(vld_params["wimAccountId"])),
 433             if vld_params.get("vim-network-name"):
 434                 RO_vld_sites = []
 435                 if isinstance(vld_params["vim-network-name"], dict):
 436                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 437                         RO_vld_sites.append({
 438                             "netmap-use": vim_net,
 439                             "datacenter": vim_account_2_RO(vim_account)
 440                         })
 441                 else:  # isinstance str
 442                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 443                 if RO_vld_sites:
 444                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 445             if vld_params.get("vim-network-id"):
 446                 RO_vld_sites = []
 447                 if isinstance(vld_params["vim-network-id"], dict):
 448                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 449                         RO_vld_sites.append({
 450                             "netmap-use": vim_net,
 451                             "datacenter": vim_account_2_RO(vim_account)
 452                         })
 453                 else:  # isinstance str
 454                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 455                 if RO_vld_sites:
 456                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 457             if vld_params.get("ns-net"):
 458                 if isinstance(vld_params["ns-net"], dict):
 459                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 460                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 461                 if RO_vld_ns_net:
 462                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 463             if "vnfd-connection-point-ref" in vld_params:
 464                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 465                     # look for interface
 466                     for constituent_vnfd in nsd["constituent-vnfd"]:
 467                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 468                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 469                             break
 470                     else:
 471                         raise LcmException(
 472                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 473                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 474                     match_cp = False
 475                     for vdu_descriptor in vnf_descriptor["vdu"]:
 476                         for interface_descriptor in vdu_descriptor["interface"]:
 477                             if interface_descriptor.get("external-connection-point-ref") == \
 478                                     cp_params["vnfd-connection-point-ref"]:
 479                                 match_cp = True
 480                                 break
 481                         if match_cp:
 482                             break
 483                     else:
 484                         raise LcmException(
 485                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 486                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 487                                 cp_params["member-vnf-index-ref"],
 488                                 cp_params["vnfd-connection-point-ref"],
 489                                 vnf_descriptor["id"]))
 490                     if cp_params.get("ip-address"):
 491                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 492                                                      vdu_descriptor["id"], "interfaces",
 493                                                      interface_descriptor["name"], "ip_address"),
 494                                       cp_params["ip-address"])
 495                     if cp_params.get("mac-address"):
 496                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 497                                                      vdu_descriptor["id"], "interfaces",
 498                                                      interface_descriptor["name"], "mac_address"),
 499                                       cp_params["mac-address"])
 500         return RO_ns_params
 501
 502     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 503         # make a copy to do not change
 504         vdu_create = copy(vdu_create)
 505         vdu_delete = copy(vdu_delete)
 506
 507         vdurs = db_vnfr.get("vdur")
 508         if vdurs is None:
 509             vdurs = []
 510         vdu_index = len(vdurs)
 511         while vdu_index:
 512             vdu_index -= 1
 513             vdur = vdurs[vdu_index]
 514             if vdur.get("pdu-type"):
 515                 continue
 516             vdu_id_ref = vdur["vdu-id-ref"]
 517             if vdu_create and vdu_create.get(vdu_id_ref):
 518                 for index in range(0, vdu_create[vdu_id_ref]):
 519                     vdur = deepcopy(vdur)
 520                     vdur["_id"] = str(uuid4())
 521                     vdur["count-index"] += 1
 522                     vdurs.insert(vdu_index+1+index, vdur)
 523                 del vdu_create[vdu_id_ref]
 524             if vdu_delete and vdu_delete.get(vdu_id_ref):
 525                 del vdurs[vdu_index]
 526                 vdu_delete[vdu_id_ref] -= 1
 527                 if not vdu_delete[vdu_id_ref]:
 528                     del vdu_delete[vdu_id_ref]
 529         # check all operations are done
 530         if vdu_create or vdu_delete:
 531             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 532                 vdu_create))
 533         if vdu_delete:
 534             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 535                 vdu_delete))
 536
 537         vnfr_update = {"vdur": vdurs}
 538         db_vnfr["vdur"] = vdurs
 539         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 540
 541     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 542         """
 543         Updates database nsr with the RO info for the created vld
 544         :param ns_update_nsr: dictionary to be filled with the updated info
 545         :param db_nsr: content of db_nsr. This is also modified
 546         :param nsr_desc_RO: nsr descriptor from RO
 547         :return: Nothing, LcmException is raised on errors
 548         """
 549
 550         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 551             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 552                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 553                     continue
 554                 vld["vim-id"] = net_RO.get("vim_net_id")
 555                 vld["name"] = net_RO.get("vim_name")
 556                 vld["status"] = net_RO.get("status")
 557                 vld["status-detailed"] = net_RO.get("error_msg")
 558                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 559                 break
 560             else:
 561                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 562
 563     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 564         """
 565         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 566         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 567         :param nsr_desc_RO: nsr descriptor from RO
 568         :return: Nothing, LcmException is raised on errors
 569         """
 570         for vnf_index, db_vnfr in db_vnfrs.items():
 571             for vnf_RO in nsr_desc_RO["vnfs"]:
 572                 if vnf_RO["member_vnf_index"] != vnf_index:
 573                     continue
 574                 vnfr_update = {}
 575                 if vnf_RO.get("ip_address"):
 576                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 577                 elif not db_vnfr.get("ip-address"):
 578                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 579
 580                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 581                     vdur_RO_count_index = 0
 582                     if vdur.get("pdu-type"):
 583                         continue
 584                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 585                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 586                             continue
 587                         if vdur["count-index"] != vdur_RO_count_index:
 588                             vdur_RO_count_index += 1
 589                             continue
 590                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 591                         if vdur_RO.get("ip_address"):
 592                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 593                         else:
 594                             vdur["ip-address"] = None
 595                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 596                         vdur["name"] = vdur_RO.get("vim_name")
 597                         vdur["status"] = vdur_RO.get("status")
 598                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 599                         for ifacer in get_iterable(vdur, "interfaces"):
 600                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 601                                 if ifacer["name"] == interface_RO.get("internal_name"):
 602                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 603                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 604                                     break
 605                             else:
 606                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 607                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 608                                                                           ifacer["name"]))
 609                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 610                         break
 611                     else:
 612                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 613                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 614
 615                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 616                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 617                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 618                             continue
 619                         vld["vim-id"] = net_RO.get("vim_net_id")
 620                         vld["name"] = net_RO.get("vim_name")
 621                         vld["status"] = net_RO.get("status")
 622                         vld["status-detailed"] = net_RO.get("error_msg")
 623                         vnfr_update["vld.{}".format(vld_index)] = vld
 624                         break
 625                     else:
 626                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 627                             vnf_index, vld["id"]))
 628
 629                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 630                 break
 631
 632             else:
 633                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 634
 635     async def instantiate(self, nsr_id, nslcmop_id):
 636
 637         # Try to lock HA task here
 638         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 639         if not task_is_locked_by_me:
 640             return
 641
 642         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 643         self.logger.debug(logging_text + "Enter")
 644         # get all needed from database
 645         start_deploy = time()
 646         db_nsr = None
 647         db_nslcmop = None
 648         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 649         db_nslcmop_update = {}
 650         nslcmop_operation_state = None
 651         db_vnfrs = {}
 652         RO_descriptor_number = 0   # number of descriptors created at RO
 653         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 654         n2vc_info = {}
 655         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 656         exc = None
 657         try:
 658             # wait for any previous tasks in process
 659             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 660
 661             step = "Getting nslcmop={} from db".format(nslcmop_id)
 662             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 663             step = "Getting nsr={} from db".format(nsr_id)
 664             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 665             ns_params = db_nslcmop.get("operationParams")
 666             nsd = db_nsr["nsd"]
 667             nsr_name = db_nsr["name"]   # TODO short-name??
 668
 669             step = "Getting vnfrs from db"
 670             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 671             db_vnfds_ref = {}
 672             db_vnfds = {}
 673             db_vnfds_index = {}
 674             for vnfr in db_vnfrs_list:
 675                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 676                 vnfd_id = vnfr["vnfd-id"]
 677                 vnfd_ref = vnfr["vnfd-ref"]
 678                 if vnfd_id not in db_vnfds:
 679                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 680                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 681                     db_vnfds_ref[vnfd_ref] = vnfd
 682                     db_vnfds[vnfd_id] = vnfd
 683                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 684
 685             # Get or generates the _admin.deployed,VCA list
 686             vca_deployed_list = None
 687             vca_model_name = None
 688             if db_nsr["_admin"].get("deployed"):
 689                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 690                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 691             if vca_deployed_list is None:
 692                 vca_deployed_list = []
 693                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 694                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 695             elif isinstance(vca_deployed_list, dict):
 696                 # maintain backward compatibility. Change a dict to list at database
 697                 vca_deployed_list = list(vca_deployed_list.values())
 698                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 699                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 700
 701             db_nsr_update["detailed-status"] = "creating"
 702             db_nsr_update["operational-status"] = "init"
 703             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 704                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 705                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 706                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 707
 708             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 709             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 710             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 711
 712             # Deploy charms
 713             # The parameters we'll need to deploy a charm
 714             number_to_configure = 0
 715
 716             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 717                 """An inner function to deploy the charm from either ns, vnf or vdu
 718                 For ns both vnf_index and vdu_id are None.
 719                 For vnf only vdu_id is None
 720                 For vdu both vnf_index and vdu_id contain a value
 721                 """
 722                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 723                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 724
 725                 machine_spec = {}
 726                 if native_charm:
 727                     machine_spec["username"] = charm_params.get("username"),
 728                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 729
 730                 # Note: The charm needs to exist on disk at the location
 731                 # specified by charm_path.
 732                 descriptor = vnfd if vnf_index else nsd
 733                 base_folder = descriptor["_admin"]["storage"]
 734                 storage_params = self.fs.get_params()
 735                 charm_path = "{}{}/{}/charms/{}".format(
 736                     storage_params["path"],
 737                     base_folder["folder"],
 738                     base_folder["pkg-dir"],
 739                     proxy_charm
 740                 )
 741
 742                 # ns_name will be ignored in the current version of N2VC
 743                 # but will be implemented for the next point release.
 744                 model_name = nsr_id
 745                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 746                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 747                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 748
 749                 vca_index = len(vca_deployed_list)
 750                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 751                 # 26*26 charm in the same NS
 752                 application_name = application_name[0:48]
 753                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 754                 vca_deployed_ = {
 755                     "member-vnf-index": vnf_index,
 756                     "vdu_id": vdu_id,
 757                     "model": model_name,
 758                     "application": application_name,
 759                     "operational-status": "init",
 760                     "detailed-status": "",
 761                     "step": "initial-deploy",
 762                     "vnfd_id": vnfd_id,
 763                     "vdu_name": vdu_name,
 764                     "vdu_count_index": vdu_count_index,
 765                 }
 766                 vca_deployed_list.append(vca_deployed_)
 767                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 768                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 769
 770                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 771                                                                                                 proxy_charm))
 772                 if not n2vc_info:
 773                     n2vc_info["nsr_id"] = nsr_id
 774                     n2vc_info["nslcmop_id"] = nslcmop_id
 775                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 776                     n2vc_info["lcmOperationType"] = "instantiate"
 777                     n2vc_info["deployed"] = vca_deployed_list
 778                     n2vc_info["db_update"] = db_nsr_update
 779                 task = asyncio.ensure_future(
 780                     self.n2vc.DeployCharms(
 781                         model_name,          # The network service name
 782                         application_name,    # The application name
 783                         descriptor,          # The vnf/nsd descriptor
 784                         charm_path,          # Path to charm
 785                         charm_params,        # Runtime params, like mgmt ip
 786                         machine_spec,        # for native charms only
 787                         self.n2vc_callback,  # Callback for status changes
 788                         n2vc_info,           # Callback parameter
 789                         None,                # Callback parameter (task)
 790                     )
 791                 )
 792                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 793                                                          n2vc_info))
 794                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 795
 796             step = "Looking for needed vnfd to configure with proxy charm"
 797             self.logger.debug(logging_text + step)
 798
 799             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 800                 vnfd_id = c_vnf["vnfd-id-ref"]
 801                 vnf_index = str(c_vnf["member-vnf-index"])
 802                 vnfd = db_vnfds_ref[vnfd_id]
 803
 804                 # Get additional parameters
 805                 vnfr_params = {}
 806                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 807                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 808                 for k, v in vnfr_params.items():
 809                     if isinstance(v, str) and v.startswith("!!yaml "):
 810                         vnfr_params[k] = yaml.safe_load(v[7:])
 811
 812                 step = "deploying proxy charms for configuration"
 813                 # Check if this VNF has a charm configuration
 814                 vnf_config = vnfd.get("vnf-configuration")
 815                 if vnf_config and vnf_config.get("juju"):
 816                     proxy_charm = vnf_config["juju"]["charm"]
 817                     if vnf_config["juju"].get("proxy") is False:
 818                         # native_charm, will be deployed after VM. Skip
 819                         proxy_charm = None
 820
 821                     if proxy_charm:
 822                         if not vca_model_name:
 823                             step = "creating VCA model name '{}'".format(nsr_id)
 824                             self.logger.debug(logging_text + step)
 825                             await self.n2vc.CreateNetworkService(nsr_id)
 826                             vca_model_name = nsr_id
 827                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 828                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 829                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 830                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 831                         charm_params = {
 832                             "user_values": vnfr_params,
 833                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 834                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 835                         }
 836
 837                         # Login to the VCA. If there are multiple calls to login(),
 838                         # subsequent calls will be a nop and return immediately.
 839                         await self.n2vc.login()
 840
 841                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 842                         number_to_configure += 1
 843
 844                 # Deploy charms for each VDU that supports one.
 845                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 846                     vdu_config = vdu.get('vdu-configuration')
 847                     proxy_charm = None
 848
 849                     if vdu_config and vdu_config.get("juju"):
 850                         proxy_charm = vdu_config["juju"]["charm"]
 851                         if vdu_config["juju"].get("proxy") is False:
 852                             # native_charm, will be deployed after VM. Skip
 853                             proxy_charm = None
 854
 855                         if proxy_charm:
 856                             if not vca_model_name:
 857                                 step = "creating VCA model name"
 858                                 await self.n2vc.CreateNetworkService(nsr_id)
 859                                 vca_model_name = nsr_id
 860                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 861                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 862                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 863                                                                                                           vdu["id"])
 864                             await self.n2vc.login()
 865                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 866                             # TODO for the moment only first vdu_id contains a charm deployed
 867                             if vdur["vdu-id-ref"] != vdu["id"]:
 868                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 869                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 870                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 871                             charm_params = {
 872                                 "user_values": vnfr_params,
 873                                 "rw_mgmt_ip": vdur["ip-address"],
 874                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 875                             }
 876                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 877                                          charm_params, n2vc_info)
 878                             number_to_configure += 1
 879
 880             # Check if this NS has a charm configuration
 881
 882             ns_config = nsd.get("ns-configuration")
 883             if ns_config and ns_config.get("juju"):
 884                 proxy_charm = ns_config["juju"]["charm"]
 885                 if ns_config["juju"].get("proxy") is False:
 886                     # native_charm, will be deployed after VM. Skip
 887                     proxy_charm = None
 888
 889                 if proxy_charm:
 890                     step = "deploying proxy charm to configure ns"
 891                     # TODO is NS magmt IP address needed?
 892
 893                     # Get additional parameters
 894                     additional_params = {}
 895                     if db_nsr.get("additionalParamsForNs"):
 896                         additional_params = db_nsr["additionalParamsForNs"].copy()
 897                     for k, v in additional_params.items():
 898                         if isinstance(v, str) and v.startswith("!!yaml "):
 899                             additional_params[k] = yaml.safe_load(v[7:])
 900
 901                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 902                     charm_params = {
 903                         "user_values": additional_params,
 904                         # "rw_mgmt_ip": db_nsr["ip-address"],
 905                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 906                     }
 907
 908                     # Login to the VCA. If there are multiple calls to login(),
 909                     # subsequent calls will be a nop and return immediately.
 910                     await self.n2vc.login()
 911                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 912                     number_to_configure += 1
 913
 914             db_nsr_update["operational-status"] = "running"
 915
 916             # Wait until all charms has reached blocked or active status
 917             step = "waiting proxy charms to be ready"
 918             if number_to_configure:
 919                 # wait until all charms are configured.
 920                 # steps are:
 921                 #       initial-deploy
 922                 #       get-ssh-public-key
 923                 #       generate-ssh-key
 924                 #       retry-get-ssh-public-key
 925                 #       ssh-public-key-obtained
 926                 while time() <= start_deploy + self.total_deploy_timeout:
 927                     if db_nsr_update:
 928                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 929                     if db_nslcmop_update:
 930                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 931
 932                     all_active = True
 933                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 934                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 935                         if vca_deployed["step"] == "initial-deploy":
 936                             if vca_deployed["operational-status"] in ("active", "blocked"):
 937                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 938                                        .format(vca_deployed["member-vnf-index"],
 939                                                vca_deployed["vdu_id"])
 940                                 self.logger.debug(logging_text + step)
 941                                 try:
 942                                     primitive_id = await self.n2vc.ExecutePrimitive(
 943                                         vca_deployed["model"],
 944                                         vca_deployed["application"],
 945                                         "get-ssh-public-key",
 946                                         None,
 947                                     )
 948                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 949                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 950                                         primitive_id
 951                                     db_nsr_update[database_entry + "operational-status"] =\
 952                                         vca_deployed["operational-status"]
 953                                 except PrimitiveDoesNotExist:
 954                                     ssh_public_key = None
 955                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 956                                         "ssh-public-key-obtained"
 957                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 958                                         ssh_public_key
 959                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 960                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 961                                     self.logger.debug(logging_text + step)
 962
 963                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 964                             primitive_id = vca_deployed["primitive_id"]
 965                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 966                                                                                   primitive_id)
 967                             if primitive_status in ("completed", "failed"):
 968                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 969                                                                                       primitive_id)
 970                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 971                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 972                                         primitive_result.get("pubkey"):
 973                                     ssh_public_key = primitive_result.get("pubkey")
 974                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 975                                         "ssh-public-key-obtained"
 976                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 977                                         ssh_public_key
 978                                     n2vc_key_list.append(ssh_public_key)
 979                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 980                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 981                                     self.logger.debug(logging_text + step)
 982                                 else:  # primitive_status == "failed":
 983                                     if vca_deployed["step"] == "get-ssh-public-key":
 984                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 985                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 986                                                                      vca_deployed["vdu_id"])
 987                                         self.logger.debug(logging_text + step)
 988                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 989                                             "generate-ssh-key"
 990                                         primitive_id = await self.n2vc.ExecutePrimitive(
 991                                             vca_deployed["model"],
 992                                             vca_deployed["application"],
 993                                             "generate-ssh-key",
 994                                             None,
 995                                         )
 996                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 997                                             primitive_id
 998                                     else:  # failed for second time
 999                                         raise LcmException(
1000                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
1001
1002                         elif vca_deployed["step"] == "generate-ssh-key":
1003                             primitive_id = vca_deployed["primitive_id"]
1004                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1005                                                                                   primitive_id)
1006                             if primitive_status in ("completed", "failed"):
1007                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1008                                                                                       primitive_id)
1009                                 vca_deployed["primitive_id"] = db_nsr_update[
1010                                     database_entry + "primitive_id"] = None
1011                                 if primitive_status == "completed":
1012                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1013                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1014                                                               vca_deployed["vdu_id"])
1015                                     self.logger.debug(logging_text + step)
1016                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1017                                         "retry-get-ssh-public-key"
1018                                     primitive_id = await self.n2vc.ExecutePrimitive(
1019                                         vca_deployed["model"],
1020                                         vca_deployed["application"],
1021                                         "get-ssh-public-key",
1022                                         None,
1023                                     )
1024                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1025                                         primitive_id
1026
1027                                 else:  # primitive_status == "failed":
1028                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1029                                                        .format(primitive_result))
1030
1031                         if vca_deployed["step"] != "ssh-public-key-obtained":
1032                             all_active = False
1033
1034                     if all_active:
1035                         break
1036                     await asyncio.sleep(5)
1037                 else:   # total_deploy_timeout
1038                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1039                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1040
1041             # deploy RO
1042             # get vnfds, instantiate at RO
1043             for c_vnf in nsd.get("constituent-vnfd", ()):
1044                 member_vnf_index = c_vnf["member-vnf-index"]
1045                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1046                 vnfd_ref = vnfd["id"]
1047                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1048                     vnfd_ref, member_vnf_index)
1049                 # self.logger.debug(logging_text + step)
1050                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1051                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1052                 RO_descriptor_number += 1
1053
1054                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1055                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1056                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1057                         break
1058                 else:
1059                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1060                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1061
1062                 # look if present
1063                 RO_update = {"member-vnf-index": member_vnf_index}
1064                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1065                 if vnfd_list:
1066                     RO_update["id"] = vnfd_list[0]["uuid"]
1067                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1068                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1069                 else:
1070                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1071                                            get("additionalParamsForVnf"), nsr_id)
1072                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1073                     RO_update["id"] = desc["uuid"]
1074                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1075                         vnfd_ref, member_vnf_index, desc["uuid"]))
1076                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1077                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1078                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1079
1080             # create nsd at RO
1081             nsd_ref = nsd["id"]
1082             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1083             # self.logger.debug(logging_text + step)
1084
1085             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1086             RO_descriptor_number += 1
1087             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1088             if nsd_list:
1089                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1090                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1091                     nsd_ref, RO_nsd_uuid))
1092             else:
1093                 nsd_RO = deepcopy(nsd)
1094                 nsd_RO["id"] = RO_osm_nsd_id
1095                 nsd_RO.pop("_id", None)
1096                 nsd_RO.pop("_admin", None)
1097                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1098                     member_vnf_index = c_vnf["member-vnf-index"]
1099                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1100                 for c_vld in nsd_RO.get("vld", ()):
1101                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1102                         member_vnf_index = cp["member-vnf-index-ref"]
1103                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1104
1105                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1106                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1107                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1108                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1109             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1110
1111             # Crate ns at RO
1112             # if present use it unless in error status
1113             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1114             if RO_nsr_id:
1115                 try:
1116                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1117                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1118                     desc = await self.RO.show("ns", RO_nsr_id)
1119                 except ROclient.ROClientException as e:
1120                     if e.http_code != HTTPStatus.NOT_FOUND:
1121                         raise
1122                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1123                 if RO_nsr_id:
1124                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1125                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1126                     if ns_status == "ERROR":
1127                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1128                         self.logger.debug(logging_text + step)
1129                         await self.RO.delete("ns", RO_nsr_id)
1130                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1131             if not RO_nsr_id:
1132                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1133                 # self.logger.debug(logging_text + step)
1134
1135                 # check if VIM is creating and wait  look if previous tasks in process
1136                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1137                 if task_dependency:
1138                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1139                     self.logger.debug(logging_text + step)
1140                     await asyncio.wait(task_dependency, timeout=3600)
1141                 if ns_params.get("vnf"):
1142                     for vnf in ns_params["vnf"]:
1143                         if "vimAccountId" in vnf:
1144                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1145                                                                                         vnf["vimAccountId"])
1146                         if task_dependency:
1147                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1148                             self.logger.debug(logging_text + step)
1149                             await asyncio.wait(task_dependency, timeout=3600)
1150
1151                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1152
1153                 # feature 1429. Add n2vc public key to needed VMs
1154                 n2vc_key = await self.n2vc.GetPublicKey()
1155                 n2vc_key_list.append(n2vc_key)
1156                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1157
1158                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1159                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1160                                             name=db_nsr["name"],
1161                                             scenario=RO_nsd_uuid)
1162                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1163                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1164                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1165                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1166             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1167
1168             # wait until NS is ready
1169             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1170             detailed_status_old = None
1171             self.logger.debug(logging_text + step)
1172
1173             while time() <= start_deploy + self.total_deploy_timeout:
1174                 desc = await self.RO.show("ns", RO_nsr_id)
1175                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1176                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1177                 if ns_status == "ERROR":
1178                     raise ROclient.ROClientException(ns_status_info)
1179                 elif ns_status == "BUILD":
1180                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1181                 elif ns_status == "ACTIVE":
1182                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1183                     try:
1184                         self.ns_update_vnfr(db_vnfrs, desc)
1185                         break
1186                     except LcmExceptionNoMgmtIP:
1187                         pass
1188                 else:
1189                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1190                 if detailed_status != detailed_status_old:
1191                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1192                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1193                 await asyncio.sleep(5, loop=self.loop)
1194             else:   # total_deploy_timeout
1195                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1196
1197             step = "Updating NSR"
1198             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1199
1200             db_nsr_update["operational-status"] = "running"
1201             db_nsr["detailed-status"] = "Configuring vnfr"
1202             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1203
1204             # Configure proxy charms once VMs are up
1205             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1206                 vnf_index = vca_deployed.get("member-vnf-index")
1207                 vdu_id = vca_deployed.get("vdu_id")
1208                 vdu_name = None
1209                 vdu_count_index = None
1210
1211                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1212                                                                                                            vdu_id)
1213                 add_params = {}
1214                 initial_config_primitive_list = []
1215                 if vnf_index:
1216                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1217                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1218                     vnfd = db_vnfds_index[vnf_index]
1219
1220                     if vdu_id:
1221                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1222                             if vdu["id"] == vdu_id:
1223                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1224                                     'initial-config-primitive', [])
1225                                 break
1226                         else:
1227                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1228                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1229                         # TODO for the moment only first vdu_id contains a charm deployed
1230                         if vdur["vdu-id-ref"] != vdu["id"]:
1231                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1232                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1233                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1234                     else:
1235                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1236                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1237                 else:
1238                     if db_nsr.get("additionalParamsForNs"):
1239                         add_params = db_nsr["additionalParamsForNs"].copy()
1240                     for k, v in add_params.items():
1241                         if isinstance(v, str) and v.startswith("!!yaml "):
1242                             add_params[k] = yaml.safe_load(v[7:])
1243                     add_params["rw_mgmt_ip"] = None
1244                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1245
1246                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1247                 initial_config_primitive_list = initial_config_primitive_list.copy()
1248                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1249                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1250
1251                 for initial_config_primitive in initial_config_primitive_list:
1252                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1253                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1254                                       .format(initial_config_primitive["name"], primitive_params_))
1255                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1256                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1257                         initial_config_primitive["name"],
1258                         primitive_params_,
1259                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1260                         retries_interval=30)
1261                     if primitive_result != "COMPLETED":
1262                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1263                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1264                                                    vca_deployed["vdu_id"], primitive_detail))
1265
1266             # Deploy native charms
1267             step = "Looking for needed vnfd to configure with native charm"
1268             self.logger.debug(logging_text + step)
1269
1270             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1271                 vnfd_id = c_vnf["vnfd-id-ref"]
1272                 vnf_index = str(c_vnf["member-vnf-index"])
1273                 vnfd = db_vnfds_ref[vnfd_id]
1274
1275                 # Get additional parameters
1276                 vnfr_params = {}
1277                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1278                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1279                 for k, v in vnfr_params.items():
1280                     if isinstance(v, str) and v.startswith("!!yaml "):
1281                         vnfr_params[k] = yaml.safe_load(v[7:])
1282
1283                 # Check if this VNF has a charm configuration
1284                 vnf_config = vnfd.get("vnf-configuration")
1285                 if vnf_config and vnf_config.get("juju"):
1286                     native_charm = vnf_config["juju"].get("proxy") is False
1287
1288                     if native_charm:
1289                         if not vca_model_name:
1290                             step = "creating VCA model name '{}'".format(nsr_id)
1291                             self.logger.debug(logging_text + step)
1292                             await self.n2vc.CreateNetworkService(nsr_id)
1293                             vca_model_name = nsr_id
1294                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1295                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1296                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1297                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1298                         charm_params = {
1299                             "user_values": vnfr_params,
1300                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1301                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1302                         }
1303
1304                         # get username
1305                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1306                         #  merged. Meanwhile let's get username from initial-config-primitive
1307                         if vnf_config.get("initial-config-primitive"):
1308                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1309                                 if param["name"] == "ssh-username":
1310                                     charm_params["username"] = param["value"]
1311                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1312                             if vnf_config["config-access"]["ssh-access"].get("required"):
1313                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1314
1315                         # Login to the VCA. If there are multiple calls to login(),
1316                         # subsequent calls will be a nop and return immediately.
1317                         await self.n2vc.login()
1318
1319                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1320                         number_to_configure += 1
1321
1322                 # Deploy charms for each VDU that supports one.
1323                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1324                     vdu_config = vdu.get('vdu-configuration')
1325                     native_charm = False
1326
1327                     if vdu_config and vdu_config.get("juju"):
1328                         native_charm = vdu_config["juju"].get("proxy") is False
1329
1330                         if native_charm:
1331                             if not vca_model_name:
1332                                 step = "creating VCA model name"
1333                                 await self.n2vc.CreateNetworkService(nsr_id)
1334                                 vca_model_name = nsr_id
1335                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1336                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1337                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1338                                                                                                      vdu["id"])
1339                             await self.n2vc.login()
1340                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1341                             # TODO for the moment only first vdu_id contains a charm deployed
1342                             if vdur["vdu-id-ref"] != vdu["id"]:
1343                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1344                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1345                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1346                             charm_params = {
1347                                 "user_values": vnfr_params,
1348                                 "rw_mgmt_ip": vdur["ip-address"],
1349                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1350                             }
1351
1352                             # get username
1353                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1354                             #  merged. Meanwhile let's get username from initial-config-primitive
1355                             if vdu_config.get("initial-config-primitive"):
1356                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1357                                     if param["name"] == "ssh-username":
1358                                         charm_params["username"] = param["value"]
1359                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1360                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1361                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1362                                         "default-user")
1363
1364                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1365                                          charm_params, n2vc_info, native_charm)
1366                             number_to_configure += 1
1367
1368             # Check if this NS has a charm configuration
1369
1370             ns_config = nsd.get("ns-configuration")
1371             if ns_config and ns_config.get("juju"):
1372                 native_charm = ns_config["juju"].get("proxy") is False
1373
1374                 if native_charm:
1375                     step = "deploying native charm to configure ns"
1376                     # TODO is NS magmt IP address needed?
1377
1378                     # Get additional parameters
1379                     additional_params = {}
1380                     if db_nsr.get("additionalParamsForNs"):
1381                         additional_params = db_nsr["additionalParamsForNs"].copy()
1382                     for k, v in additional_params.items():
1383                         if isinstance(v, str) and v.startswith("!!yaml "):
1384                             additional_params[k] = yaml.safe_load(v[7:])
1385
1386                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1387                     charm_params = {
1388                         "user_values": additional_params,
1389                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1390                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1391                     }
1392
1393                     # get username
1394                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1395                     #  merged. Meanwhile let's get username from initial-config-primitive
1396                     if ns_config.get("initial-config-primitive"):
1397                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1398                             if param["name"] == "ssh-username":
1399                                 charm_params["username"] = param["value"]
1400                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1401                         if ns_config["config-access"]["ssh-access"].get("required"):
1402                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1403
1404                     # Login to the VCA. If there are multiple calls to login(),
1405                     # subsequent calls will be a nop and return immediately.
1406                     await self.n2vc.login()
1407                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1408                     number_to_configure += 1
1409
1410             # waiting all charms are ok
1411             configuration_failed = False
1412             if number_to_configure:
1413                 step = "Waiting all charms are active"
1414                 old_status = "configuring: init: {}".format(number_to_configure)
1415                 db_nsr_update["config-status"] = old_status
1416                 db_nsr_update["detailed-status"] = old_status
1417                 db_nslcmop_update["detailed-status"] = old_status
1418
1419                 # wait until all are configured.
1420                 while time() <= start_deploy + self.total_deploy_timeout:
1421                     if db_nsr_update:
1422                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1423                     if db_nslcmop_update:
1424                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1425                     # TODO add a fake task that set n2vc_event after some time
1426                     await n2vc_info["n2vc_event"].wait()
1427                     n2vc_info["n2vc_event"].clear()
1428                     all_active = True
1429                     status_map = {}
1430                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1431                     now = time()
1432                     for vca_deployed in vca_deployed_list:
1433                         vca_status = vca_deployed["operational-status"]
1434                         if vca_status not in status_map:
1435                             # Initialize it
1436                             status_map[vca_status] = 0
1437                         status_map[vca_status] += 1
1438
1439                         if vca_status == "active":
1440                             vca_deployed.pop("time_first_error", None)
1441                             vca_deployed.pop("status_first_error", None)
1442                             continue
1443
1444                         all_active = False
1445                         if vca_status in ("error", "blocked"):
1446                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1447                             # if not first time in this status error
1448                             if not vca_deployed.get("time_first_error"):
1449                                 vca_deployed["time_first_error"] = now
1450                                 continue
1451                         if vca_deployed.get("time_first_error") and \
1452                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1453                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1454                                                    .format(vca_deployed["member-vnf-index"],
1455                                                            vca_deployed["vdu_id"], vca_status,
1456                                                            vca_deployed["detailed-status-error"]))
1457
1458                     if all_active:
1459                         break
1460                     elif n2vc_error_text:
1461                         db_nsr_update["config-status"] = "failed"
1462                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1463                         db_nsr_update["detailed-status"] = error_text
1464                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1465                         db_nslcmop_update["detailed-status"] = error_text
1466                         db_nslcmop_update["statusEnteredTime"] = time()
1467                         configuration_failed = True
1468                         break
1469                     else:
1470                         cs = "configuring: "
1471                         separator = ""
1472                         for status, num in status_map.items():
1473                             cs += separator + "{}: {}".format(status, num)
1474                             separator = ", "
1475                         if old_status != cs:
1476                             db_nsr_update["config-status"] = cs
1477                             db_nsr_update["detailed-status"] = cs
1478                             db_nslcmop_update["detailed-status"] = cs
1479                             old_status = cs
1480                 else:   # total_deploy_timeout
1481                     raise LcmException("Timeout waiting ns to be configured")
1482
1483             if not configuration_failed:
1484                 # all is done
1485                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1486                 db_nslcmop_update["statusEnteredTime"] = time()
1487                 db_nslcmop_update["detailed-status"] = "done"
1488                 db_nsr_update["config-status"] = "configured"
1489                 db_nsr_update["detailed-status"] = "done"
1490
1491             return
1492
1493         except (ROclient.ROClientException, DbException, LcmException) as e:
1494             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1495             exc = e
1496         except asyncio.CancelledError:
1497             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1498             exc = "Operation was cancelled"
1499         except Exception as e:
1500             exc = traceback.format_exc()
1501             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1502                                  exc_info=True)
1503         finally:
1504             if exc:
1505                 if db_nsr:
1506                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1507                     db_nsr_update["operational-status"] = "failed"
1508                 if db_nslcmop:
1509                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1510                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1511                     db_nslcmop_update["statusEnteredTime"] = time()
1512             try:
1513                 if db_nsr:
1514                     db_nsr_update["_admin.nslcmop"] = None
1515                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1516                 if db_nslcmop_update:
1517                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1518             except DbException as e:
1519                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1520             if nslcmop_operation_state:
1521                 try:
1522                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1523                                                                    "operationState": nslcmop_operation_state},
1524                                             loop=self.loop)
1525                 except Exception as e:
1526                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1527
1528             self.logger.debug(logging_text + "Exit")
1529             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1530
1531     async def _destroy_charm(self, model, application):
1532         """
1533         Order N2VC destroy a charm
1534         :param model:
1535         :param application:
1536         :return: True if charm does not exist. False if it exist
1537         """
1538         if not await self.n2vc.HasApplication(model, application):
1539             return True  # Already removed
1540         await self.n2vc.RemoveCharms(model, application)
1541         return False
1542
1543     async def _wait_charm_destroyed(self, model, application, timeout):
1544         """
1545         Wait until charm does not exist
1546         :param model:
1547         :param application:
1548         :param timeout:
1549         :return: True if not exist, False if timeout
1550         """
1551         while True:
1552             if not await self.n2vc.HasApplication(model, application):
1553                 return True
1554             if timeout < 0:
1555                 return False
1556             await asyncio.sleep(10)
1557             timeout -= 10
1558
1559     # Check if this VNFD has a configured terminate action
1560     def _has_terminate_config_primitive(self, vnfd):
1561         vnf_config = vnfd.get("vnf-configuration")
1562         if vnf_config and vnf_config.get("terminate-config-primitive"):
1563             return True
1564         else:
1565             return False
1566
1567     @staticmethod
1568     def _get_terminate_config_primitive_seq_list(vnfd):
1569         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1570         # No need to check for existing primitive twice, already done before
1571         vnf_config = vnfd.get("vnf-configuration")
1572         seq_list = vnf_config.get("terminate-config-primitive")
1573         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1574         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1575         return seq_list_sorted
1576
1577     @staticmethod
1578     def _create_nslcmop(nsr_id, operation, params):
1579         """
1580         Creates a ns-lcm-opp content to be stored at database.
1581         :param nsr_id: internal id of the instance
1582         :param operation: instantiate, terminate, scale, action, ...
1583         :param params: user parameters for the operation
1584         :return: dictionary following SOL005 format
1585         """
1586         # Raise exception if invalid arguments
1587         if not (nsr_id and operation and params):
1588             raise LcmException(
1589                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1590         now = time()
1591         _id = str(uuid4())
1592         nslcmop = {
1593             "id": _id,
1594             "_id": _id,
1595             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1596             "operationState": "PROCESSING",
1597             "statusEnteredTime": now,
1598             "nsInstanceId": nsr_id,
1599             "lcmOperationType": operation,
1600             "startTime": now,
1601             "isAutomaticInvocation": False,
1602             "operationParams": params,
1603             "isCancelPending": False,
1604             "links": {
1605                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1606                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1607             }
1608         }
1609         return nslcmop
1610
1611     def _get_terminate_primitive_params(self, seq, vnf_index):
1612         primitive = seq.get('name')
1613         primitive_params = {}
1614         params = {
1615             "member_vnf_index": vnf_index,
1616             "primitive": primitive,
1617             "primitive_params": primitive_params,
1618         }
1619         desc_params = {}
1620         return self._map_primitive_params(seq, params, desc_params)
1621
1622     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1623                           vdu_name, primitive, mapped_primitive_params):
1624         # Create the "_admin.operations" array, or append operation if array already exists
1625         key_admin = '_admin'
1626         key_operations = 'operations'
1627         db_nslcmop_update = {}
1628         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1629         op_list = db_nslcmop_admin.get(key_operations)
1630         new_op = {'member_vnf_index': vnf_index,
1631                   'vdu_id': vdu_id,
1632                   'vdu_count_index': vdu_count_index,
1633                   'primitive': primitive,
1634                   'primitive_params': mapped_primitive_params}
1635         if db_nslcmop_admin:
1636             if not op_list:
1637                 # First operation, create key 'operations' with current operation as first list element
1638                 db_nslcmop_admin.update({key_operations: [new_op]})
1639                 op_list = db_nslcmop_admin.get(key_operations)
1640             else:
1641                 # Not first operation, append operation to existing list
1642                 op_list.append(new_op)
1643
1644         db_nslcmop_update['_admin.operations'] = op_list
1645         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1646
1647     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1648         """ Create a primitive with params from VNFD
1649             Called from terminate() before deleting instance
1650             Calls action() to execute the primitive """
1651         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1652         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1653         db_vnfds = {}
1654         # Loop over VNFRs
1655         for vnfr in db_vnfrs_list:
1656             vnfd_id = vnfr["vnfd-id"]
1657             vnf_index = vnfr["member-vnf-index-ref"]
1658             if vnfd_id not in db_vnfds:
1659                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1660                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1661                 db_vnfds[vnfd_id] = vnfd
1662             vnfd = db_vnfds[vnfd_id]
1663             if not self._has_terminate_config_primitive(vnfd):
1664                 continue
1665             # Get the primitive's sorted sequence list
1666             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1667             for seq in seq_list:
1668                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1669                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1670                     vnf_index, seq.get("name"))
1671                 self.logger.debug(logging_text + step)
1672                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1673                 primitive = seq.get('name')
1674                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1675                 # The following 3 parameters are currently set to None for 'terminate':
1676                 # vdu_id, vdu_count_index, vdu_name
1677                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1678                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1679                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1680                 # Add suboperation
1681                 self._add_suboperation(db_nslcmop,
1682                                        nslcmop_id,
1683                                        vnf_index,
1684                                        vdu_id,
1685                                        vdu_count_index,
1686                                        vdu_name,
1687                                        primitive,
1688                                        mapped_primitive_params)
1689                 # Suboperations: Call _ns_execute_primitive() instead of action()
1690                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1691                 nsr_deployed = db_nsr["_admin"]["deployed"]
1692                 result, result_detail = await self._ns_execute_primitive(
1693                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1694                     mapped_primitive_params)
1695
1696                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1697                 #    nsr_id, nslcmop_terminate_action_id)
1698                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1699                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1700                 if result not in result_ok:
1701                     raise LcmException(
1702                         "terminate_primitive_action for vnf_member_index={}",
1703                         " primitive={} fails with error {}".format(
1704                             vnf_index, seq.get("name"), result_detail))
1705
1706     async def terminate(self, nsr_id, nslcmop_id):
1707
1708         # Try to lock HA task here
1709         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1710         if not task_is_locked_by_me:
1711             return
1712
1713         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1714         self.logger.debug(logging_text + "Enter")
1715         db_nsr = None
1716         db_nslcmop = None
1717         exc = None
1718         failed_detail = []   # annotates all failed error messages
1719         vca_time_destroy = None   # time of where destroy charm order
1720         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1721         db_nslcmop_update = {}
1722         nslcmop_operation_state = None
1723         autoremove = False  # autoremove after terminated
1724         try:
1725             # wait for any previous tasks in process
1726             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1727
1728             step = "Getting nslcmop={} from db".format(nslcmop_id)
1729             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1730             step = "Getting nsr={} from db".format(nsr_id)
1731             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1732             # nsd = db_nsr["nsd"]
1733             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1734             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1735                 return
1736             # #TODO check if VIM is creating and wait
1737             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1738             # Call internal terminate action
1739             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1740
1741             db_nsr_update["operational-status"] = "terminating"
1742             db_nsr_update["config-status"] = "terminating"
1743
1744             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1745                 vca_model_name = nsr_deployed["VCA-model-name"]
1746                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1747                 self.logger.debug(logging_text + step)
1748                 try:
1749                     await self.n2vc.DestroyNetworkService(vca_model_name)
1750                 except NetworkServiceDoesNotExist:
1751                     pass
1752                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1753                 if nsr_deployed.get("VCA"):
1754                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1755                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1756                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1757             # for backward compatibility if charm have been created with "default"  model name delete one by one
1758             elif nsr_deployed and nsr_deployed.get("VCA"):
1759                 try:
1760                     step = "Scheduling configuration charms removing"
1761                     db_nsr_update["detailed-status"] = "Deleting charms"
1762                     self.logger.debug(logging_text + step)
1763                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1764                     # for backward compatibility
1765                     if isinstance(nsr_deployed["VCA"], dict):
1766                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1767                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1768                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1769
1770                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1771                         if vca_deployed:
1772                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1773                                 vca_deployed.clear()
1774                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1775                             else:
1776                                 vca_time_destroy = time()
1777                 except Exception as e:
1778                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1779
1780             # remove from RO
1781             RO_fail = False
1782
1783             # Delete ns
1784             RO_nsr_id = RO_delete_action = None
1785             if nsr_deployed and nsr_deployed.get("RO"):
1786                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1787                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1788             try:
1789                 if RO_nsr_id:
1790                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1791                         "Deleting ns from VIM"
1792                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1793                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1794                     self.logger.debug(logging_text + step)
1795                     desc = await self.RO.delete("ns", RO_nsr_id)
1796                     RO_delete_action = desc["action_id"]
1797                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1798                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1799                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1800                 if RO_delete_action:
1801                     # wait until NS is deleted from VIM
1802                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1803                         format(RO_nsr_id, RO_delete_action)
1804                     detailed_status_old = None
1805                     self.logger.debug(logging_text + step)
1806
1807                     delete_timeout = 20 * 60   # 20 minutes
1808                     while delete_timeout > 0:
1809                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1810                                                   extra_item_id=RO_delete_action)
1811                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1812                         if ns_status == "ERROR":
1813                             raise ROclient.ROClientException(ns_status_info)
1814                         elif ns_status == "BUILD":
1815                             detailed_status = step + "; {}".format(ns_status_info)
1816                         elif ns_status == "ACTIVE":
1817                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1818                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1819                             break
1820                         else:
1821                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1822                         if detailed_status != detailed_status_old:
1823                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1824                                 db_nsr_update["detailed-status"] = detailed_status
1825                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1826                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1827                         await asyncio.sleep(5, loop=self.loop)
1828                         delete_timeout -= 5
1829                     else:  # delete_timeout <= 0:
1830                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1831
1832             except ROclient.ROClientException as e:
1833                 if e.http_code == 404:  # not found
1834                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1835                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1836                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1837                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1838                 elif e.http_code == 409:   # conflict
1839                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1840                     self.logger.debug(logging_text + failed_detail[-1])
1841                     RO_fail = True
1842                 else:
1843                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1844                     self.logger.error(logging_text + failed_detail[-1])
1845                     RO_fail = True
1846
1847             # Delete nsd
1848             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1849                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1850                 try:
1851                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1852                         "Deleting nsd from RO"
1853                     await self.RO.delete("nsd", RO_nsd_id)
1854                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1855                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1856                 except ROclient.ROClientException as e:
1857                     if e.http_code == 404:  # not found
1858                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1859                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1860                     elif e.http_code == 409:   # conflict
1861                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1862                         self.logger.debug(logging_text + failed_detail[-1])
1863                         RO_fail = True
1864                     else:
1865                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1866                         self.logger.error(logging_text + failed_detail[-1])
1867                         RO_fail = True
1868
1869             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1870                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1871                     if not vnf_deployed or not vnf_deployed["id"]:
1872                         continue
1873                     try:
1874                         RO_vnfd_id = vnf_deployed["id"]
1875                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1876                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1877                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1878                         await self.RO.delete("vnfd", RO_vnfd_id)
1879                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1880                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1881                     except ROclient.ROClientException as e:
1882                         if e.http_code == 404:  # not found
1883                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1884                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1885                         elif e.http_code == 409:   # conflict
1886                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1887                             self.logger.debug(logging_text + failed_detail[-1])
1888                         else:
1889                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1890                             self.logger.error(logging_text + failed_detail[-1])
1891
1892             # wait until charm deleted
1893             if vca_time_destroy:
1894                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1895                     "Waiting for deletion of configuration charms"
1896                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1897                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1898                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1899                     if not vca_deployed:
1900                         continue
1901                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1902                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1903                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1904                                                             timeout):
1905                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1906                             vca_deployed["application"]))
1907                     else:
1908                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1909
1910             if failed_detail:
1911                 self.logger.error(logging_text + " ;".join(failed_detail))
1912                 db_nsr_update["operational-status"] = "failed"
1913                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1914                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1915                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1916                 db_nslcmop_update["statusEnteredTime"] = time()
1917             else:
1918                 db_nsr_update["operational-status"] = "terminated"
1919                 db_nsr_update["detailed-status"] = "Done"
1920                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1921                 db_nslcmop_update["detailed-status"] = "Done"
1922                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1923                 db_nslcmop_update["statusEnteredTime"] = time()
1924                 if db_nslcmop["operationParams"].get("autoremove"):
1925                     autoremove = True
1926
1927         except (ROclient.ROClientException, DbException, LcmException) as e:
1928             self.logger.error(logging_text + "Exit Exception {}".format(e))
1929             exc = e
1930         except asyncio.CancelledError:
1931             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1932             exc = "Operation was cancelled"
1933         except Exception as e:
1934             exc = traceback.format_exc()
1935             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1936         finally:
1937             if exc and db_nslcmop:
1938                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1939                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1940                 db_nslcmop_update["statusEnteredTime"] = time()
1941             try:
1942                 if db_nslcmop and db_nslcmop_update:
1943                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1944                 if db_nsr:
1945                     db_nsr_update["_admin.nslcmop"] = None
1946                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1947             except DbException as e:
1948                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1949             if nslcmop_operation_state:
1950                 try:
1951                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1952                                                                  "operationState": nslcmop_operation_state,
1953                                                                  "autoremove": autoremove},
1954                                             loop=self.loop)
1955                 except Exception as e:
1956                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1957             self.logger.debug(logging_text + "Exit")
1958             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1959
1960     @staticmethod
1961     def _map_primitive_params(primitive_desc, params, instantiation_params):
1962         """
1963         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1964         The default-value is used. If it is between < > it look for a value at instantiation_params
1965         :param primitive_desc: portion of VNFD/NSD that describes primitive
1966         :param params: Params provided by user
1967         :param instantiation_params: Instantiation params provided by user
1968         :return: a dictionary with the calculated params
1969         """
1970         calculated_params = {}
1971         for parameter in primitive_desc.get("parameter", ()):
1972             param_name = parameter["name"]
1973             if param_name in params:
1974                 calculated_params[param_name] = params[param_name]
1975             elif "default-value" in parameter or "value" in parameter:
1976                 if "value" in parameter:
1977                     calculated_params[param_name] = parameter["value"]
1978                 else:
1979                     calculated_params[param_name] = parameter["default-value"]
1980                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1981                         and calculated_params[param_name].endswith(">"):
1982                     if calculated_params[param_name][1:-1] in instantiation_params:
1983                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1984                     else:
1985                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1986                                            format(parameter["default-value"], primitive_desc["name"]))
1987             else:
1988                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1989                                    format(param_name, primitive_desc["name"]))
1990
1991             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1992                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1993                                                                width=256)
1994             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1995                 calculated_params[param_name] = calculated_params[param_name][7:]
1996         return calculated_params
1997
1998     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1999                                     primitive, primitive_params, retries=0, retries_interval=30):
2000         start_primitive_time = time()
2001         try:
2002             for vca_deployed in db_deployed["VCA"]:
2003                 if not vca_deployed:
2004                     continue
2005                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2006                     continue
2007                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2008                     continue
2009                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2010                     continue
2011                 break
2012             else:
2013                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2014                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2015             model_name = vca_deployed.get("model")
2016             application_name = vca_deployed.get("application")
2017             if not model_name or not application_name:
2018                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2019                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2020                                                                        vdu_count_index))
2021             # if vca_deployed["operational-status"] != "active":
2022             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2023             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2024             callback = None  # self.n2vc_callback
2025             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2026             await self.n2vc.login()
2027             if primitive == "config":
2028                 primitive_params = {"params": primitive_params}
2029             while retries >= 0:
2030                 primitive_id = await self.n2vc.ExecutePrimitive(
2031                     model_name,
2032                     application_name,
2033                     primitive,
2034                     callback,
2035                     *callback_args,
2036                     **primitive_params
2037                 )
2038                 while time() - start_primitive_time < self.timeout_primitive:
2039                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2040                     if primitive_result_ in ("completed", "failed"):
2041                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2042                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2043                         break
2044                     elif primitive_result_ is None and primitive == "config":
2045                         primitive_result = "COMPLETED"
2046                         detailed_result = None
2047                         break
2048                     else:  # ("running", "pending", None):
2049                         pass
2050                     await asyncio.sleep(5)
2051                 else:
2052                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2053                 if primitive_result == "COMPLETED":
2054                     break
2055                 retries -= 1
2056                 if retries >= 0:
2057                     await asyncio.sleep(retries_interval)
2058
2059             return primitive_result, detailed_result
2060         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2061             return "FAILED", str(e)
2062
2063     async def action(self, nsr_id, nslcmop_id):
2064
2065         # Try to lock HA task here
2066         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2067         if not task_is_locked_by_me:
2068             return
2069
2070         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2071         self.logger.debug(logging_text + "Enter")
2072         # get all needed from database
2073         db_nsr = None
2074         db_nslcmop = None
2075         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2076         db_nslcmop_update = {}
2077         nslcmop_operation_state = None
2078         nslcmop_operation_state_detail = None
2079         exc = None
2080         try:
2081             # wait for any previous tasks in process
2082             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2083
2084             step = "Getting information from database"
2085             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2086             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2087
2088             nsr_deployed = db_nsr["_admin"].get("deployed")
2089             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2090             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2091             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2092             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2093
2094             if vnf_index:
2095                 step = "Getting vnfr from database"
2096                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2097                 step = "Getting vnfd from database"
2098                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2099             else:
2100                 if db_nsr.get("nsd"):
2101                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2102                 else:
2103                     step = "Getting nsd from database"
2104                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2105
2106             # for backward compatibility
2107             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2108                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2109                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2110                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2111
2112             primitive = db_nslcmop["operationParams"]["primitive"]
2113             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2114
2115             # look for primitive
2116             config_primitive_desc = None
2117             if vdu_id:
2118                 for vdu in get_iterable(db_vnfd, "vdu"):
2119                     if vdu_id == vdu["id"]:
2120                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2121                             if config_primitive["name"] == primitive:
2122                                 config_primitive_desc = config_primitive
2123                                 break
2124             elif vnf_index:
2125                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2126                     if config_primitive["name"] == primitive:
2127                         config_primitive_desc = config_primitive
2128                         break
2129             else:
2130                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2131                     if config_primitive["name"] == primitive:
2132                         config_primitive_desc = config_primitive
2133                         break
2134
2135             if not config_primitive_desc:
2136                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2137                                    format(primitive))
2138
2139             desc_params = {}
2140             if vnf_index:
2141                 if db_vnfr.get("additionalParamsForVnf"):
2142                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2143             else:
2144                 if db_nsr.get("additionalParamsForVnf"):
2145                     desc_params.update(db_nsr["additionalParamsForNs"])
2146
2147             # TODO check if ns is in a proper status
2148             result, result_detail = await self._ns_execute_primitive(
2149                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2150                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2151             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2152             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2153             db_nslcmop_update["statusEnteredTime"] = time()
2154             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2155             return  # database update is called inside finally
2156
2157         except (DbException, LcmException) as e:
2158             self.logger.error(logging_text + "Exit Exception {}".format(e))
2159             exc = e
2160         except asyncio.CancelledError:
2161             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2162             exc = "Operation was cancelled"
2163         except Exception as e:
2164             exc = traceback.format_exc()
2165             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2166         finally:
2167             if exc and db_nslcmop:
2168                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2169                     "FAILED {}: {}".format(step, exc)
2170                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2171                 db_nslcmop_update["statusEnteredTime"] = time()
2172             try:
2173                 if db_nslcmop_update:
2174                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2175                 if db_nsr:
2176                     db_nsr_update["_admin.nslcmop"] = None
2177                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2178             except DbException as e:
2179                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2180             self.logger.debug(logging_text + "Exit")
2181             if nslcmop_operation_state:
2182                 try:
2183                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2184                                                                "operationState": nslcmop_operation_state},
2185                                             loop=self.loop)
2186                 except Exception as e:
2187                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2188             self.logger.debug(logging_text + "Exit")
2189             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2190             return nslcmop_operation_state, nslcmop_operation_state_detail
2191
2192     async def scale(self, nsr_id, nslcmop_id):
2193
2194         # Try to lock HA task here
2195         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2196         if not task_is_locked_by_me:
2197             return
2198
2199         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2200         self.logger.debug(logging_text + "Enter")
2201         # get all needed from database
2202         db_nsr = None
2203         db_nslcmop = None
2204         db_nslcmop_update = {}
2205         nslcmop_operation_state = None
2206         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2207         exc = None
2208         # in case of error, indicates what part of scale was failed to put nsr at error status
2209         scale_process = None
2210         old_operational_status = ""
2211         old_config_status = ""
2212         vnfr_scaled = False
2213         try:
2214             # wait for any previous tasks in process
2215             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2216
2217             step = "Getting nslcmop from database"
2218             self.logger.debug(step + " after having waited for previous tasks to be completed")
2219             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2220             step = "Getting nsr from database"
2221             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2222
2223             old_operational_status = db_nsr["operational-status"]
2224             old_config_status = db_nsr["config-status"]
2225             step = "Parsing scaling parameters"
2226             # self.logger.debug(step)
2227             db_nsr_update["operational-status"] = "scaling"
2228             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2229             nsr_deployed = db_nsr["_admin"].get("deployed")
2230             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2231             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2232             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2233             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2234             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2235
2236             # for backward compatibility
2237             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2238                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2239                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2240                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2241
2242             step = "Getting vnfr from database"
2243             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2244             step = "Getting vnfd from database"
2245             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2246
2247             step = "Getting scaling-group-descriptor"
2248             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2249                 if scaling_descriptor["name"] == scaling_group:
2250                     break
2251             else:
2252                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2253                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2254
2255             # cooldown_time = 0
2256             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2257             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2258             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2259             #         break
2260
2261             # TODO check if ns is in a proper status
2262             step = "Sending scale order to VIM"
2263             nb_scale_op = 0
2264             if not db_nsr["_admin"].get("scaling-group"):
2265                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2266                 admin_scale_index = 0
2267             else:
2268                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2269                     if admin_scale_info["name"] == scaling_group:
2270                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2271                         break
2272                 else:  # not found, set index one plus last element and add new entry with the name
2273                     admin_scale_index += 1
2274                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2275             RO_scaling_info = []
2276             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2277             if scaling_type == "SCALE_OUT":
2278                 # count if max-instance-count is reached
2279                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2280                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2281                 if nb_scale_op >= max_instance_count:
2282                     raise LcmException("reached the limit of {} (max-instance-count) "
2283                                        "scaling-out operations for the "
2284                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2285
2286                 nb_scale_op += 1
2287                 vdu_scaling_info["scaling_direction"] = "OUT"
2288                 vdu_scaling_info["vdu-create"] = {}
2289                 for vdu_scale_info in scaling_descriptor["vdu"]:
2290                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2291                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2292                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2293
2294             elif scaling_type == "SCALE_IN":
2295                 # count if min-instance-count is reached
2296                 min_instance_count = 0
2297                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2298                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2299                 if nb_scale_op <= min_instance_count:
2300                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2301                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2302                 nb_scale_op -= 1
2303                 vdu_scaling_info["scaling_direction"] = "IN"
2304                 vdu_scaling_info["vdu-delete"] = {}
2305                 for vdu_scale_info in scaling_descriptor["vdu"]:
2306                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2307                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2308                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2309
2310             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2311             vdu_create = vdu_scaling_info.get("vdu-create")
2312             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2313             if vdu_scaling_info["scaling_direction"] == "IN":
2314                 for vdur in reversed(db_vnfr["vdur"]):
2315                     if vdu_delete.get(vdur["vdu-id-ref"]):
2316                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2317                         vdu_scaling_info["vdu"].append({
2318                             "name": vdur["name"],
2319                             "vdu_id": vdur["vdu-id-ref"],
2320                             "interface": []
2321                         })
2322                         for interface in vdur["interfaces"]:
2323                             vdu_scaling_info["vdu"][-1]["interface"].append({
2324                                 "name": interface["name"],
2325                                 "ip_address": interface["ip-address"],
2326                                 "mac_address": interface.get("mac-address"),
2327                             })
2328                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2329
2330             # execute primitive service PRE-SCALING
2331             step = "Executing pre-scale vnf-config-primitive"
2332             if scaling_descriptor.get("scaling-config-action"):
2333                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2334                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2335                             and scaling_type == "SCALE_IN":
2336                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2337                         step = db_nslcmop_update["detailed-status"] = \
2338                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2339
2340                         # look for primitive
2341                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2342                             if config_primitive["name"] == vnf_config_primitive:
2343                                 break
2344                         else:
2345                             raise LcmException(
2346                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2347                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2348                                 "primitive".format(scaling_group, config_primitive))
2349
2350                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2351                         if db_vnfr.get("additionalParamsForVnf"):
2352                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2353
2354                         scale_process = "VCA"
2355                         db_nsr_update["config-status"] = "configuring pre-scaling"
2356                         result, result_detail = await self._ns_execute_primitive(
2357                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2358                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2359                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2360                             vnf_config_primitive, result, result_detail))
2361                         if result == "FAILED":
2362                             raise LcmException(result_detail)
2363                         db_nsr_update["config-status"] = old_config_status
2364                         scale_process = None
2365
2366             if RO_scaling_info:
2367                 scale_process = "RO"
2368                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2369                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2370                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2371                 # wait until ready
2372                 RO_nslcmop_id = RO_desc["instance_action_id"]
2373                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2374
2375                 RO_task_done = False
2376                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2377                 detailed_status_old = None
2378                 self.logger.debug(logging_text + step)
2379
2380                 deployment_timeout = 1 * 3600   # One hour
2381                 while deployment_timeout > 0:
2382                     if not RO_task_done:
2383                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2384                                                   extra_item_id=RO_nslcmop_id)
2385                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2386                         if ns_status == "ERROR":
2387                             raise ROclient.ROClientException(ns_status_info)
2388                         elif ns_status == "BUILD":
2389                             detailed_status = step + "; {}".format(ns_status_info)
2390                         elif ns_status == "ACTIVE":
2391                             RO_task_done = True
2392                             step = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
2393                             self.logger.debug(logging_text + step)
2394                         else:
2395                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2396                     else:
2397                         desc = await self.RO.show("ns", RO_nsr_id)
2398                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2399                         if ns_status == "ERROR":
2400                             raise ROclient.ROClientException(ns_status_info)
2401                         elif ns_status == "BUILD":
2402                             detailed_status = step + "; {}".format(ns_status_info)
2403                         elif ns_status == "ACTIVE":
2404                             step = detailed_status = \
2405                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2406                             if not vnfr_scaled:
2407                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2408                                 vnfr_scaled = True
2409                             try:
2410                                 desc = await self.RO.show("ns", RO_nsr_id)
2411                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2412                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2413                                 break
2414                             except LcmExceptionNoMgmtIP:
2415                                 pass
2416                         else:
2417                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2418                     if detailed_status != detailed_status_old:
2419                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2420                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2421
2422                     await asyncio.sleep(5, loop=self.loop)
2423                     deployment_timeout -= 5
2424                 if deployment_timeout <= 0:
2425                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2426
2427                 # update VDU_SCALING_INFO with the obtained ip_addresses
2428                 if vdu_scaling_info["scaling_direction"] == "OUT":
2429                     for vdur in reversed(db_vnfr["vdur"]):
2430                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2431                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2432                             vdu_scaling_info["vdu"].append({
2433                                 "name": vdur["name"],
2434                                 "vdu_id": vdur["vdu-id-ref"],
2435                                 "interface": []
2436                             })
2437                             for interface in vdur["interfaces"]:
2438                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2439                                     "name": interface["name"],
2440                                     "ip_address": interface["ip-address"],
2441                                     "mac_address": interface.get("mac-address"),
2442                                 })
2443                     del vdu_scaling_info["vdu-create"]
2444
2445             scale_process = None
2446             if db_nsr_update:
2447                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2448
2449             # execute primitive service POST-SCALING
2450             step = "Executing post-scale vnf-config-primitive"
2451             if scaling_descriptor.get("scaling-config-action"):
2452                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2453                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2454                             and scaling_type == "SCALE_OUT":
2455                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2456                         step = db_nslcmop_update["detailed-status"] = \
2457                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2458
2459                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2460                         if db_vnfr.get("additionalParamsForVnf"):
2461                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2462
2463                         # look for primitive
2464                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2465                             if config_primitive["name"] == vnf_config_primitive:
2466                                 break
2467                         else:
2468                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2469                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2470                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2471                                                                                                      config_primitive))
2472                         scale_process = "VCA"
2473                         db_nsr_update["config-status"] = "configuring post-scaling"
2474
2475                         result, result_detail = await self._ns_execute_primitive(
2476                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2477                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2478                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2479                             vnf_config_primitive, result, result_detail))
2480                         if result == "FAILED":
2481                             raise LcmException(result_detail)
2482                         db_nsr_update["config-status"] = old_config_status
2483                         scale_process = None
2484
2485             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2486             db_nslcmop_update["statusEnteredTime"] = time()
2487             db_nslcmop_update["detailed-status"] = "done"
2488             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2489             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2490                 else old_operational_status
2491             db_nsr_update["config-status"] = old_config_status
2492             return
2493         except (ROclient.ROClientException, DbException, LcmException) as e:
2494             self.logger.error(logging_text + "Exit Exception {}".format(e))
2495             exc = e
2496         except asyncio.CancelledError:
2497             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2498             exc = "Operation was cancelled"
2499         except Exception as e:
2500             exc = traceback.format_exc()
2501             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2502         finally:
2503             if exc:
2504                 if db_nslcmop:
2505                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2506                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2507                     db_nslcmop_update["statusEnteredTime"] = time()
2508                 if db_nsr:
2509                     db_nsr_update["operational-status"] = old_operational_status
2510                     db_nsr_update["config-status"] = old_config_status
2511                     db_nsr_update["detailed-status"] = ""
2512                     db_nsr_update["_admin.nslcmop"] = None
2513                     if scale_process:
2514                         if "VCA" in scale_process:
2515                             db_nsr_update["config-status"] = "failed"
2516                         if "RO" in scale_process:
2517                             db_nsr_update["operational-status"] = "failed"
2518                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2519                                                                                                      exc)
2520             try:
2521                 if db_nslcmop and db_nslcmop_update:
2522                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2523                 if db_nsr:
2524                     db_nsr_update["_admin.nslcmop"] = None
2525                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2526             except DbException as e:
2527                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2528             if nslcmop_operation_state:
2529                 try:
2530                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2531                                                              "operationState": nslcmop_operation_state},
2532                                             loop=self.loop)
2533                     # if cooldown_time:
2534                     #     await asyncio.sleep(cooldown_time)
2535                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2536                 except Exception as e:
2537                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2538             self.logger.debug(logging_text + "Exit")
2539             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")