osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Update target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104             api_proxy=vca_config.get('apiproxy')
 105         )
 106         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 107
 108     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 109         """
 110         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 111         :param vnfd: input vnfd
 112         :param new_id: overrides vnf id if provided
 113         :param additionalParams: Instantiation params for VNFs provided
 114         :param nsrId: Id of the NSR
 115         :return: copy of vnfd
 116         """
 117         try:
 118             vnfd_RO = deepcopy(vnfd)
 119             # remove unused by RO configuration, monitoring, scaling and internal keys
 120             vnfd_RO.pop("_id", None)
 121             vnfd_RO.pop("_admin", None)
 122             vnfd_RO.pop("vnf-configuration", None)
 123             vnfd_RO.pop("monitoring-param", None)
 124             vnfd_RO.pop("scaling-group-descriptor", None)
 125             if new_id:
 126                 vnfd_RO["id"] = new_id
 127
 128             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 129             for vdu in get_iterable(vnfd_RO, "vdu"):
 130                 cloud_init_file = None
 131                 if vdu.get("cloud-init-file"):
 132                     base_folder = vnfd["_admin"]["storage"]
 133                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 134                                                                    vdu["cloud-init-file"])
 135                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 136                         cloud_init_content = ci_file.read()
 137                     vdu.pop("cloud-init-file", None)
 138                 elif vdu.get("cloud-init"):
 139                     cloud_init_content = vdu["cloud-init"]
 140                 else:
 141                     continue
 142
 143                 env = Environment()
 144                 ast = env.parse(cloud_init_content)
 145                 mandatory_vars = meta.find_undeclared_variables(ast)
 146                 if mandatory_vars:
 147                     for var in mandatory_vars:
 148                         if not additionalParams or var not in additionalParams.keys():
 149                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 150                                                "file, must be provided in the instantiation parameters inside the "
 151                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 152                 template = Template(cloud_init_content)
 153                 cloud_init_content = template.render(additionalParams or {})
 154                 vdu["cloud-init"] = cloud_init_content
 155
 156             return vnfd_RO
 157         except FsException as e:
 158             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 159                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 160         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 161             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 162                                format(vnfd["id"], vdu["id"], e))
 163
 164     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 165         """
 166         Callback both for charm status change and task completion
 167         :param model_name: Charm model name
 168         :param application_name: Charm application name
 169         :param status: Can be
 170             - blocked: The unit needs manual intervention
 171             - maintenance: The unit is actively deploying/configuring
 172             - waiting: The unit is waiting for another charm to be ready
 173             - active: The unit is deployed, configured, and ready
 174             - error: The charm has failed and needs attention.
 175             - terminated: The charm has been destroyed
 176             - removing,
 177             - removed
 178         :param message: detailed message error
 179         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 180             nsr_id:
 181             nslcmop_id:
 182             lcmOperationType: currently "instantiate"
 183             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 184             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 185             n2vc_event: event used to notify instantiation task that some change has been produced
 186         :param task: None for charm status change, or task for completion task callback
 187         :return:
 188         """
 189         try:
 190             nsr_id = n2vc_info["nsr_id"]
 191             deployed = n2vc_info["deployed"]
 192             db_nsr_update = n2vc_info["db_update"]
 193             nslcmop_id = n2vc_info["nslcmop_id"]
 194             ns_operation = n2vc_info["lcmOperationType"]
 195             n2vc_event = n2vc_info["n2vc_event"]
 196             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 197                                                                                     application_name)
 198             for vca_index, vca_deployed in enumerate(deployed):
 199                 if not vca_deployed:
 200                     continue
 201                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 202                     break
 203             else:
 204                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 205                                   format(model_name))
 206                 return
 207             if task:
 208                 if task.cancelled():
 209                     self.logger.debug(logging_text + " task Cancelled")
 210                     vca_deployed['operational-status'] = "error"
 211                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 212                     vca_deployed['detailed-status'] = "Task Cancelled"
 213                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 214
 215                 elif task.done():
 216                     exc = task.exception()
 217                     if exc:
 218                         self.logger.error(logging_text + " task Exception={}".format(exc))
 219                         vca_deployed['operational-status'] = "error"
 220                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 221                         vca_deployed['detailed-status'] = str(exc)
 222                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 223                     else:
 224                         self.logger.debug(logging_text + " task Done")
 225                         # task is Done, but callback is still ongoing. So ignore
 226                         return
 227             elif status:
 228                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 229                 if vca_deployed['operational-status'] == status:
 230                     return  # same status, ignore
 231                 vca_deployed['operational-status'] = status
 232                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 233                 vca_deployed['detailed-status'] = str(message)
 234                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 235             else:
 236                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 237                 return
 238             # wake up instantiate task
 239             n2vc_event.set()
 240         except Exception as e:
 241             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 242
 243     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 244         """
 245         Creates a RO ns descriptor from OSM ns_instantiate params
 246         :param ns_params: OSM instantiate params
 247         :return: The RO ns descriptor
 248         """
 249         vim_2_RO = {}
 250         wim_2_RO = {}
 251         # TODO feature 1417: Check that no instantiation is set over PDU
 252         # check if PDU forces a concrete vim-network-id and add it
 253         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 254
 255         def vim_account_2_RO(vim_account):
 256             if vim_account in vim_2_RO:
 257                 return vim_2_RO[vim_account]
 258
 259             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 260             if db_vim["_admin"]["operationalState"] != "ENABLED":
 261                 raise LcmException("VIM={} is not available. operationalState={}".format(
 262                     vim_account, db_vim["_admin"]["operationalState"]))
 263             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 264             vim_2_RO[vim_account] = RO_vim_id
 265             return RO_vim_id
 266
 267         def wim_account_2_RO(wim_account):
 268             if isinstance(wim_account, str):
 269                 if wim_account in wim_2_RO:
 270                     return wim_2_RO[wim_account]
 271
 272                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 273                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 274                     raise LcmException("WIM={} is not available. operationalState={}".format(
 275                         wim_account, db_wim["_admin"]["operationalState"]))
 276                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 277                 wim_2_RO[wim_account] = RO_wim_id
 278                 return RO_wim_id
 279             else:
 280                 return wim_account
 281
 282         def ip_profile_2_RO(ip_profile):
 283             RO_ip_profile = deepcopy((ip_profile))
 284             if "dns-server" in RO_ip_profile:
 285                 if isinstance(RO_ip_profile["dns-server"], list):
 286                     RO_ip_profile["dns-address"] = []
 287                     for ds in RO_ip_profile.pop("dns-server"):
 288                         RO_ip_profile["dns-address"].append(ds['address'])
 289                 else:
 290                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 291             if RO_ip_profile.get("ip-version") == "ipv4":
 292                 RO_ip_profile["ip-version"] = "IPv4"
 293             if RO_ip_profile.get("ip-version") == "ipv6":
 294                 RO_ip_profile["ip-version"] = "IPv6"
 295             if "dhcp-params" in RO_ip_profile:
 296                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 297             return RO_ip_profile
 298
 299         if not ns_params:
 300             return None
 301         RO_ns_params = {
 302             # "name": ns_params["nsName"],
 303             # "description": ns_params.get("nsDescription"),
 304             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 305             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 306             # "scenario": ns_params["nsdId"],
 307         }
 308         if n2vc_key_list:
 309             for vnfd_ref, vnfd in vnfd_dict.items():
 310                 vdu_needed_access = []
 311                 mgmt_cp = None
 312                 if vnfd.get("vnf-configuration"):
 313                     if vnfd.get("mgmt-interface"):
 314                         if vnfd["mgmt-interface"].get("vdu-id"):
 315                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 316                         elif vnfd["mgmt-interface"].get("cp"):
 317                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 318
 319                 for vdu in vnfd.get("vdu", ()):
 320                     if vdu.get("vdu-configuration"):
 321                         vdu_needed_access.append(vdu["id"])
 322                     elif mgmt_cp:
 323                         for vdu_interface in vdu.get("interface"):
 324                             if vdu_interface.get("external-connection-point-ref") and \
 325                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 326                                 vdu_needed_access.append(vdu["id"])
 327                                 mgmt_cp = None
 328                                 break
 329
 330                 if vdu_needed_access:
 331                     for vnf_member in nsd.get("constituent-vnfd"):
 332                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 333                             continue
 334                         for vdu in vdu_needed_access:
 335                             populate_dict(RO_ns_params,
 336                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 337                                           n2vc_key_list)
 338
 339         if ns_params.get("vduImage"):
 340             RO_ns_params["vduImage"] = ns_params["vduImage"]
 341
 342         if ns_params.get("ssh_keys"):
 343             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 344         for vnf_params in get_iterable(ns_params, "vnf"):
 345             for constituent_vnfd in nsd["constituent-vnfd"]:
 346                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 347                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 348                     break
 349             else:
 350                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 351                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 352             if vnf_params.get("vimAccountId"):
 353                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 354                               vim_account_2_RO(vnf_params["vimAccountId"]))
 355
 356             for vdu_params in get_iterable(vnf_params, "vdu"):
 357                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 358                 if vdu_params.get("volume"):
 359                     for volume_params in vdu_params["volume"]:
 360                         if volume_params.get("vim-volume-id"):
 361                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 362                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 363                                           volume_params["vim-volume-id"])
 364                 if vdu_params.get("interface"):
 365                     for interface_params in vdu_params["interface"]:
 366                         if interface_params.get("ip-address"):
 367                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 368                                                          vdu_params["id"], "interfaces", interface_params["name"],
 369                                                          "ip_address"),
 370                                           interface_params["ip-address"])
 371                         if interface_params.get("mac-address"):
 372                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 373                                                          vdu_params["id"], "interfaces", interface_params["name"],
 374                                                          "mac_address"),
 375                                           interface_params["mac-address"])
 376                         if interface_params.get("floating-ip-required"):
 377                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 378                                                          vdu_params["id"], "interfaces", interface_params["name"],
 379                                                          "floating-ip"),
 380                                           interface_params["floating-ip-required"])
 381
 382             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 383                 if internal_vld_params.get("vim-network-name"):
 384                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 385                                                  internal_vld_params["name"], "vim-network-name"),
 386                                   internal_vld_params["vim-network-name"])
 387                 if internal_vld_params.get("vim-network-id"):
 388                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 389                                                  internal_vld_params["name"], "vim-network-id"),
 390                                   internal_vld_params["vim-network-id"])
 391                 if internal_vld_params.get("ip-profile"):
 392                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 393                                                  internal_vld_params["name"], "ip-profile"),
 394                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 395
 396                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 397                     # look for interface
 398                     iface_found = False
 399                     for vdu_descriptor in vnf_descriptor["vdu"]:
 400                         for vdu_interface in vdu_descriptor["interface"]:
 401                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 402                                 if icp_params.get("ip-address"):
 403                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 404                                                                  vdu_descriptor["id"], "interfaces",
 405                                                                  vdu_interface["name"], "ip_address"),
 406                                                   icp_params["ip-address"])
 407
 408                                 if icp_params.get("mac-address"):
 409                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 410                                                                  vdu_descriptor["id"], "interfaces",
 411                                                                  vdu_interface["name"], "mac_address"),
 412                                                   icp_params["mac-address"])
 413                                 iface_found = True
 414                                 break
 415                         if iface_found:
 416                             break
 417                     else:
 418                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 419                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 420                                            "connection-point".format(vnf_params["member-vnf-index"],
 421                                                                      icp_params["id-ref"]))
 422
 423         for vld_params in get_iterable(ns_params, "vld"):
 424             if "ip-profile" in vld_params:
 425                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 426                               ip_profile_2_RO(vld_params["ip-profile"]))
 427
 428             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 429                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 430                               wim_account_2_RO(vld_params["wimAccountId"])),
 431             if vld_params.get("vim-network-name"):
 432                 RO_vld_sites = []
 433                 if isinstance(vld_params["vim-network-name"], dict):
 434                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 435                         RO_vld_sites.append({
 436                             "netmap-use": vim_net,
 437                             "datacenter": vim_account_2_RO(vim_account)
 438                         })
 439                 else:  # isinstance str
 440                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 441                 if RO_vld_sites:
 442                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 443             if vld_params.get("vim-network-id"):
 444                 RO_vld_sites = []
 445                 if isinstance(vld_params["vim-network-id"], dict):
 446                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 447                         RO_vld_sites.append({
 448                             "netmap-use": vim_net,
 449                             "datacenter": vim_account_2_RO(vim_account)
 450                         })
 451                 else:  # isinstance str
 452                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 453                 if RO_vld_sites:
 454                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 455             if vld_params.get("ns-net"):
 456                 if isinstance(vld_params["ns-net"], dict):
 457                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 458                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 459                 if RO_vld_ns_net:
 460                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 461             if "vnfd-connection-point-ref" in vld_params:
 462                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 463                     # look for interface
 464                     for constituent_vnfd in nsd["constituent-vnfd"]:
 465                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 466                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 467                             break
 468                     else:
 469                         raise LcmException(
 470                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 471                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 472                     match_cp = False
 473                     for vdu_descriptor in vnf_descriptor["vdu"]:
 474                         for interface_descriptor in vdu_descriptor["interface"]:
 475                             if interface_descriptor.get("external-connection-point-ref") == \
 476                                     cp_params["vnfd-connection-point-ref"]:
 477                                 match_cp = True
 478                                 break
 479                         if match_cp:
 480                             break
 481                     else:
 482                         raise LcmException(
 483                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 484                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 485                                 cp_params["member-vnf-index-ref"],
 486                                 cp_params["vnfd-connection-point-ref"],
 487                                 vnf_descriptor["id"]))
 488                     if cp_params.get("ip-address"):
 489                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 490                                                      vdu_descriptor["id"], "interfaces",
 491                                                      interface_descriptor["name"], "ip_address"),
 492                                       cp_params["ip-address"])
 493                     if cp_params.get("mac-address"):
 494                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 495                                                      vdu_descriptor["id"], "interfaces",
 496                                                      interface_descriptor["name"], "mac_address"),
 497                                       cp_params["mac-address"])
 498         return RO_ns_params
 499
 500     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 501         # make a copy to do not change
 502         vdu_create = copy(vdu_create)
 503         vdu_delete = copy(vdu_delete)
 504
 505         vdurs = db_vnfr.get("vdur")
 506         if vdurs is None:
 507             vdurs = []
 508         vdu_index = len(vdurs)
 509         while vdu_index:
 510             vdu_index -= 1
 511             vdur = vdurs[vdu_index]
 512             if vdur.get("pdu-type"):
 513                 continue
 514             vdu_id_ref = vdur["vdu-id-ref"]
 515             if vdu_create and vdu_create.get(vdu_id_ref):
 516                 for index in range(0, vdu_create[vdu_id_ref]):
 517                     vdur = deepcopy(vdur)
 518                     vdur["_id"] = str(uuid4())
 519                     vdur["count-index"] += 1
 520                     vdurs.insert(vdu_index+1+index, vdur)
 521                 del vdu_create[vdu_id_ref]
 522             if vdu_delete and vdu_delete.get(vdu_id_ref):
 523                 del vdurs[vdu_index]
 524                 vdu_delete[vdu_id_ref] -= 1
 525                 if not vdu_delete[vdu_id_ref]:
 526                     del vdu_delete[vdu_id_ref]
 527         # check all operations are done
 528         if vdu_create or vdu_delete:
 529             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 530                 vdu_create))
 531         if vdu_delete:
 532             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 533                 vdu_delete))
 534
 535         vnfr_update = {"vdur": vdurs}
 536         db_vnfr["vdur"] = vdurs
 537         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 538
 539     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 540         """
 541         Updates database nsr with the RO info for the created vld
 542         :param ns_update_nsr: dictionary to be filled with the updated info
 543         :param db_nsr: content of db_nsr. This is also modified
 544         :param nsr_desc_RO: nsr descriptor from RO
 545         :return: Nothing, LcmException is raised on errors
 546         """
 547
 548         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 549             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 550                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 551                     continue
 552                 vld["vim-id"] = net_RO.get("vim_net_id")
 553                 vld["name"] = net_RO.get("vim_name")
 554                 vld["status"] = net_RO.get("status")
 555                 vld["status-detailed"] = net_RO.get("error_msg")
 556                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 557                 break
 558             else:
 559                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 560
 561     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 562         """
 563         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 564         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 565         :param nsr_desc_RO: nsr descriptor from RO
 566         :return: Nothing, LcmException is raised on errors
 567         """
 568         for vnf_index, db_vnfr in db_vnfrs.items():
 569             for vnf_RO in nsr_desc_RO["vnfs"]:
 570                 if vnf_RO["member_vnf_index"] != vnf_index:
 571                     continue
 572                 vnfr_update = {}
 573                 if vnf_RO.get("ip_address"):
 574                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 575                 elif not db_vnfr.get("ip-address"):
 576                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 577
 578                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 579                     vdur_RO_count_index = 0
 580                     if vdur.get("pdu-type"):
 581                         continue
 582                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 583                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 584                             continue
 585                         if vdur["count-index"] != vdur_RO_count_index:
 586                             vdur_RO_count_index += 1
 587                             continue
 588                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 589                         if vdur_RO.get("ip_address"):
 590                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 591                         else:
 592                             vdur["ip-address"] = None
 593                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 594                         vdur["name"] = vdur_RO.get("vim_name")
 595                         vdur["status"] = vdur_RO.get("status")
 596                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 597                         for ifacer in get_iterable(vdur, "interfaces"):
 598                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 599                                 if ifacer["name"] == interface_RO.get("internal_name"):
 600                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 601                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 602                                     break
 603                             else:
 604                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 605                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 606                                                                           ifacer["name"]))
 607                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 608                         break
 609                     else:
 610                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 611                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 612
 613                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 614                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 615                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 616                             continue
 617                         vld["vim-id"] = net_RO.get("vim_net_id")
 618                         vld["name"] = net_RO.get("vim_name")
 619                         vld["status"] = net_RO.get("status")
 620                         vld["status-detailed"] = net_RO.get("error_msg")
 621                         vnfr_update["vld.{}".format(vld_index)] = vld
 622                         break
 623                     else:
 624                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 625                             vnf_index, vld["id"]))
 626
 627                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 628                 break
 629
 630             else:
 631                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 632
 633     async def instantiate(self, nsr_id, nslcmop_id):
 634
 635         # Try to lock HA task here
 636         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 637         if not task_is_locked_by_me:
 638             return
 639
 640         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 641         self.logger.debug(logging_text + "Enter")
 642         # get all needed from database
 643         start_deploy = time()
 644         db_nsr = None
 645         db_nslcmop = None
 646         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 647         db_nslcmop_update = {}
 648         nslcmop_operation_state = None
 649         db_vnfrs = {}
 650         RO_descriptor_number = 0   # number of descriptors created at RO
 651         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 652         n2vc_info = {}
 653         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 654         exc = None
 655         try:
 656             # wait for any previous tasks in process
 657             step = "Waiting for previous tasks"
 658             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 659
 660             step = "Getting nslcmop={} from db".format(nslcmop_id)
 661             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 662             step = "Getting nsr={} from db".format(nsr_id)
 663             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 664             ns_params = db_nslcmop.get("operationParams")
 665             nsd = db_nsr["nsd"]
 666             nsr_name = db_nsr["name"]   # TODO short-name??
 667
 668             step = "Getting vnfrs from db"
 669             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 670             db_vnfds_ref = {}
 671             db_vnfds = {}
 672             db_vnfds_index = {}
 673             for vnfr in db_vnfrs_list:
 674                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 675                 vnfd_id = vnfr["vnfd-id"]
 676                 vnfd_ref = vnfr["vnfd-ref"]
 677                 if vnfd_id not in db_vnfds:
 678                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 679                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 680                     db_vnfds_ref[vnfd_ref] = vnfd
 681                     db_vnfds[vnfd_id] = vnfd
 682                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 683
 684             # Get or generates the _admin.deployed,VCA list
 685             vca_deployed_list = None
 686             vca_model_name = None
 687             if db_nsr["_admin"].get("deployed"):
 688                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 689                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 690             if vca_deployed_list is None:
 691                 vca_deployed_list = []
 692                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 693                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 694             elif isinstance(vca_deployed_list, dict):
 695                 # maintain backward compatibility. Change a dict to list at database
 696                 vca_deployed_list = list(vca_deployed_list.values())
 697                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 698                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 699
 700             db_nsr_update["detailed-status"] = "creating"
 701             db_nsr_update["operational-status"] = "init"
 702             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 703                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 704                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 705                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 706
 707             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 708             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 709             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 710
 711             # Deploy charms
 712             # The parameters we'll need to deploy a charm
 713             number_to_configure = 0
 714
 715             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 716                 """An inner function to deploy the charm from either ns, vnf or vdu
 717                 For ns both vnf_index and vdu_id are None.
 718                 For vnf only vdu_id is None
 719                 For vdu both vnf_index and vdu_id contain a value
 720                 """
 721                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 722                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 723
 724                 machine_spec = {}
 725                 if native_charm:
 726                     machine_spec["username"] = charm_params.get("username")
 727                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 728
 729                 # Note: The charm needs to exist on disk at the location
 730                 # specified by charm_path.
 731                 descriptor = vnfd if vnf_index else nsd
 732                 base_folder = descriptor["_admin"]["storage"]
 733                 storage_params = self.fs.get_params()
 734                 charm_path = "{}{}/{}/charms/{}".format(
 735                     storage_params["path"],
 736                     base_folder["folder"],
 737                     base_folder["pkg-dir"],
 738                     proxy_charm
 739                 )
 740
 741                 # ns_name will be ignored in the current version of N2VC
 742                 # but will be implemented for the next point release.
 743                 model_name = nsr_id
 744                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 745                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 746                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 747
 748                 vca_index = len(vca_deployed_list)
 749                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 750                 # 26*26 charm in the same NS
 751                 application_name = application_name[0:48]
 752                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 753                 vca_deployed_ = {
 754                     "member-vnf-index": vnf_index,
 755                     "vdu_id": vdu_id,
 756                     "model": model_name,
 757                     "application": application_name,
 758                     "operational-status": "init",
 759                     "detailed-status": "",
 760                     "step": "initial-deploy",
 761                     "vnfd_id": vnfd_id,
 762                     "vdu_name": vdu_name,
 763                     "vdu_count_index": vdu_count_index,
 764                 }
 765                 vca_deployed_list.append(vca_deployed_)
 766                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 767                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 768
 769                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 770                                                                                                 proxy_charm))
 771                 if not n2vc_info:
 772                     n2vc_info["nsr_id"] = nsr_id
 773                     n2vc_info["nslcmop_id"] = nslcmop_id
 774                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 775                     n2vc_info["lcmOperationType"] = "instantiate"
 776                     n2vc_info["deployed"] = vca_deployed_list
 777                     n2vc_info["db_update"] = db_nsr_update
 778                 task = asyncio.ensure_future(
 779                     self.n2vc.DeployCharms(
 780                         model_name,          # The network service name
 781                         application_name,    # The application name
 782                         descriptor,          # The vnf/nsd descriptor
 783                         charm_path,          # Path to charm
 784                         charm_params,        # Runtime params, like mgmt ip
 785                         machine_spec,        # for native charms only
 786                         self.n2vc_callback,  # Callback for status changes
 787                         n2vc_info,           # Callback parameter
 788                         None,                # Callback parameter (task)
 789                     )
 790                 )
 791                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 792                                                          n2vc_info))
 793                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 794
 795             step = "Looking for needed vnfd to configure with proxy charm"
 796             self.logger.debug(logging_text + step)
 797
 798             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 799                 vnfd_id = c_vnf["vnfd-id-ref"]
 800                 vnf_index = str(c_vnf["member-vnf-index"])
 801                 vnfd = db_vnfds_ref[vnfd_id]
 802
 803                 # Get additional parameters
 804                 vnfr_params = {}
 805                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 806                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 807                 for k, v in vnfr_params.items():
 808                     if isinstance(v, str) and v.startswith("!!yaml "):
 809                         vnfr_params[k] = yaml.safe_load(v[7:])
 810
 811                 step = "deploying proxy charms for configuration"
 812                 # Check if this VNF has a charm configuration
 813                 vnf_config = vnfd.get("vnf-configuration")
 814                 if vnf_config and vnf_config.get("juju"):
 815                     proxy_charm = vnf_config["juju"]["charm"]
 816                     if vnf_config["juju"].get("proxy") is False:
 817                         # native_charm, will be deployed after VM. Skip
 818                         proxy_charm = None
 819
 820                     if proxy_charm:
 821                         if not vca_model_name:
 822                             step = "creating VCA model name '{}'".format(nsr_id)
 823                             self.logger.debug(logging_text + step)
 824                             await self.n2vc.CreateNetworkService(nsr_id)
 825                             vca_model_name = nsr_id
 826                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 827                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 828                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 829                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 830                         charm_params = {
 831                             "user_values": vnfr_params,
 832                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 833                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 834                         }
 835
 836                         # Login to the VCA. If there are multiple calls to login(),
 837                         # subsequent calls will be a nop and return immediately.
 838                         await self.n2vc.login()
 839
 840                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 841                         number_to_configure += 1
 842
 843                 # Deploy charms for each VDU that supports one.
 844                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 845                     vdu_config = vdu.get('vdu-configuration')
 846                     proxy_charm = None
 847
 848                     if vdu_config and vdu_config.get("juju"):
 849                         proxy_charm = vdu_config["juju"]["charm"]
 850                         if vdu_config["juju"].get("proxy") is False:
 851                             # native_charm, will be deployed after VM. Skip
 852                             proxy_charm = None
 853                         if proxy_charm:
 854                             if not vca_model_name:
 855                                 step = "creating VCA model name"
 856                                 await self.n2vc.CreateNetworkService(nsr_id)
 857                                 vca_model_name = nsr_id
 858                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 859                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 860                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 861                                                                                                           vdu["id"])
 862                             await self.n2vc.login()
 863                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 864                             # TODO for the moment only first vdu_id contains a charm deployed
 865                             if vdur["vdu-id-ref"] != vdu["id"]:
 866                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 867                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 868                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 869                             charm_params = {
 870                                 "user_values": vnfr_params,
 871                                 "rw_mgmt_ip": vdur["ip-address"],
 872                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 873                             }
 874                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 875                                          charm_params, n2vc_info)
 876                             number_to_configure += 1
 877
 878             # Check if this NS has a charm configuration
 879
 880             ns_config = nsd.get("ns-configuration")
 881             if ns_config and ns_config.get("juju"):
 882                 proxy_charm = ns_config["juju"]["charm"]
 883                 if ns_config["juju"].get("proxy") is False:
 884                     # native_charm, will be deployed after VM. Skip
 885                     proxy_charm = None
 886                 if proxy_charm:
 887                     step = "deploying proxy charm to configure ns"
 888                     # TODO is NS magmt IP address needed?
 889
 890                     # Get additional parameters
 891                     additional_params = {}
 892                     if db_nsr.get("additionalParamsForNs"):
 893                         additional_params = db_nsr["additionalParamsForNs"].copy()
 894                     for k, v in additional_params.items():
 895                         if isinstance(v, str) and v.startswith("!!yaml "):
 896                             additional_params[k] = yaml.safe_load(v[7:])
 897
 898                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 899                     charm_params = {
 900                         "user_values": additional_params,
 901                         # "rw_mgmt_ip": db_nsr["ip-address"],
 902                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 903                     }
 904
 905                     # Login to the VCA. If there are multiple calls to login(),
 906                     # subsequent calls will be a nop and return immediately.
 907                     await self.n2vc.login()
 908                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 909                     number_to_configure += 1
 910
 911             db_nsr_update["operational-status"] = "running"
 912
 913             # Wait until all charms has reached blocked or active status
 914             step = "waiting proxy charms to be ready"
 915             if number_to_configure:
 916                 # wait until all charms are configured.
 917                 # steps are:
 918                 #       initial-deploy
 919                 #       get-ssh-public-key
 920                 #       generate-ssh-key
 921                 #       retry-get-ssh-public-key
 922                 #       ssh-public-key-obtained
 923                 while time() <= start_deploy + self.total_deploy_timeout:
 924                     if db_nsr_update:
 925                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 926                     if db_nslcmop_update:
 927                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 928
 929                     all_active = True
 930                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 931                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 932                         if vca_deployed["step"] == "initial-deploy":
 933                             if vca_deployed["operational-status"] in ("active", "blocked"):
 934                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 935                                        .format(vca_deployed["member-vnf-index"],
 936                                                vca_deployed["vdu_id"])
 937                                 self.logger.debug(logging_text + step)
 938                                 try:
 939                                     primitive_id = await self.n2vc.ExecutePrimitive(
 940                                         vca_deployed["model"],
 941                                         vca_deployed["application"],
 942                                         "get-ssh-public-key",
 943                                         None,
 944                                     )
 945                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 946                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 947                                         primitive_id
 948                                     db_nsr_update[database_entry + "operational-status"] =\
 949                                         vca_deployed["operational-status"]
 950                                 except PrimitiveDoesNotExist:
 951                                     ssh_public_key = None
 952                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 953                                         "ssh-public-key-obtained"
 954                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 955                                         ssh_public_key
 956                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 957                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 958                                     self.logger.debug(logging_text + step)
 959
 960                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 961                             primitive_id = vca_deployed["primitive_id"]
 962                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 963                                                                                   primitive_id)
 964                             if primitive_status in ("completed", "failed"):
 965                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 966                                                                                       primitive_id)
 967                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 968                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 969                                         primitive_result.get("pubkey"):
 970                                     ssh_public_key = primitive_result.get("pubkey")
 971                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 972                                         "ssh-public-key-obtained"
 973                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 974                                         ssh_public_key
 975                                     n2vc_key_list.append(ssh_public_key)
 976                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 977                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 978                                     self.logger.debug(logging_text + step)
 979                                 else:  # primitive_status == "failed":
 980                                     if vca_deployed["step"] == "get-ssh-public-key":
 981                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 982                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 983                                                                      vca_deployed["vdu_id"])
 984                                         self.logger.debug(logging_text + step)
 985                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 986                                             "generate-ssh-key"
 987                                         primitive_id = await self.n2vc.ExecutePrimitive(
 988                                             vca_deployed["model"],
 989                                             vca_deployed["application"],
 990                                             "generate-ssh-key",
 991                                             None,
 992                                         )
 993                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 994                                             primitive_id
 995                                     else:  # failed for second time
 996                                         raise LcmException(
 997                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 998
 999                         elif vca_deployed["step"] == "generate-ssh-key":
1000                             primitive_id = vca_deployed["primitive_id"]
1001                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1002                                                                                   primitive_id)
1003                             if primitive_status in ("completed", "failed"):
1004                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1005                                                                                       primitive_id)
1006                                 vca_deployed["primitive_id"] = db_nsr_update[
1007                                     database_entry + "primitive_id"] = None
1008                                 if primitive_status == "completed":
1009                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1010                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1011                                                               vca_deployed["vdu_id"])
1012                                     self.logger.debug(logging_text + step)
1013                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1014                                         "retry-get-ssh-public-key"
1015                                     primitive_id = await self.n2vc.ExecutePrimitive(
1016                                         vca_deployed["model"],
1017                                         vca_deployed["application"],
1018                                         "get-ssh-public-key",
1019                                         None,
1020                                     )
1021                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1022                                         primitive_id
1023
1024                                 else:  # primitive_status == "failed":
1025                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1026                                                        .format(primitive_result))
1027
1028                         if vca_deployed["step"] != "ssh-public-key-obtained":
1029                             all_active = False
1030
1031                     if all_active:
1032                         break
1033                     await asyncio.sleep(5)
1034                 else:   # total_deploy_timeout
1035                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1036                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1037
1038             # deploy RO
1039             # get vnfds, instantiate at RO
1040             for c_vnf in nsd.get("constituent-vnfd", ()):
1041                 member_vnf_index = c_vnf["member-vnf-index"]
1042                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1043                 vnfd_ref = vnfd["id"]
1044                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1045                     vnfd_ref, member_vnf_index)
1046                 # self.logger.debug(logging_text + step)
1047                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1048                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1049                 RO_descriptor_number += 1
1050
1051                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1052                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1053                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1054                         break
1055                 else:
1056                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1057                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1058
1059                 # look if present
1060                 RO_update = {"member-vnf-index": member_vnf_index}
1061                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1062                 if vnfd_list:
1063                     RO_update["id"] = vnfd_list[0]["uuid"]
1064                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1065                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1066                 else:
1067                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1068                                            get("additionalParamsForVnf"), nsr_id)
1069                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1070                     RO_update["id"] = desc["uuid"]
1071                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1072                         vnfd_ref, member_vnf_index, desc["uuid"]))
1073                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1074                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1075                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1076
1077             # create nsd at RO
1078             nsd_ref = nsd["id"]
1079             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1080             # self.logger.debug(logging_text + step)
1081
1082             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1083             RO_descriptor_number += 1
1084             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1085             if nsd_list:
1086                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1087                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1088                     nsd_ref, RO_nsd_uuid))
1089             else:
1090                 nsd_RO = deepcopy(nsd)
1091                 nsd_RO["id"] = RO_osm_nsd_id
1092                 nsd_RO.pop("_id", None)
1093                 nsd_RO.pop("_admin", None)
1094                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1095                     member_vnf_index = c_vnf["member-vnf-index"]
1096                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1097                 for c_vld in nsd_RO.get("vld", ()):
1098                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1099                         member_vnf_index = cp["member-vnf-index-ref"]
1100                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1101
1102                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1103                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1104                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1105                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1106             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1107
1108             # Crate ns at RO
1109             # if present use it unless in error status
1110             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1111             if RO_nsr_id:
1112                 try:
1113                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1114                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1115                     desc = await self.RO.show("ns", RO_nsr_id)
1116                 except ROclient.ROClientException as e:
1117                     if e.http_code != HTTPStatus.NOT_FOUND:
1118                         raise
1119                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1120                 if RO_nsr_id:
1121                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1122                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1123                     if ns_status == "ERROR":
1124                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1125                         self.logger.debug(logging_text + step)
1126                         await self.RO.delete("ns", RO_nsr_id)
1127                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1128             if not RO_nsr_id:
1129                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1130                 # self.logger.debug(logging_text + step)
1131
1132                 # check if VIM is creating and wait  look if previous tasks in process
1133                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1134                 if task_dependency:
1135                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1136                     self.logger.debug(logging_text + step)
1137                     await asyncio.wait(task_dependency, timeout=3600)
1138                 if ns_params.get("vnf"):
1139                     for vnf in ns_params["vnf"]:
1140                         if "vimAccountId" in vnf:
1141                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1142                                                                                         vnf["vimAccountId"])
1143                         if task_dependency:
1144                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1145                             self.logger.debug(logging_text + step)
1146                             await asyncio.wait(task_dependency, timeout=3600)
1147
1148                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1149
1150                 # feature 1429. Add n2vc public key to needed VMs
1151                 n2vc_key = await self.n2vc.GetPublicKey()
1152                 n2vc_key_list.append(n2vc_key)
1153                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1154
1155                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1156                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1157                                             name=db_nsr["name"],
1158                                             scenario=RO_nsd_uuid)
1159                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1160                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1161                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1162                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1163             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1164
1165             # wait until NS is ready
1166             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1167             detailed_status_old = None
1168             self.logger.debug(logging_text + step)
1169
1170             while time() <= start_deploy + self.total_deploy_timeout:
1171                 desc = await self.RO.show("ns", RO_nsr_id)
1172                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1173                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1174                 if ns_status == "ERROR":
1175                     raise ROclient.ROClientException(ns_status_info)
1176                 elif ns_status == "BUILD":
1177                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1178                 elif ns_status == "ACTIVE":
1179                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1180                     try:
1181                         self.ns_update_vnfr(db_vnfrs, desc)
1182                         break
1183                     except LcmExceptionNoMgmtIP:
1184                         pass
1185                 else:
1186                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1187                 if detailed_status != detailed_status_old:
1188                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1189                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1190                 await asyncio.sleep(5, loop=self.loop)
1191             else:   # total_deploy_timeout
1192                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1193
1194             step = "Updating NSR"
1195             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1196
1197             db_nsr_update["operational-status"] = "running"
1198             db_nsr["detailed-status"] = "Configuring vnfr"
1199             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1200
1201             # Configure proxy charms once VMs are up
1202             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1203                 vnf_index = vca_deployed.get("member-vnf-index")
1204                 vdu_id = vca_deployed.get("vdu_id")
1205                 vdu_name = None
1206                 vdu_count_index = None
1207
1208                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1209                                                                                                            vdu_id)
1210                 add_params = {}
1211                 initial_config_primitive_list = []
1212                 if vnf_index:
1213                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1214                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1215                     vnfd = db_vnfds_index[vnf_index]
1216
1217                     if vdu_id:
1218                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1219                             if vdu["id"] == vdu_id:
1220                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1221                                     'initial-config-primitive', [])
1222                                 break
1223                         else:
1224                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1225                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1226                         # TODO for the moment only first vdu_id contains a charm deployed
1227                         if vdur["vdu-id-ref"] != vdu["id"]:
1228                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1229                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1230                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1231                     else:
1232                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1233                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1234                 else:
1235                     if db_nsr.get("additionalParamsForNs"):
1236                         add_params = db_nsr["additionalParamsForNs"].copy()
1237                     for k, v in add_params.items():
1238                         if isinstance(v, str) and v.startswith("!!yaml "):
1239                             add_params[k] = yaml.safe_load(v[7:])
1240                     add_params["rw_mgmt_ip"] = None
1241                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1242
1243                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1244                 initial_config_primitive_list = initial_config_primitive_list.copy()
1245                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1246                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1247
1248                 for initial_config_primitive in initial_config_primitive_list:
1249                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1250                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1251                                       .format(initial_config_primitive["name"], primitive_params_))
1252                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1253                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1254                         initial_config_primitive["name"],
1255                         primitive_params_,
1256                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1257                         retries_interval=30)
1258                     if primitive_result != "COMPLETED":
1259                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1260                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1261                                                    vca_deployed["vdu_id"], primitive_detail))
1262
1263             # Deploy native charms
1264             step = "Looking for needed vnfd to configure with native charm"
1265             self.logger.debug(logging_text + step)
1266
1267             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1268                 vnfd_id = c_vnf["vnfd-id-ref"]
1269                 vnf_index = str(c_vnf["member-vnf-index"])
1270                 vnfd = db_vnfds_ref[vnfd_id]
1271
1272                 # Get additional parameters
1273                 vnfr_params = {}
1274                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1275                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1276                 for k, v in vnfr_params.items():
1277                     if isinstance(v, str) and v.startswith("!!yaml "):
1278                         vnfr_params[k] = yaml.safe_load(v[7:])
1279
1280                 # Check if this VNF has a charm configuration
1281                 vnf_config = vnfd.get("vnf-configuration")
1282                 if vnf_config and vnf_config.get("juju"):
1283                     native_charm = vnf_config["juju"].get("proxy") is False
1284                     proxy_charm = vnf_config["juju"]["charm"]
1285                     if native_charm and proxy_charm:
1286                         if not vca_model_name:
1287                             step = "creating VCA model name '{}'".format(nsr_id)
1288                             self.logger.debug(logging_text + step)
1289                             await self.n2vc.CreateNetworkService(nsr_id)
1290                             vca_model_name = nsr_id
1291                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1292                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1293                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1294                         self.logger.debug(logging_text + step)
1295
1296                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1297                         charm_params = {
1298                             "user_values": vnfr_params,
1299                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1300                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1301                         }
1302
1303                         # get username
1304                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1305                         #  merged. Meanwhile let's get username from initial-config-primitive
1306                         if vnf_config.get("initial-config-primitive"):
1307                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1308                                 if param["name"] == "ssh-username":
1309                                     charm_params["username"] = param["value"]
1310                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1311                             if vnf_config["config-access"]["ssh-access"].get("required"):
1312                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1313
1314                         # Login to the VCA. If there are multiple calls to login(),
1315                         # subsequent calls will be a nop and return immediately.
1316                         await self.n2vc.login()
1317
1318                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1319                         number_to_configure += 1
1320
1321                 # Deploy charms for each VDU that supports one.
1322                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1323                     vdu_config = vdu.get('vdu-configuration')
1324                     native_charm = False
1325
1326                     if vdu_config and vdu_config.get("juju"):
1327                         native_charm = vdu_config["juju"].get("proxy") is False
1328                         proxy_charm = vdu_config["juju"]["charm"]
1329                         if native_charm and proxy_charm:
1330                             if not vca_model_name:
1331                                 step = "creating VCA model name"
1332                                 await self.n2vc.CreateNetworkService(nsr_id)
1333                                 vca_model_name = nsr_id
1334                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1335                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1336                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1337                                                                                                      vdu["id"])
1338
1339                             self.logger.debug(logging_text + step)
1340
1341                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1342
1343                             # TODO for the moment only first vdu_id contains a charm deployed
1344                             if vdur["vdu-id-ref"] != vdu["id"]:
1345                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1346                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1347                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1348                             charm_params = {
1349                                 "user_values": vnfr_params,
1350                                 "rw_mgmt_ip": vdur["ip-address"],
1351                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1352                             }
1353
1354                             # get username
1355                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1356                             #  merged. Meanwhile let's get username from initial-config-primitive
1357                             if vdu_config.get("initial-config-primitive"):
1358                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1359                                     if param["name"] == "ssh-username":
1360                                         charm_params["username"] = param["value"]
1361                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1362                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1363                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1364                                         "default-user")
1365
1366                             await self.n2vc.login()
1367
1368                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1369                                          charm_params, n2vc_info, native_charm)
1370                             number_to_configure += 1
1371
1372             # Check if this NS has a charm configuration
1373
1374             ns_config = nsd.get("ns-configuration")
1375             if ns_config and ns_config.get("juju"):
1376                 native_charm = ns_config["juju"].get("proxy") is False
1377                 proxy_charm = ns_config["juju"]["charm"]
1378                 if native_charm and proxy_charm:
1379                     step = "deploying native charm to configure ns"
1380                     # TODO is NS magmt IP address needed?
1381
1382                     # Get additional parameters
1383                     additional_params = {}
1384                     if db_nsr.get("additionalParamsForNs"):
1385                         additional_params = db_nsr["additionalParamsForNs"].copy()
1386                     for k, v in additional_params.items():
1387                         if isinstance(v, str) and v.startswith("!!yaml "):
1388                             additional_params[k] = yaml.safe_load(v[7:])
1389
1390                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1391                     charm_params = {
1392                         "user_values": additional_params,
1393                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1394                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1395                     }
1396
1397                     # get username
1398                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1399                     #  merged. Meanwhile let's get username from initial-config-primitive
1400                     if ns_config.get("initial-config-primitive"):
1401                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1402                             if param["name"] == "ssh-username":
1403                                 charm_params["username"] = param["value"]
1404                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1405                         if ns_config["config-access"]["ssh-access"].get("required"):
1406                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1407
1408                     # Login to the VCA. If there are multiple calls to login(),
1409                     # subsequent calls will be a nop and return immediately.
1410                     await self.n2vc.login()
1411                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1412                     number_to_configure += 1
1413
1414             # waiting all charms are ok
1415             configuration_failed = False
1416             if number_to_configure:
1417                 step = "Waiting all charms are active"
1418                 old_status = "configuring: init: {}".format(number_to_configure)
1419                 db_nsr_update["config-status"] = old_status
1420                 db_nsr_update["detailed-status"] = old_status
1421                 db_nslcmop_update["detailed-status"] = old_status
1422
1423                 # wait until all are configured.
1424                 while time() <= start_deploy + self.total_deploy_timeout:
1425                     if db_nsr_update:
1426                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1427                     if db_nslcmop_update:
1428                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1429                     # TODO add a fake task that set n2vc_event after some time
1430                     await n2vc_info["n2vc_event"].wait()
1431                     n2vc_info["n2vc_event"].clear()
1432                     all_active = True
1433                     status_map = {}
1434                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1435                     now = time()
1436                     for vca_deployed in vca_deployed_list:
1437                         vca_status = vca_deployed["operational-status"]
1438                         if vca_status not in status_map:
1439                             # Initialize it
1440                             status_map[vca_status] = 0
1441                         status_map[vca_status] += 1
1442
1443                         if vca_status == "active":
1444                             vca_deployed.pop("time_first_error", None)
1445                             vca_deployed.pop("status_first_error", None)
1446                             continue
1447
1448                         all_active = False
1449                         if vca_status in ("error", "blocked"):
1450                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1451                             # if not first time in this status error
1452                             if not vca_deployed.get("time_first_error"):
1453                                 vca_deployed["time_first_error"] = now
1454                                 continue
1455                         if vca_deployed.get("time_first_error") and \
1456                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1457                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1458                                                    .format(vca_deployed["member-vnf-index"],
1459                                                            vca_deployed["vdu_id"], vca_status,
1460                                                            vca_deployed["detailed-status-error"]))
1461
1462                     if all_active:
1463                         break
1464                     elif n2vc_error_text:
1465                         db_nsr_update["config-status"] = "failed"
1466                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1467                         db_nsr_update["detailed-status"] = error_text
1468                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1469                         db_nslcmop_update["detailed-status"] = error_text
1470                         db_nslcmop_update["statusEnteredTime"] = time()
1471                         configuration_failed = True
1472                         break
1473                     else:
1474                         cs = "configuring: "
1475                         separator = ""
1476                         for status, num in status_map.items():
1477                             cs += separator + "{}: {}".format(status, num)
1478                             separator = ", "
1479                         if old_status != cs:
1480                             db_nsr_update["config-status"] = cs
1481                             db_nsr_update["detailed-status"] = cs
1482                             db_nslcmop_update["detailed-status"] = cs
1483                             old_status = cs
1484                 else:   # total_deploy_timeout
1485                     raise LcmException("Timeout waiting ns to be configured")
1486
1487             if not configuration_failed:
1488                 # all is done
1489                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1490                 db_nslcmop_update["statusEnteredTime"] = time()
1491                 db_nslcmop_update["detailed-status"] = "done"
1492                 db_nsr_update["config-status"] = "configured"
1493                 db_nsr_update["detailed-status"] = "done"
1494
1495             return
1496
1497         except (ROclient.ROClientException, DbException, LcmException) as e:
1498             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1499             exc = e
1500         except asyncio.CancelledError:
1501             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1502             exc = "Operation was cancelled"
1503         except Exception as e:
1504             exc = traceback.format_exc()
1505             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1506                                  exc_info=True)
1507         finally:
1508             if exc:
1509                 if db_nsr:
1510                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1511                     db_nsr_update["operational-status"] = "failed"
1512                 if db_nslcmop:
1513                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1514                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1515                     db_nslcmop_update["statusEnteredTime"] = time()
1516             try:
1517                 if db_nsr:
1518                     db_nsr_update["_admin.nslcmop"] = None
1519                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1520                 if db_nslcmop_update:
1521                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1522             except DbException as e:
1523                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1524             if nslcmop_operation_state:
1525                 try:
1526                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1527                                                                    "operationState": nslcmop_operation_state},
1528                                             loop=self.loop)
1529                 except Exception as e:
1530                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1531
1532             self.logger.debug(logging_text + "Exit")
1533             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1534
1535     async def _destroy_charm(self, model, application):
1536         """
1537         Order N2VC destroy a charm
1538         :param model:
1539         :param application:
1540         :return: True if charm does not exist. False if it exist
1541         """
1542         if not await self.n2vc.HasApplication(model, application):
1543             return True  # Already removed
1544         await self.n2vc.RemoveCharms(model, application)
1545         return False
1546
1547     async def _wait_charm_destroyed(self, model, application, timeout):
1548         """
1549         Wait until charm does not exist
1550         :param model:
1551         :param application:
1552         :param timeout:
1553         :return: True if not exist, False if timeout
1554         """
1555         while True:
1556             if not await self.n2vc.HasApplication(model, application):
1557                 return True
1558             if timeout < 0:
1559                 return False
1560             await asyncio.sleep(10)
1561             timeout -= 10
1562
1563     # Check if this VNFD has a configured terminate action
1564     def _has_terminate_config_primitive(self, vnfd):
1565         vnf_config = vnfd.get("vnf-configuration")
1566         if vnf_config and vnf_config.get("terminate-config-primitive"):
1567             return True
1568         else:
1569             return False
1570
1571     @staticmethod
1572     def _get_terminate_config_primitive_seq_list(vnfd):
1573         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1574         # No need to check for existing primitive twice, already done before
1575         vnf_config = vnfd.get("vnf-configuration")
1576         seq_list = vnf_config.get("terminate-config-primitive")
1577         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1578         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1579         return seq_list_sorted
1580
1581     @staticmethod
1582     def _create_nslcmop(nsr_id, operation, params):
1583         """
1584         Creates a ns-lcm-opp content to be stored at database.
1585         :param nsr_id: internal id of the instance
1586         :param operation: instantiate, terminate, scale, action, ...
1587         :param params: user parameters for the operation
1588         :return: dictionary following SOL005 format
1589         """
1590         # Raise exception if invalid arguments
1591         if not (nsr_id and operation and params):
1592             raise LcmException(
1593                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1594         now = time()
1595         _id = str(uuid4())
1596         nslcmop = {
1597             "id": _id,
1598             "_id": _id,
1599             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1600             "operationState": "PROCESSING",
1601             "statusEnteredTime": now,
1602             "nsInstanceId": nsr_id,
1603             "lcmOperationType": operation,
1604             "startTime": now,
1605             "isAutomaticInvocation": False,
1606             "operationParams": params,
1607             "isCancelPending": False,
1608             "links": {
1609                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1610                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1611             }
1612         }
1613         return nslcmop
1614
1615     def _get_terminate_primitive_params(self, seq, vnf_index):
1616         primitive = seq.get('name')
1617         primitive_params = {}
1618         params = {
1619             "member_vnf_index": vnf_index,
1620             "primitive": primitive,
1621             "primitive_params": primitive_params,
1622         }
1623         desc_params = {}
1624         return self._map_primitive_params(seq, params, desc_params)
1625
1626     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1627                           vdu_name, primitive, mapped_primitive_params):
1628         # Create the "_admin.operations" array, or append operation if array already exists
1629         key_admin = '_admin'
1630         key_operations = 'operations'
1631         db_nslcmop_update = {}
1632         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1633         op_list = db_nslcmop_admin.get(key_operations)
1634         new_op = {'member_vnf_index': vnf_index,
1635                   'vdu_id': vdu_id,
1636                   'vdu_count_index': vdu_count_index,
1637                   'primitive': primitive,
1638                   'primitive_params': mapped_primitive_params}
1639         if db_nslcmop_admin:
1640             if not op_list:
1641                 # First operation, create key 'operations' with current operation as first list element
1642                 db_nslcmop_admin.update({key_operations: [new_op]})
1643                 op_list = db_nslcmop_admin.get(key_operations)
1644             else:
1645                 # Not first operation, append operation to existing list
1646                 op_list.append(new_op)
1647
1648         db_nslcmop_update['_admin.operations'] = op_list
1649         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1650
1651     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1652         """ Create a primitive with params from VNFD
1653             Called from terminate() before deleting instance
1654             Calls action() to execute the primitive """
1655         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1656         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1657         db_vnfds = {}
1658         # Loop over VNFRs
1659         for vnfr in db_vnfrs_list:
1660             vnfd_id = vnfr["vnfd-id"]
1661             vnf_index = vnfr["member-vnf-index-ref"]
1662             if vnfd_id not in db_vnfds:
1663                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1664                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1665                 db_vnfds[vnfd_id] = vnfd
1666             vnfd = db_vnfds[vnfd_id]
1667             if not self._has_terminate_config_primitive(vnfd):
1668                 continue
1669             # Get the primitive's sorted sequence list
1670             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1671             for seq in seq_list:
1672                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1673                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1674                     vnf_index, seq.get("name"))
1675                 self.logger.debug(logging_text + step)
1676                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1677                 primitive = seq.get('name')
1678                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1679                 # The following 3 parameters are currently set to None for 'terminate':
1680                 # vdu_id, vdu_count_index, vdu_name
1681                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1682                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1683                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1684                 # Add suboperation
1685                 self._add_suboperation(db_nslcmop,
1686                                        nslcmop_id,
1687                                        vnf_index,
1688                                        vdu_id,
1689                                        vdu_count_index,
1690                                        vdu_name,
1691                                        primitive,
1692                                        mapped_primitive_params)
1693                 # Suboperations: Call _ns_execute_primitive() instead of action()
1694                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1695                 nsr_deployed = db_nsr["_admin"]["deployed"]
1696                 result, result_detail = await self._ns_execute_primitive(
1697                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1698                     mapped_primitive_params)
1699
1700                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1701                 #    nsr_id, nslcmop_terminate_action_id)
1702                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1703                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1704                 if result not in result_ok:
1705                     raise LcmException(
1706                         "terminate_primitive_action for vnf_member_index={}",
1707                         " primitive={} fails with error {}".format(
1708                             vnf_index, seq.get("name"), result_detail))
1709
1710     async def terminate(self, nsr_id, nslcmop_id):
1711
1712         # Try to lock HA task here
1713         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1714         if not task_is_locked_by_me:
1715             return
1716
1717         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1718         self.logger.debug(logging_text + "Enter")
1719         db_nsr = None
1720         db_nslcmop = None
1721         exc = None
1722         failed_detail = []   # annotates all failed error messages
1723         vca_time_destroy = None   # time of where destroy charm order
1724         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1725         db_nslcmop_update = {}
1726         nslcmop_operation_state = None
1727         autoremove = False  # autoremove after terminated
1728         try:
1729             # wait for any previous tasks in process
1730             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1731
1732             step = "Getting nslcmop={} from db".format(nslcmop_id)
1733             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1734             step = "Getting nsr={} from db".format(nsr_id)
1735             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1736             # nsd = db_nsr["nsd"]
1737             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1738             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1739                 return
1740             # #TODO check if VIM is creating and wait
1741             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1742             # Call internal terminate action
1743             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1744
1745             db_nsr_update["operational-status"] = "terminating"
1746             db_nsr_update["config-status"] = "terminating"
1747
1748             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1749                 vca_model_name = nsr_deployed["VCA-model-name"]
1750                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1751                 self.logger.debug(logging_text + step)
1752                 try:
1753                     await self.n2vc.DestroyNetworkService(vca_model_name)
1754                 except NetworkServiceDoesNotExist:
1755                     pass
1756                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1757                 if nsr_deployed.get("VCA"):
1758                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1759                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1760                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1761             # for backward compatibility if charm have been created with "default"  model name delete one by one
1762             elif nsr_deployed and nsr_deployed.get("VCA"):
1763                 try:
1764                     step = "Scheduling configuration charms removing"
1765                     db_nsr_update["detailed-status"] = "Deleting charms"
1766                     self.logger.debug(logging_text + step)
1767                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1768                     # for backward compatibility
1769                     if isinstance(nsr_deployed["VCA"], dict):
1770                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1771                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1772                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1773
1774                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1775                         if vca_deployed:
1776                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1777                                 vca_deployed.clear()
1778                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1779                             else:
1780                                 vca_time_destroy = time()
1781                 except Exception as e:
1782                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1783
1784             # remove from RO
1785             RO_fail = False
1786
1787             # Delete ns
1788             RO_nsr_id = RO_delete_action = None
1789             if nsr_deployed and nsr_deployed.get("RO"):
1790                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1791                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1792             try:
1793                 if RO_nsr_id:
1794                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1795                         "Deleting ns from VIM"
1796                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1797                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1798                     self.logger.debug(logging_text + step)
1799                     desc = await self.RO.delete("ns", RO_nsr_id)
1800                     RO_delete_action = desc["action_id"]
1801                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1802                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1803                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1804                 if RO_delete_action:
1805                     # wait until NS is deleted from VIM
1806                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1807                         format(RO_nsr_id, RO_delete_action)
1808                     detailed_status_old = None
1809                     self.logger.debug(logging_text + step)
1810
1811                     delete_timeout = 20 * 60   # 20 minutes
1812                     while delete_timeout > 0:
1813                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1814                                                   extra_item_id=RO_delete_action)
1815                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1816                         if ns_status == "ERROR":
1817                             raise ROclient.ROClientException(ns_status_info)
1818                         elif ns_status == "BUILD":
1819                             detailed_status = step + "; {}".format(ns_status_info)
1820                         elif ns_status == "ACTIVE":
1821                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1822                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1823                             break
1824                         else:
1825                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1826                         if detailed_status != detailed_status_old:
1827                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1828                                 db_nsr_update["detailed-status"] = detailed_status
1829                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1830                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1831                         await asyncio.sleep(5, loop=self.loop)
1832                         delete_timeout -= 5
1833                     else:  # delete_timeout <= 0:
1834                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1835
1836             except ROclient.ROClientException as e:
1837                 if e.http_code == 404:  # not found
1838                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1839                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1840                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1841                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1842                 elif e.http_code == 409:   # conflict
1843                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1844                     self.logger.debug(logging_text + failed_detail[-1])
1845                     RO_fail = True
1846                 else:
1847                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1848                     self.logger.error(logging_text + failed_detail[-1])
1849                     RO_fail = True
1850
1851             # Delete nsd
1852             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1853                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1854                 try:
1855                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1856                         "Deleting nsd from RO"
1857                     await self.RO.delete("nsd", RO_nsd_id)
1858                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1859                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1860                 except ROclient.ROClientException as e:
1861                     if e.http_code == 404:  # not found
1862                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1863                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1864                     elif e.http_code == 409:   # conflict
1865                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1866                         self.logger.debug(logging_text + failed_detail[-1])
1867                         RO_fail = True
1868                     else:
1869                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1870                         self.logger.error(logging_text + failed_detail[-1])
1871                         RO_fail = True
1872
1873             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1874                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1875                     if not vnf_deployed or not vnf_deployed["id"]:
1876                         continue
1877                     try:
1878                         RO_vnfd_id = vnf_deployed["id"]
1879                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1880                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1881                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1882                         await self.RO.delete("vnfd", RO_vnfd_id)
1883                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1884                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1885                     except ROclient.ROClientException as e:
1886                         if e.http_code == 404:  # not found
1887                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1888                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1889                         elif e.http_code == 409:   # conflict
1890                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1891                             self.logger.debug(logging_text + failed_detail[-1])
1892                         else:
1893                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1894                             self.logger.error(logging_text + failed_detail[-1])
1895
1896             # wait until charm deleted
1897             if vca_time_destroy:
1898                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1899                     "Waiting for deletion of configuration charms"
1900                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1901                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1902                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1903                     if not vca_deployed:
1904                         continue
1905                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1906                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1907                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1908                                                             timeout):
1909                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1910                             vca_deployed["application"]))
1911                     else:
1912                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1913
1914             if failed_detail:
1915                 self.logger.error(logging_text + " ;".join(failed_detail))
1916                 db_nsr_update["operational-status"] = "failed"
1917                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1918                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1919                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1920                 db_nslcmop_update["statusEnteredTime"] = time()
1921             else:
1922                 db_nsr_update["operational-status"] = "terminated"
1923                 db_nsr_update["detailed-status"] = "Done"
1924                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1925                 db_nslcmop_update["detailed-status"] = "Done"
1926                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1927                 db_nslcmop_update["statusEnteredTime"] = time()
1928                 if db_nslcmop["operationParams"].get("autoremove"):
1929                     autoremove = True
1930
1931         except (ROclient.ROClientException, DbException, LcmException) as e:
1932             self.logger.error(logging_text + "Exit Exception {}".format(e))
1933             exc = e
1934         except asyncio.CancelledError:
1935             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1936             exc = "Operation was cancelled"
1937         except Exception as e:
1938             exc = traceback.format_exc()
1939             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1940         finally:
1941             if exc and db_nslcmop:
1942                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1943                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1944                 db_nslcmop_update["statusEnteredTime"] = time()
1945             try:
1946                 if db_nslcmop and db_nslcmop_update:
1947                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1948                 if db_nsr:
1949                     db_nsr_update["_admin.nslcmop"] = None
1950                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1951             except DbException as e:
1952                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1953             if nslcmop_operation_state:
1954                 try:
1955                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1956                                                                  "operationState": nslcmop_operation_state,
1957                                                                  "autoremove": autoremove},
1958                                             loop=self.loop)
1959                 except Exception as e:
1960                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1961             self.logger.debug(logging_text + "Exit")
1962             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1963
1964     @staticmethod
1965     def _map_primitive_params(primitive_desc, params, instantiation_params):
1966         """
1967         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1968         The default-value is used. If it is between < > it look for a value at instantiation_params
1969         :param primitive_desc: portion of VNFD/NSD that describes primitive
1970         :param params: Params provided by user
1971         :param instantiation_params: Instantiation params provided by user
1972         :return: a dictionary with the calculated params
1973         """
1974         calculated_params = {}
1975         for parameter in primitive_desc.get("parameter", ()):
1976             param_name = parameter["name"]
1977             if param_name in params:
1978                 calculated_params[param_name] = params[param_name]
1979             elif "default-value" in parameter or "value" in parameter:
1980                 if "value" in parameter:
1981                     calculated_params[param_name] = parameter["value"]
1982                 else:
1983                     calculated_params[param_name] = parameter["default-value"]
1984                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1985                         and calculated_params[param_name].endswith(">"):
1986                     if calculated_params[param_name][1:-1] in instantiation_params:
1987                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1988                     else:
1989                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1990                                            format(parameter["default-value"], primitive_desc["name"]))
1991             else:
1992                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1993                                    format(param_name, primitive_desc["name"]))
1994
1995             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1996                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1997                                                                width=256)
1998             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1999                 calculated_params[param_name] = calculated_params[param_name][7:]
2000         return calculated_params
2001
2002     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
2003                                     primitive, primitive_params, retries=0, retries_interval=30):
2004         start_primitive_time = time()
2005         try:
2006             for vca_deployed in db_deployed["VCA"]:
2007                 if not vca_deployed:
2008                     continue
2009                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2010                     continue
2011                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2012                     continue
2013                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2014                     continue
2015                 break
2016             else:
2017                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2018                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2019             model_name = vca_deployed.get("model")
2020             application_name = vca_deployed.get("application")
2021             if not model_name or not application_name:
2022                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2023                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2024                                                                        vdu_count_index))
2025             # if vca_deployed["operational-status"] != "active":
2026             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2027             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2028             callback = None  # self.n2vc_callback
2029             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2030             await self.n2vc.login()
2031             if primitive == "config":
2032                 primitive_params = {"params": primitive_params}
2033             while retries >= 0:
2034                 primitive_id = await self.n2vc.ExecutePrimitive(
2035                     model_name,
2036                     application_name,
2037                     primitive,
2038                     callback,
2039                     *callback_args,
2040                     **primitive_params
2041                 )
2042                 while time() - start_primitive_time < self.timeout_primitive:
2043                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2044                     if primitive_result_ in ("completed", "failed"):
2045                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2046                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2047                         break
2048                     elif primitive_result_ is None and primitive == "config":
2049                         primitive_result = "COMPLETED"
2050                         detailed_result = None
2051                         break
2052                     else:  # ("running", "pending", None):
2053                         pass
2054                     await asyncio.sleep(5)
2055                 else:
2056                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2057                 if primitive_result == "COMPLETED":
2058                     break
2059                 retries -= 1
2060                 if retries >= 0:
2061                     await asyncio.sleep(retries_interval)
2062
2063             return primitive_result, detailed_result
2064         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2065             return "FAILED", str(e)
2066
2067     async def action(self, nsr_id, nslcmop_id):
2068
2069         # Try to lock HA task here
2070         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2071         if not task_is_locked_by_me:
2072             return
2073
2074         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2075         self.logger.debug(logging_text + "Enter")
2076         # get all needed from database
2077         db_nsr = None
2078         db_nslcmop = None
2079         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2080         db_nslcmop_update = {}
2081         nslcmop_operation_state = None
2082         nslcmop_operation_state_detail = None
2083         exc = None
2084         try:
2085             # wait for any previous tasks in process
2086             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2087
2088             step = "Getting information from database"
2089             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2090             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2091
2092             nsr_deployed = db_nsr["_admin"].get("deployed")
2093             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2094             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2095             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2096             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2097
2098             if vnf_index:
2099                 step = "Getting vnfr from database"
2100                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2101                 step = "Getting vnfd from database"
2102                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2103             else:
2104                 if db_nsr.get("nsd"):
2105                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2106                 else:
2107                     step = "Getting nsd from database"
2108                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2109
2110             # for backward compatibility
2111             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2112                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2113                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2114                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2115
2116             primitive = db_nslcmop["operationParams"]["primitive"]
2117             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2118
2119             # look for primitive
2120             config_primitive_desc = None
2121             if vdu_id:
2122                 for vdu in get_iterable(db_vnfd, "vdu"):
2123                     if vdu_id == vdu["id"]:
2124                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2125                             if config_primitive["name"] == primitive:
2126                                 config_primitive_desc = config_primitive
2127                                 break
2128             elif vnf_index:
2129                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2130                     if config_primitive["name"] == primitive:
2131                         config_primitive_desc = config_primitive
2132                         break
2133             else:
2134                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2135                     if config_primitive["name"] == primitive:
2136                         config_primitive_desc = config_primitive
2137                         break
2138
2139             if not config_primitive_desc:
2140                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2141                                    format(primitive))
2142
2143             desc_params = {}
2144             if vnf_index:
2145                 if db_vnfr.get("additionalParamsForVnf"):
2146                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2147             else:
2148                 if db_nsr.get("additionalParamsForVnf"):
2149                     desc_params.update(db_nsr["additionalParamsForNs"])
2150
2151             # TODO check if ns is in a proper status
2152             result, result_detail = await self._ns_execute_primitive(
2153                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2154                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2155             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2156             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2157             db_nslcmop_update["statusEnteredTime"] = time()
2158             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2159             return  # database update is called inside finally
2160
2161         except (DbException, LcmException) as e:
2162             self.logger.error(logging_text + "Exit Exception {}".format(e))
2163             exc = e
2164         except asyncio.CancelledError:
2165             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2166             exc = "Operation was cancelled"
2167         except Exception as e:
2168             exc = traceback.format_exc()
2169             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2170         finally:
2171             if exc and db_nslcmop:
2172                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2173                     "FAILED {}: {}".format(step, exc)
2174                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2175                 db_nslcmop_update["statusEnteredTime"] = time()
2176             try:
2177                 if db_nslcmop_update:
2178                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2179                 if db_nsr:
2180                     db_nsr_update["_admin.nslcmop"] = None
2181                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2182             except DbException as e:
2183                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2184             self.logger.debug(logging_text + "Exit")
2185             if nslcmop_operation_state:
2186                 try:
2187                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2188                                                                "operationState": nslcmop_operation_state},
2189                                             loop=self.loop)
2190                 except Exception as e:
2191                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2192             self.logger.debug(logging_text + "Exit")
2193             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2194             return nslcmop_operation_state, nslcmop_operation_state_detail
2195
2196     async def scale(self, nsr_id, nslcmop_id):
2197
2198         # Try to lock HA task here
2199         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2200         if not task_is_locked_by_me:
2201             return
2202
2203         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2204         self.logger.debug(logging_text + "Enter")
2205         # get all needed from database
2206         db_nsr = None
2207         db_nslcmop = None
2208         db_nslcmop_update = {}
2209         nslcmop_operation_state = None
2210         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2211         exc = None
2212         # in case of error, indicates what part of scale was failed to put nsr at error status
2213         scale_process = None
2214         old_operational_status = ""
2215         old_config_status = ""
2216         vnfr_scaled = False
2217         try:
2218             # wait for any previous tasks in process
2219             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2220
2221             step = "Getting nslcmop from database"
2222             self.logger.debug(step + " after having waited for previous tasks to be completed")
2223             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2224             step = "Getting nsr from database"
2225             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2226
2227             old_operational_status = db_nsr["operational-status"]
2228             old_config_status = db_nsr["config-status"]
2229             step = "Parsing scaling parameters"
2230             # self.logger.debug(step)
2231             db_nsr_update["operational-status"] = "scaling"
2232             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2233             nsr_deployed = db_nsr["_admin"].get("deployed")
2234             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2235             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2236             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2237             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2238             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2239
2240             # for backward compatibility
2241             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2242                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2243                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2244                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2245
2246             step = "Getting vnfr from database"
2247             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2248             step = "Getting vnfd from database"
2249             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2250
2251             step = "Getting scaling-group-descriptor"
2252             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2253                 if scaling_descriptor["name"] == scaling_group:
2254                     break
2255             else:
2256                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2257                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2258
2259             # cooldown_time = 0
2260             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2261             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2262             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2263             #         break
2264
2265             # TODO check if ns is in a proper status
2266             step = "Sending scale order to VIM"
2267             nb_scale_op = 0
2268             if not db_nsr["_admin"].get("scaling-group"):
2269                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2270                 admin_scale_index = 0
2271             else:
2272                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2273                     if admin_scale_info["name"] == scaling_group:
2274                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2275                         break
2276                 else:  # not found, set index one plus last element and add new entry with the name
2277                     admin_scale_index += 1
2278                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2279             RO_scaling_info = []
2280             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2281             if scaling_type == "SCALE_OUT":
2282                 # count if max-instance-count is reached
2283                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2284                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2285                 if nb_scale_op >= max_instance_count:
2286                     raise LcmException("reached the limit of {} (max-instance-count) "
2287                                        "scaling-out operations for the "
2288                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2289
2290                 nb_scale_op += 1
2291                 vdu_scaling_info["scaling_direction"] = "OUT"
2292                 vdu_scaling_info["vdu-create"] = {}
2293                 for vdu_scale_info in scaling_descriptor["vdu"]:
2294                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2295                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2296                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2297
2298             elif scaling_type == "SCALE_IN":
2299                 # count if min-instance-count is reached
2300                 min_instance_count = 0
2301                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2302                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2303                 if nb_scale_op <= min_instance_count:
2304                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2305                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2306                 nb_scale_op -= 1
2307                 vdu_scaling_info["scaling_direction"] = "IN"
2308                 vdu_scaling_info["vdu-delete"] = {}
2309                 for vdu_scale_info in scaling_descriptor["vdu"]:
2310                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2311                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2312                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2313
2314             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2315             vdu_create = vdu_scaling_info.get("vdu-create")
2316             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2317             if vdu_scaling_info["scaling_direction"] == "IN":
2318                 for vdur in reversed(db_vnfr["vdur"]):
2319                     if vdu_delete.get(vdur["vdu-id-ref"]):
2320                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2321                         vdu_scaling_info["vdu"].append({
2322                             "name": vdur["name"],
2323                             "vdu_id": vdur["vdu-id-ref"],
2324                             "interface": []
2325                         })
2326                         for interface in vdur["interfaces"]:
2327                             vdu_scaling_info["vdu"][-1]["interface"].append({
2328                                 "name": interface["name"],
2329                                 "ip_address": interface["ip-address"],
2330                                 "mac_address": interface.get("mac-address"),
2331                             })
2332                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2333
2334             # execute primitive service PRE-SCALING
2335             step = "Executing pre-scale vnf-config-primitive"
2336             if scaling_descriptor.get("scaling-config-action"):
2337                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2338                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2339                             and scaling_type == "SCALE_IN":
2340                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2341                         step = db_nslcmop_update["detailed-status"] = \
2342                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2343
2344                         # look for primitive
2345                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2346                             if config_primitive["name"] == vnf_config_primitive:
2347                                 break
2348                         else:
2349                             raise LcmException(
2350                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2351                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2352                                 "primitive".format(scaling_group, config_primitive))
2353
2354                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2355                         if db_vnfr.get("additionalParamsForVnf"):
2356                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2357
2358                         scale_process = "VCA"
2359                         db_nsr_update["config-status"] = "configuring pre-scaling"
2360                         result, result_detail = await self._ns_execute_primitive(
2361                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2362                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2363                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2364                             vnf_config_primitive, result, result_detail))
2365                         if result == "FAILED":
2366                             raise LcmException(result_detail)
2367                         db_nsr_update["config-status"] = old_config_status
2368                         scale_process = None
2369
2370             if RO_scaling_info:
2371                 scale_process = "RO"
2372                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2373                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2374                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2375                 # wait until ready
2376                 RO_nslcmop_id = RO_desc["instance_action_id"]
2377                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2378
2379                 RO_task_done = False
2380                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2381                 detailed_status_old = None
2382                 self.logger.debug(logging_text + step)
2383
2384                 deployment_timeout = 1 * 3600   # One hour
2385                 while deployment_timeout > 0:
2386                     if not RO_task_done:
2387                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2388                                                   extra_item_id=RO_nslcmop_id)
2389                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2390                         if ns_status == "ERROR":
2391                             raise ROclient.ROClientException(ns_status_info)
2392                         elif ns_status == "BUILD":
2393                             detailed_status = step + "; {}".format(ns_status_info)
2394                         elif ns_status == "ACTIVE":
2395                             RO_task_done = True
2396                             step = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
2397                             self.logger.debug(logging_text + step)
2398                         else:
2399                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2400                     else:
2401                         desc = await self.RO.show("ns", RO_nsr_id)
2402                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2403                         if ns_status == "ERROR":
2404                             raise ROclient.ROClientException(ns_status_info)
2405                         elif ns_status == "BUILD":
2406                             detailed_status = step + "; {}".format(ns_status_info)
2407                         elif ns_status == "ACTIVE":
2408                             step = detailed_status = \
2409                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2410                             if not vnfr_scaled:
2411                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2412                                 vnfr_scaled = True
2413                             try:
2414                                 desc = await self.RO.show("ns", RO_nsr_id)
2415                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2416                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2417                                 break
2418                             except LcmExceptionNoMgmtIP:
2419                                 pass
2420                         else:
2421                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2422                     if detailed_status != detailed_status_old:
2423                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2424                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2425
2426                     await asyncio.sleep(5, loop=self.loop)
2427                     deployment_timeout -= 5
2428                 if deployment_timeout <= 0:
2429                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2430
2431                 # update VDU_SCALING_INFO with the obtained ip_addresses
2432                 if vdu_scaling_info["scaling_direction"] == "OUT":
2433                     for vdur in reversed(db_vnfr["vdur"]):
2434                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2435                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2436                             vdu_scaling_info["vdu"].append({
2437                                 "name": vdur["name"],
2438                                 "vdu_id": vdur["vdu-id-ref"],
2439                                 "interface": []
2440                             })
2441                             for interface in vdur["interfaces"]:
2442                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2443                                     "name": interface["name"],
2444                                     "ip_address": interface["ip-address"],
2445                                     "mac_address": interface.get("mac-address"),
2446                                 })
2447                     del vdu_scaling_info["vdu-create"]
2448
2449             scale_process = None
2450             if db_nsr_update:
2451                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2452
2453             # execute primitive service POST-SCALING
2454             step = "Executing post-scale vnf-config-primitive"
2455             if scaling_descriptor.get("scaling-config-action"):
2456                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2457                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2458                             and scaling_type == "SCALE_OUT":
2459                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2460                         step = db_nslcmop_update["detailed-status"] = \
2461                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2462
2463                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2464                         if db_vnfr.get("additionalParamsForVnf"):
2465                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2466
2467                         # look for primitive
2468                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2469                             if config_primitive["name"] == vnf_config_primitive:
2470                                 break
2471                         else:
2472                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2473                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2474                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2475                                                                                                      config_primitive))
2476                         scale_process = "VCA"
2477                         db_nsr_update["config-status"] = "configuring post-scaling"
2478
2479                         result, result_detail = await self._ns_execute_primitive(
2480                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2481                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2482                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2483                             vnf_config_primitive, result, result_detail))
2484                         if result == "FAILED":
2485                             raise LcmException(result_detail)
2486                         db_nsr_update["config-status"] = old_config_status
2487                         scale_process = None
2488
2489             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2490             db_nslcmop_update["statusEnteredTime"] = time()
2491             db_nslcmop_update["detailed-status"] = "done"
2492             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2493             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2494                 else old_operational_status
2495             db_nsr_update["config-status"] = old_config_status
2496             return
2497         except (ROclient.ROClientException, DbException, LcmException) as e:
2498             self.logger.error(logging_text + "Exit Exception {}".format(e))
2499             exc = e
2500         except asyncio.CancelledError:
2501             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2502             exc = "Operation was cancelled"
2503         except Exception as e:
2504             exc = traceback.format_exc()
2505             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2506         finally:
2507             if exc:
2508                 if db_nslcmop:
2509                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2510                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2511                     db_nslcmop_update["statusEnteredTime"] = time()
2512                 if db_nsr:
2513                     db_nsr_update["operational-status"] = old_operational_status
2514                     db_nsr_update["config-status"] = old_config_status
2515                     db_nsr_update["detailed-status"] = ""
2516                     db_nsr_update["_admin.nslcmop"] = None
2517                     if scale_process:
2518                         if "VCA" in scale_process:
2519                             db_nsr_update["config-status"] = "failed"
2520                         if "RO" in scale_process:
2521                             db_nsr_update["operational-status"] = "failed"
2522                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2523                                                                                                      exc)
2524             try:
2525                 if db_nslcmop and db_nslcmop_update:
2526                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2527                 if db_nsr:
2528                     db_nsr_update["_admin.nslcmop"] = None
2529                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2530             except DbException as e:
2531                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2532             if nslcmop_operation_state:
2533                 try:
2534                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2535                                                              "operationState": nslcmop_operation_state},
2536                                             loop=self.loop)
2537                     # if cooldown_time:
2538                     #     await asyncio.sleep(cooldown_time)
2539                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2540                 except Exception as e:
2541                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2542             self.logger.debug(logging_text + "Exit")
2543             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")