osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105
 106     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 107         """
 108         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 109         :param vnfd: input vnfd
 110         :param new_id: overrides vnf id if provided
 111         :param additionalParams: Instantiation params for VNFs provided
 112         :param nsrId: Id of the NSR
 113         :return: copy of vnfd
 114         """
 115         try:
 116             vnfd_RO = deepcopy(vnfd)
 117             # remove unused by RO configuration, monitoring, scaling and internal keys
 118             vnfd_RO.pop("_id", None)
 119             vnfd_RO.pop("_admin", None)
 120             vnfd_RO.pop("vnf-configuration", None)
 121             vnfd_RO.pop("monitoring-param", None)
 122             vnfd_RO.pop("scaling-group-descriptor", None)
 123             if new_id:
 124                 vnfd_RO["id"] = new_id
 125
 126             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 127             for vdu in get_iterable(vnfd_RO, "vdu"):
 128                 cloud_init_file = None
 129                 if vdu.get("cloud-init-file"):
 130                     base_folder = vnfd["_admin"]["storage"]
 131                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 132                                                                    vdu["cloud-init-file"])
 133                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 134                         cloud_init_content = ci_file.read()
 135                     vdu.pop("cloud-init-file", None)
 136                 elif vdu.get("cloud-init"):
 137                     cloud_init_content = vdu["cloud-init"]
 138                 else:
 139                     continue
 140
 141                 env = Environment()
 142                 ast = env.parse(cloud_init_content)
 143                 mandatory_vars = meta.find_undeclared_variables(ast)
 144                 if mandatory_vars:
 145                     for var in mandatory_vars:
 146                         if not additionalParams or var not in additionalParams.keys():
 147                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 148                                                "file, must be provided in the instantiation parameters inside the "
 149                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 150                 template = Template(cloud_init_content)
 151                 cloud_init_content = template.render(additionalParams or {})
 152                 vdu["cloud-init"] = cloud_init_content
 153
 154             return vnfd_RO
 155         except FsException as e:
 156             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 157                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 158         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 159             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 160                                format(vnfd["id"], vdu["id"], e))
 161
 162     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 163         """
 164         Callback both for charm status change and task completion
 165         :param model_name: Charm model name
 166         :param application_name: Charm application name
 167         :param status: Can be
 168             - blocked: The unit needs manual intervention
 169             - maintenance: The unit is actively deploying/configuring
 170             - waiting: The unit is waiting for another charm to be ready
 171             - active: The unit is deployed, configured, and ready
 172             - error: The charm has failed and needs attention.
 173             - terminated: The charm has been destroyed
 174             - removing,
 175             - removed
 176         :param message: detailed message error
 177         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 178             nsr_id:
 179             nslcmop_id:
 180             lcmOperationType: currently "instantiate"
 181             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 182             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 183             n2vc_event: event used to notify instantiation task that some change has been produced
 184         :param task: None for charm status change, or task for completion task callback
 185         :return:
 186         """
 187         try:
 188             nsr_id = n2vc_info["nsr_id"]
 189             deployed = n2vc_info["deployed"]
 190             db_nsr_update = n2vc_info["db_update"]
 191             nslcmop_id = n2vc_info["nslcmop_id"]
 192             ns_operation = n2vc_info["lcmOperationType"]
 193             n2vc_event = n2vc_info["n2vc_event"]
 194             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 195                                                                                     application_name)
 196             for vca_index, vca_deployed in enumerate(deployed):
 197                 if not vca_deployed:
 198                     continue
 199                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 200                     break
 201             else:
 202                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 203                                   format(model_name))
 204                 return
 205             if task:
 206                 if task.cancelled():
 207                     self.logger.debug(logging_text + " task Cancelled")
 208                     vca_deployed['operational-status'] = "error"
 209                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 210                     vca_deployed['detailed-status'] = "Task Cancelled"
 211                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 212
 213                 elif task.done():
 214                     exc = task.exception()
 215                     if exc:
 216                         self.logger.error(logging_text + " task Exception={}".format(exc))
 217                         vca_deployed['operational-status'] = "error"
 218                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 219                         vca_deployed['detailed-status'] = str(exc)
 220                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 221                     else:
 222                         self.logger.debug(logging_text + " task Done")
 223                         # task is Done, but callback is still ongoing. So ignore
 224                         return
 225             elif status:
 226                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 227                 if vca_deployed['operational-status'] == status:
 228                     return  # same status, ignore
 229                 vca_deployed['operational-status'] = status
 230                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 231                 vca_deployed['detailed-status'] = str(message)
 232                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 233             else:
 234                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 235                 return
 236             # wake up instantiate task
 237             n2vc_event.set()
 238         except Exception as e:
 239             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 240
 241     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 242         """
 243         Creates a RO ns descriptor from OSM ns_instantiate params
 244         :param ns_params: OSM instantiate params
 245         :return: The RO ns descriptor
 246         """
 247         vim_2_RO = {}
 248         wim_2_RO = {}
 249         # TODO feature 1417: Check that no instantiation is set over PDU
 250         # check if PDU forces a concrete vim-network-id and add it
 251         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 252
 253         def vim_account_2_RO(vim_account):
 254             if vim_account in vim_2_RO:
 255                 return vim_2_RO[vim_account]
 256
 257             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 258             if db_vim["_admin"]["operationalState"] != "ENABLED":
 259                 raise LcmException("VIM={} is not available. operationalState={}".format(
 260                     vim_account, db_vim["_admin"]["operationalState"]))
 261             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 262             vim_2_RO[vim_account] = RO_vim_id
 263             return RO_vim_id
 264
 265         def wim_account_2_RO(wim_account):
 266             if isinstance(wim_account, str):
 267                 if wim_account in wim_2_RO:
 268                     return wim_2_RO[wim_account]
 269
 270                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 271                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 272                     raise LcmException("WIM={} is not available. operationalState={}".format(
 273                         wim_account, db_wim["_admin"]["operationalState"]))
 274                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 275                 wim_2_RO[wim_account] = RO_wim_id
 276                 return RO_wim_id
 277             else:
 278                 return wim_account
 279
 280         def ip_profile_2_RO(ip_profile):
 281             RO_ip_profile = deepcopy((ip_profile))
 282             if "dns-server" in RO_ip_profile:
 283                 if isinstance(RO_ip_profile["dns-server"], list):
 284                     RO_ip_profile["dns-address"] = []
 285                     for ds in RO_ip_profile.pop("dns-server"):
 286                         RO_ip_profile["dns-address"].append(ds['address'])
 287                 else:
 288                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 289             if RO_ip_profile.get("ip-version") == "ipv4":
 290                 RO_ip_profile["ip-version"] = "IPv4"
 291             if RO_ip_profile.get("ip-version") == "ipv6":
 292                 RO_ip_profile["ip-version"] = "IPv6"
 293             if "dhcp-params" in RO_ip_profile:
 294                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 295             return RO_ip_profile
 296
 297         if not ns_params:
 298             return None
 299         RO_ns_params = {
 300             # "name": ns_params["nsName"],
 301             # "description": ns_params.get("nsDescription"),
 302             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 303             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 304             # "scenario": ns_params["nsdId"],
 305         }
 306         if n2vc_key_list:
 307             for vnfd_ref, vnfd in vnfd_dict.items():
 308                 vdu_needed_access = []
 309                 mgmt_cp = None
 310                 if vnfd.get("vnf-configuration"):
 311                     if vnfd.get("mgmt-interface"):
 312                         if vnfd["mgmt-interface"].get("vdu-id"):
 313                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 314                         elif vnfd["mgmt-interface"].get("cp"):
 315                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 316
 317                 for vdu in vnfd.get("vdu", ()):
 318                     if vdu.get("vdu-configuration"):
 319                         vdu_needed_access.append(vdu["id"])
 320                     elif mgmt_cp:
 321                         for vdu_interface in vdu.get("interface"):
 322                             if vdu_interface.get("external-connection-point-ref") and \
 323                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 324                                 vdu_needed_access.append(vdu["id"])
 325                                 mgmt_cp = None
 326                                 break
 327
 328                 if vdu_needed_access:
 329                     for vnf_member in nsd.get("constituent-vnfd"):
 330                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 331                             continue
 332                         for vdu in vdu_needed_access:
 333                             populate_dict(RO_ns_params,
 334                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 335                                           n2vc_key_list)
 336
 337         if ns_params.get("vduImage"):
 338             RO_ns_params["vduImage"] = ns_params["vduImage"]
 339
 340         if ns_params.get("ssh_keys"):
 341             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 342         for vnf_params in get_iterable(ns_params, "vnf"):
 343             for constituent_vnfd in nsd["constituent-vnfd"]:
 344                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 345                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 346                     break
 347             else:
 348                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 349                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 350             if vnf_params.get("vimAccountId"):
 351                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 352                               vim_account_2_RO(vnf_params["vimAccountId"]))
 353
 354             for vdu_params in get_iterable(vnf_params, "vdu"):
 355                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 356                 if vdu_params.get("volume"):
 357                     for volume_params in vdu_params["volume"]:
 358                         if volume_params.get("vim-volume-id"):
 359                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 360                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 361                                           volume_params["vim-volume-id"])
 362                 if vdu_params.get("interface"):
 363                     for interface_params in vdu_params["interface"]:
 364                         if interface_params.get("ip-address"):
 365                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 366                                                          vdu_params["id"], "interfaces", interface_params["name"],
 367                                                          "ip_address"),
 368                                           interface_params["ip-address"])
 369                         if interface_params.get("mac-address"):
 370                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 371                                                          vdu_params["id"], "interfaces", interface_params["name"],
 372                                                          "mac_address"),
 373                                           interface_params["mac-address"])
 374                         if interface_params.get("floating-ip-required"):
 375                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 376                                                          vdu_params["id"], "interfaces", interface_params["name"],
 377                                                          "floating-ip"),
 378                                           interface_params["floating-ip-required"])
 379
 380             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 381                 if internal_vld_params.get("vim-network-name"):
 382                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 383                                                  internal_vld_params["name"], "vim-network-name"),
 384                                   internal_vld_params["vim-network-name"])
 385                 if internal_vld_params.get("vim-network-id"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-id"),
 388                                   internal_vld_params["vim-network-id"])
 389                 if internal_vld_params.get("ip-profile"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "ip-profile"),
 392                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 393
 394                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 395                     # look for interface
 396                     iface_found = False
 397                     for vdu_descriptor in vnf_descriptor["vdu"]:
 398                         for vdu_interface in vdu_descriptor["interface"]:
 399                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 400                                 if icp_params.get("ip-address"):
 401                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 402                                                                  vdu_descriptor["id"], "interfaces",
 403                                                                  vdu_interface["name"], "ip_address"),
 404                                                   icp_params["ip-address"])
 405
 406                                 if icp_params.get("mac-address"):
 407                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 408                                                                  vdu_descriptor["id"], "interfaces",
 409                                                                  vdu_interface["name"], "mac_address"),
 410                                                   icp_params["mac-address"])
 411                                 iface_found = True
 412                                 break
 413                         if iface_found:
 414                             break
 415                     else:
 416                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 417                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 418                                            "connection-point".format(vnf_params["member-vnf-index"],
 419                                                                      icp_params["id-ref"]))
 420
 421         for vld_params in get_iterable(ns_params, "vld"):
 422             if "ip-profile" in vld_params:
 423                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 424                               ip_profile_2_RO(vld_params["ip-profile"]))
 425
 426             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 428                               wim_account_2_RO(vld_params["wimAccountId"])),
 429             if vld_params.get("vim-network-name"):
 430                 RO_vld_sites = []
 431                 if isinstance(vld_params["vim-network-name"], dict):
 432                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 433                         RO_vld_sites.append({
 434                             "netmap-use": vim_net,
 435                             "datacenter": vim_account_2_RO(vim_account)
 436                         })
 437                 else:  # isinstance str
 438                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 439                 if RO_vld_sites:
 440                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 441             if vld_params.get("vim-network-id"):
 442                 RO_vld_sites = []
 443                 if isinstance(vld_params["vim-network-id"], dict):
 444                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 445                         RO_vld_sites.append({
 446                             "netmap-use": vim_net,
 447                             "datacenter": vim_account_2_RO(vim_account)
 448                         })
 449                 else:  # isinstance str
 450                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 451                 if RO_vld_sites:
 452                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 453             if vld_params.get("ns-net"):
 454                 if isinstance(vld_params["ns-net"], dict):
 455                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 456                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 457                 if RO_vld_ns_net:
 458                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 459             if "vnfd-connection-point-ref" in vld_params:
 460                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 461                     # look for interface
 462                     for constituent_vnfd in nsd["constituent-vnfd"]:
 463                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 464                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 465                             break
 466                     else:
 467                         raise LcmException(
 468                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 469                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 470                     match_cp = False
 471                     for vdu_descriptor in vnf_descriptor["vdu"]:
 472                         for interface_descriptor in vdu_descriptor["interface"]:
 473                             if interface_descriptor.get("external-connection-point-ref") == \
 474                                     cp_params["vnfd-connection-point-ref"]:
 475                                 match_cp = True
 476                                 break
 477                         if match_cp:
 478                             break
 479                     else:
 480                         raise LcmException(
 481                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 482                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 483                                 cp_params["member-vnf-index-ref"],
 484                                 cp_params["vnfd-connection-point-ref"],
 485                                 vnf_descriptor["id"]))
 486                     if cp_params.get("ip-address"):
 487                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 488                                                      vdu_descriptor["id"], "interfaces",
 489                                                      interface_descriptor["name"], "ip_address"),
 490                                       cp_params["ip-address"])
 491                     if cp_params.get("mac-address"):
 492                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 493                                                      vdu_descriptor["id"], "interfaces",
 494                                                      interface_descriptor["name"], "mac_address"),
 495                                       cp_params["mac-address"])
 496         return RO_ns_params
 497
 498     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 499         # make a copy to do not change
 500         vdu_create = copy(vdu_create)
 501         vdu_delete = copy(vdu_delete)
 502
 503         vdurs = db_vnfr.get("vdur")
 504         if vdurs is None:
 505             vdurs = []
 506         vdu_index = len(vdurs)
 507         while vdu_index:
 508             vdu_index -= 1
 509             vdur = vdurs[vdu_index]
 510             if vdur.get("pdu-type"):
 511                 continue
 512             vdu_id_ref = vdur["vdu-id-ref"]
 513             if vdu_create and vdu_create.get(vdu_id_ref):
 514                 for index in range(0, vdu_create[vdu_id_ref]):
 515                     vdur = deepcopy(vdur)
 516                     vdur["_id"] = str(uuid4())
 517                     vdur["count-index"] += 1
 518                     vdurs.insert(vdu_index+1+index, vdur)
 519                 del vdu_create[vdu_id_ref]
 520             if vdu_delete and vdu_delete.get(vdu_id_ref):
 521                 del vdurs[vdu_index]
 522                 vdu_delete[vdu_id_ref] -= 1
 523                 if not vdu_delete[vdu_id_ref]:
 524                     del vdu_delete[vdu_id_ref]
 525         # check all operations are done
 526         if vdu_create or vdu_delete:
 527             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 528                 vdu_create))
 529         if vdu_delete:
 530             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 531                 vdu_delete))
 532
 533         vnfr_update = {"vdur": vdurs}
 534         db_vnfr["vdur"] = vdurs
 535         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 536
 537     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 538         """
 539         Updates database nsr with the RO info for the created vld
 540         :param ns_update_nsr: dictionary to be filled with the updated info
 541         :param db_nsr: content of db_nsr. This is also modified
 542         :param nsr_desc_RO: nsr descriptor from RO
 543         :return: Nothing, LcmException is raised on errors
 544         """
 545
 546         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 547             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 548                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 549                     continue
 550                 vld["vim-id"] = net_RO.get("vim_net_id")
 551                 vld["name"] = net_RO.get("vim_name")
 552                 vld["status"] = net_RO.get("status")
 553                 vld["status-detailed"] = net_RO.get("error_msg")
 554                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 555                 break
 556             else:
 557                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 558
 559     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 560         """
 561         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 562         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 563         :param nsr_desc_RO: nsr descriptor from RO
 564         :return: Nothing, LcmException is raised on errors
 565         """
 566         for vnf_index, db_vnfr in db_vnfrs.items():
 567             for vnf_RO in nsr_desc_RO["vnfs"]:
 568                 if vnf_RO["member_vnf_index"] != vnf_index:
 569                     continue
 570                 vnfr_update = {}
 571                 if vnf_RO.get("ip_address"):
 572                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 573                 elif not db_vnfr.get("ip-address"):
 574                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 575
 576                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 577                     vdur_RO_count_index = 0
 578                     if vdur.get("pdu-type"):
 579                         continue
 580                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 581                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 582                             continue
 583                         if vdur["count-index"] != vdur_RO_count_index:
 584                             vdur_RO_count_index += 1
 585                             continue
 586                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 587                         if vdur_RO.get("ip_address"):
 588                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 589                         else:
 590                             vdur["ip-address"] = None
 591                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 592                         vdur["name"] = vdur_RO.get("vim_name")
 593                         vdur["status"] = vdur_RO.get("status")
 594                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 595                         for ifacer in get_iterable(vdur, "interfaces"):
 596                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 597                                 if ifacer["name"] == interface_RO.get("internal_name"):
 598                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 599                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 600                                     break
 601                             else:
 602                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 603                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 604                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 605                         break
 606                     else:
 607                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 608                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 609
 610                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 611                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 612                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 613                             continue
 614                         vld["vim-id"] = net_RO.get("vim_net_id")
 615                         vld["name"] = net_RO.get("vim_name")
 616                         vld["status"] = net_RO.get("status")
 617                         vld["status-detailed"] = net_RO.get("error_msg")
 618                         vnfr_update["vld.{}".format(vld_index)] = vld
 619                         break
 620                     else:
 621                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 622                             vnf_index, vld["id"]))
 623
 624                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 625                 break
 626
 627             else:
 628                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 629
 630     async def instantiate(self, nsr_id, nslcmop_id):
 631         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 632         self.logger.debug(logging_text + "Enter")
 633         # get all needed from database
 634         start_deploy = time()
 635         db_nsr = None
 636         db_nslcmop = None
 637         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 638         db_nslcmop_update = {}
 639         nslcmop_operation_state = None
 640         db_vnfrs = {}
 641         RO_descriptor_number = 0   # number of descriptors created at RO
 642         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 643         n2vc_info = {}
 644         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 645         exc = None
 646         try:
 647             step = "Getting nslcmop={} from db".format(nslcmop_id)
 648             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 649             step = "Getting nsr={} from db".format(nsr_id)
 650             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 651             ns_params = db_nslcmop.get("operationParams")
 652             nsd = db_nsr["nsd"]
 653             nsr_name = db_nsr["name"]   # TODO short-name??
 654
 655             # look if previous tasks in process
 656             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
 657             if task_dependency:
 658                 step = db_nslcmop_update["detailed-status"] = \
 659                     "Waiting for related tasks to be completed: {}".format(task_name)
 660                 self.logger.debug(logging_text + step)
 661                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 662                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 663                 if pending:
 664                     raise LcmException("Timeout waiting related tasks to be completed")
 665
 666             step = "Getting vnfrs from db"
 667             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 668             db_vnfds_ref = {}
 669             db_vnfds = {}
 670             db_vnfds_index = {}
 671             for vnfr in db_vnfrs_list:
 672                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 673                 vnfd_id = vnfr["vnfd-id"]
 674                 vnfd_ref = vnfr["vnfd-ref"]
 675                 if vnfd_id not in db_vnfds:
 676                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 677                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 678                     db_vnfds_ref[vnfd_ref] = vnfd
 679                     db_vnfds[vnfd_id] = vnfd
 680                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 681
 682             # Get or generates the _admin.deployed,VCA list
 683             vca_deployed_list = None
 684             vca_model_name = None
 685             if db_nsr["_admin"].get("deployed"):
 686                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 687                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 688             if vca_deployed_list is None:
 689                 vca_deployed_list = []
 690                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 691                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 692             elif isinstance(vca_deployed_list, dict):
 693                 # maintain backward compatibility. Change a dict to list at database
 694                 vca_deployed_list = list(vca_deployed_list.values())
 695                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 696                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 697
 698             db_nsr_update["detailed-status"] = "creating"
 699             db_nsr_update["operational-status"] = "init"
 700             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 701                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 702                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 703                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 704
 705             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 706             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 707             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 708
 709             # Deploy charms
 710             # The parameters we'll need to deploy a charm
 711             number_to_configure = 0
 712
 713             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 714                 """An inner function to deploy the charm from either ns, vnf or vdu
 715                 For ns both vnf_index and vdu_id are None.
 716                 For vnf only vdu_id is None
 717                 For vdu both vnf_index and vdu_id contain a value
 718                 """
 719                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 720                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 721
 722                 machine_spec = {}
 723                 if native_charm:
 724                     machine_spec["username"] = charm_params.get("username"),
 725                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 726
 727                 # Note: The charm needs to exist on disk at the location
 728                 # specified by charm_path.
 729                 descriptor = vnfd if vnf_index else nsd
 730                 base_folder = descriptor["_admin"]["storage"]
 731                 storage_params = self.fs.get_params()
 732                 charm_path = "{}{}/{}/charms/{}".format(
 733                     storage_params["path"],
 734                     base_folder["folder"],
 735                     base_folder["pkg-dir"],
 736                     proxy_charm
 737                 )
 738
 739                 # ns_name will be ignored in the current version of N2VC
 740                 # but will be implemented for the next point release.
 741                 model_name = nsr_id
 742                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 743                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 744                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 745
 746                 vca_index = len(vca_deployed_list)
 747                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 748                 # 26*26 charm in the same NS
 749                 application_name = application_name[0:48]
 750                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 751                 vca_deployed_ = {
 752                     "member-vnf-index": vnf_index,
 753                     "vdu_id": vdu_id,
 754                     "model": model_name,
 755                     "application": application_name,
 756                     "operational-status": "init",
 757                     "detailed-status": "",
 758                     "step": "initial-deploy",
 759                     "vnfd_id": vnfd_id,
 760                     "vdu_name": vdu_name,
 761                     "vdu_count_index": vdu_count_index,
 762                 }
 763                 vca_deployed_list.append(vca_deployed_)
 764                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 765                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 766
 767                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 768                                                                                                 proxy_charm))
 769                 if not n2vc_info:
 770                     n2vc_info["nsr_id"] = nsr_id
 771                     n2vc_info["nslcmop_id"] = nslcmop_id
 772                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 773                     n2vc_info["lcmOperationType"] = "instantiate"
 774                     n2vc_info["deployed"] = vca_deployed_list
 775                     n2vc_info["db_update"] = db_nsr_update
 776                 task = asyncio.ensure_future(
 777                     self.n2vc.DeployCharms(
 778                         model_name,          # The network service name
 779                         application_name,    # The application name
 780                         descriptor,          # The vnf/nsd descriptor
 781                         charm_path,          # Path to charm
 782                         charm_params,        # Runtime params, like mgmt ip
 783                         machine_spec,        # for native charms only
 784                         self.n2vc_callback,  # Callback for status changes
 785                         n2vc_info,           # Callback parameter
 786                         None,                # Callback parameter (task)
 787                     )
 788                 )
 789                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 790                                                          n2vc_info))
 791                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 792
 793             step = "Looking for needed vnfd to configure with proxy charm"
 794             self.logger.debug(logging_text + step)
 795
 796             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 797                 vnfd_id = c_vnf["vnfd-id-ref"]
 798                 vnf_index = str(c_vnf["member-vnf-index"])
 799                 vnfd = db_vnfds_ref[vnfd_id]
 800
 801                 # Get additional parameters
 802                 vnfr_params = {}
 803                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 804                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 805                 for k, v in vnfr_params.items():
 806                     if isinstance(v, str) and v.startswith("!!yaml "):
 807                         vnfr_params[k] = yaml.safe_load(v[7:])
 808
 809                 step = "deploying proxy charms for configuration"
 810                 # Check if this VNF has a charm configuration
 811                 vnf_config = vnfd.get("vnf-configuration")
 812                 if vnf_config and vnf_config.get("juju"):
 813                     proxy_charm = vnf_config["juju"]["charm"]
 814                     if vnf_config["juju"].get("proxy") is False:
 815                         # native_charm, will be deployed after VM. Skip
 816                         proxy_charm = None
 817
 818                     if proxy_charm:
 819                         if not vca_model_name:
 820                             step = "creating VCA model name '{}'".format(nsr_id)
 821                             self.logger.debug(logging_text + step)
 822                             await self.n2vc.CreateNetworkService(nsr_id)
 823                             vca_model_name = nsr_id
 824                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 825                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 826                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 827                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 828                         charm_params = {
 829                             "user_values": vnfr_params,
 830                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 831                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 832                         }
 833
 834                         # Login to the VCA. If there are multiple calls to login(),
 835                         # subsequent calls will be a nop and return immediately.
 836                         await self.n2vc.login()
 837
 838                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 839                         number_to_configure += 1
 840
 841                 # Deploy charms for each VDU that supports one.
 842                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 843                     vdu_config = vdu.get('vdu-configuration')
 844                     proxy_charm = None
 845
 846                     if vdu_config and vdu_config.get("juju"):
 847                         proxy_charm = vdu_config["juju"]["charm"]
 848                         if vdu_config["juju"].get("proxy") is False:
 849                             # native_charm, will be deployed after VM. Skip
 850                             proxy_charm = None
 851
 852                         if proxy_charm:
 853                             if not vca_model_name:
 854                                 step = "creating VCA model name"
 855                                 await self.n2vc.CreateNetworkService(nsr_id)
 856                                 vca_model_name = nsr_id
 857                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 858                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 859                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 860                                                                                                           vdu["id"])
 861                             await self.n2vc.login()
 862                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 863                             # TODO for the moment only first vdu_id contains a charm deployed
 864                             if vdur["vdu-id-ref"] != vdu["id"]:
 865                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 866                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 867                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 868                             charm_params = {
 869                                 "user_values": vnfr_params,
 870                                 "rw_mgmt_ip": vdur["ip-address"],
 871                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 872                             }
 873                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 874                                          charm_params, n2vc_info)
 875                             number_to_configure += 1
 876
 877             # Check if this NS has a charm configuration
 878
 879             ns_config = nsd.get("ns-configuration")
 880             if ns_config and ns_config.get("juju"):
 881                 proxy_charm = ns_config["juju"]["charm"]
 882                 if ns_config["juju"].get("proxy") is False:
 883                     # native_charm, will be deployed after VM. Skip
 884                     proxy_charm = None
 885
 886                 if proxy_charm:
 887                     step = "deploying proxy charm to configure ns"
 888                     # TODO is NS magmt IP address needed?
 889
 890                     # Get additional parameters
 891                     additional_params = {}
 892                     if db_nsr.get("additionalParamsForNs"):
 893                         additional_params = db_nsr["additionalParamsForNs"].copy()
 894                     for k, v in additional_params.items():
 895                         if isinstance(v, str) and v.startswith("!!yaml "):
 896                             additional_params[k] = yaml.safe_load(v[7:])
 897
 898                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 899                     charm_params = {
 900                         "user_values": additional_params,
 901                         # "rw_mgmt_ip": db_nsr["ip-address"],
 902                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 903                     }
 904
 905                     # Login to the VCA. If there are multiple calls to login(),
 906                     # subsequent calls will be a nop and return immediately.
 907                     await self.n2vc.login()
 908                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 909                     number_to_configure += 1
 910
 911             db_nsr_update["operational-status"] = "running"
 912
 913             # Wait until all charms has reached blocked or active status
 914             step = "waiting proxy charms to be ready"
 915             if number_to_configure:
 916                 # wait until all charms are configured.
 917                 # steps are:
 918                 #       initial-deploy
 919                 #       get-ssh-public-key
 920                 #       generate-ssh-key
 921                 #       retry-get-ssh-public-key
 922                 #       ssh-public-key-obtained
 923                 while time() <= start_deploy + self.total_deploy_timeout:
 924                     if db_nsr_update:
 925                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 926                     if db_nslcmop_update:
 927                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 928
 929                     all_active = True
 930                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 931                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 932                         if vca_deployed["step"] == "initial-deploy":
 933                             if vca_deployed["operational-status"] in ("active", "blocked"):
 934                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 935                                        .format(vca_deployed["member-vnf-index"],
 936                                                vca_deployed["vdu_id"])
 937                                 self.logger.debug(logging_text + step)
 938                                 primitive_id = await self.n2vc.ExecutePrimitive(
 939                                     vca_deployed["model"],
 940                                     vca_deployed["application"],
 941                                     "get-ssh-public-key",
 942                                     None,
 943                                 )
 944                                 vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 945                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 946                                     primitive_id
 947                                 db_nsr_update[database_entry + "operational-status"] =\
 948                                     vca_deployed["operational-status"]
 949                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 950                             primitive_id = vca_deployed["primitive_id"]
 951                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 952                                                                                   primitive_id)
 953                             if primitive_status in ("completed", "failed"):
 954                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 955                                                                                       primitive_id)
 956                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 957                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 958                                         primitive_result.get("pubkey"):
 959                                     ssh_public_key = primitive_result.get("pubkey")
 960                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 961                                         "ssh-public-key-obtained"
 962                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 963                                         ssh_public_key
 964                                     n2vc_key_list.append(ssh_public_key)
 965                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 966                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 967                                     self.logger.debug(logging_text + step)
 968                                 else:  # primitive_status == "failed":
 969                                     if vca_deployed["step"] == "get-ssh-public-key":
 970                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 971                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 972                                                                      vca_deployed["vdu_id"])
 973                                         self.logger.debug(logging_text + step)
 974                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 975                                             "generate-ssh-key"
 976                                         primitive_id = await self.n2vc.ExecutePrimitive(
 977                                             vca_deployed["model"],
 978                                             vca_deployed["application"],
 979                                             "generate-ssh-key",
 980                                             None,
 981                                         )
 982                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 983                                             primitive_id
 984                                     else:  # failed for second time
 985                                         raise LcmException(
 986                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 987
 988                         elif vca_deployed["step"] == "generate-ssh-key":
 989                             primitive_id = vca_deployed["primitive_id"]
 990                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 991                                                                                   primitive_id)
 992                             if primitive_status in ("completed", "failed"):
 993                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 994                                                                                       primitive_id)
 995                                 vca_deployed["primitive_id"] = db_nsr_update[
 996                                     database_entry + "primitive_id"] = None
 997                                 if primitive_status == "completed":
 998                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
 999                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1000                                                               vca_deployed["vdu_id"])
1001                                     self.logger.debug(logging_text + step)
1002                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1003                                         "retry-get-ssh-public-key"
1004                                     primitive_id = await self.n2vc.ExecutePrimitive(
1005                                         vca_deployed["model"],
1006                                         vca_deployed["application"],
1007                                         "get-ssh-public-key",
1008                                         None,
1009                                     )
1010                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1011                                         primitive_id
1012
1013                                 else:  # primitive_status == "failed":
1014                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1015                                                        .format(primitive_result))
1016
1017                         if vca_deployed["step"] != "ssh-public-key-obtained":
1018                             all_active = False
1019
1020                     if all_active:
1021                         break
1022                     await asyncio.sleep(5)
1023                 else:   # total_deploy_timeout
1024                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1025                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1026
1027             # deploy RO
1028             RO = ROclient.ROClient(self.loop, **self.ro_config)
1029             # get vnfds, instantiate at RO
1030             for c_vnf in nsd.get("constituent-vnfd", ()):
1031                 member_vnf_index = c_vnf["member-vnf-index"]
1032                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1033                 vnfd_ref = vnfd["id"]
1034                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1035                     vnfd_ref, member_vnf_index)
1036                 # self.logger.debug(logging_text + step)
1037                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1038                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1039                 RO_descriptor_number += 1
1040
1041                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1042                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1043                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1044                         break
1045                 else:
1046                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1047                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1048
1049                 # look if present
1050                 RO_update = {"member-vnf-index": member_vnf_index}
1051                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1052                 if vnfd_list:
1053                     RO_update["id"] = vnfd_list[0]["uuid"]
1054                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1055                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1056                 else:
1057                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1058                                            get("additionalParamsForVnf"), nsr_id)
1059                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
1060                     RO_update["id"] = desc["uuid"]
1061                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1062                         vnfd_ref, member_vnf_index, desc["uuid"]))
1063                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1064                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1065                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1066
1067             # create nsd at RO
1068             nsd_ref = nsd["id"]
1069             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1070             # self.logger.debug(logging_text + step)
1071
1072             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1073             RO_descriptor_number += 1
1074             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1075             if nsd_list:
1076                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1077                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1078                     nsd_ref, RO_nsd_uuid))
1079             else:
1080                 nsd_RO = deepcopy(nsd)
1081                 nsd_RO["id"] = RO_osm_nsd_id
1082                 nsd_RO.pop("_id", None)
1083                 nsd_RO.pop("_admin", None)
1084                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1085                     member_vnf_index = c_vnf["member-vnf-index"]
1086                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1087                 for c_vld in nsd_RO.get("vld", ()):
1088                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1089                         member_vnf_index = cp["member-vnf-index-ref"]
1090                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1091
1092                 desc = await RO.create("nsd", descriptor=nsd_RO)
1093                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1094                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1095                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1096             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1097
1098             # Crate ns at RO
1099             # if present use it unless in error status
1100             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1101             if RO_nsr_id:
1102                 try:
1103                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1104                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1105                     desc = await RO.show("ns", RO_nsr_id)
1106                 except ROclient.ROClientException as e:
1107                     if e.http_code != HTTPStatus.NOT_FOUND:
1108                         raise
1109                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1110                 if RO_nsr_id:
1111                     ns_status, ns_status_info = RO.check_ns_status(desc)
1112                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1113                     if ns_status == "ERROR":
1114                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1115                         self.logger.debug(logging_text + step)
1116                         await RO.delete("ns", RO_nsr_id)
1117                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1118             if not RO_nsr_id:
1119                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1120                 # self.logger.debug(logging_text + step)
1121
1122                 # check if VIM is creating and wait  look if previous tasks in process
1123                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1124                 if task_dependency:
1125                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1126                     self.logger.debug(logging_text + step)
1127                     await asyncio.wait(task_dependency, timeout=3600)
1128                 if ns_params.get("vnf"):
1129                     for vnf in ns_params["vnf"]:
1130                         if "vimAccountId" in vnf:
1131                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1132                                                                                         vnf["vimAccountId"])
1133                         if task_dependency:
1134                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1135                             self.logger.debug(logging_text + step)
1136                             await asyncio.wait(task_dependency, timeout=3600)
1137
1138                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1139
1140                 # feature 1429. Add n2vc public key to needed VMs
1141                 n2vc_key = await self.n2vc.GetPublicKey()
1142                 n2vc_key_list.append(n2vc_key)
1143                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1144
1145                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
1146                 desc = await RO.create("ns", descriptor=RO_ns_params,
1147                                        name=db_nsr["name"],
1148                                        scenario=RO_nsd_uuid)
1149                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1150                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1151                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1152                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1153             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1154
1155             # wait until NS is ready
1156             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1157             detailed_status_old = None
1158             self.logger.debug(logging_text + step)
1159
1160             while time() <= start_deploy + self.total_deploy_timeout:
1161                 desc = await RO.show("ns", RO_nsr_id)
1162                 ns_status, ns_status_info = RO.check_ns_status(desc)
1163                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1164                 if ns_status == "ERROR":
1165                     raise ROclient.ROClientException(ns_status_info)
1166                 elif ns_status == "BUILD":
1167                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1168                 elif ns_status == "ACTIVE":
1169                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1170                     try:
1171                         self.ns_update_vnfr(db_vnfrs, desc)
1172                         break
1173                     except LcmExceptionNoMgmtIP:
1174                         pass
1175                 else:
1176                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1177                 if detailed_status != detailed_status_old:
1178                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1179                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1180                 await asyncio.sleep(5, loop=self.loop)
1181             else:   # total_deploy_timeout
1182                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1183
1184             step = "Updating NSR"
1185             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1186
1187             db_nsr_update["operational-status"] = "running"
1188             db_nsr["detailed-status"] = "Configuring vnfr"
1189             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1190
1191             # Configure proxy charms once VMs are up
1192             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1193                 vnf_index = vca_deployed.get("member-vnf-index")
1194                 vdu_id = vca_deployed.get("vdu_id")
1195                 vdu_name = None
1196                 vdu_count_index = None
1197
1198                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1199                                                                                                            vdu_id)
1200                 add_params = {}
1201                 initial_config_primitive_list = []
1202                 if vnf_index:
1203                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1204                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1205                     vnfd = db_vnfds_index[vnf_index]
1206
1207                     if vdu_id:
1208                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1209                             if vdu["id"] == vdu_id:
1210                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1211                                     'initial-config-primitive', [])
1212                                 break
1213                         else:
1214                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1215                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1216                         # TODO for the moment only first vdu_id contains a charm deployed
1217                         if vdur["vdu-id-ref"] != vdu["id"]:
1218                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1219                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1220                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1221                     else:
1222                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1223                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1224                 else:
1225                     if db_nsr.get("additionalParamsForNs"):
1226                         add_params = db_nsr["additionalParamsForNs"].copy()
1227                     for k, v in add_params.items():
1228                         if isinstance(v, str) and v.startswith("!!yaml "):
1229                             add_params[k] = yaml.safe_load(v[7:])
1230                     add_params["rw_mgmt_ip"] = None
1231                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1232
1233                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1234                 initial_config_primitive_list = initial_config_primitive_list.copy()
1235                 if initial_config_primitive_list and vnf_index:
1236                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1237
1238                 for initial_config_primitive in initial_config_primitive_list:
1239                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1240                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1241                                       .format(initial_config_primitive["name"], primitive_params_))
1242                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1243                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1244                         initial_config_primitive["name"],
1245                         primitive_params_,
1246                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1247                         retries_interval=30)
1248                     if primitive_result != "COMPLETED":
1249                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1250                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1251                                                    vca_deployed["vdu_id"], primitive_detail))
1252
1253             # Deploy native charms
1254             step = "Looking for needed vnfd to configure with native charm"
1255             self.logger.debug(logging_text + step)
1256
1257             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1258                 vnfd_id = c_vnf["vnfd-id-ref"]
1259                 vnf_index = str(c_vnf["member-vnf-index"])
1260                 vnfd = db_vnfds_ref[vnfd_id]
1261
1262                 # Get additional parameters
1263                 vnfr_params = {}
1264                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1265                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1266                 for k, v in vnfr_params.items():
1267                     if isinstance(v, str) and v.startswith("!!yaml "):
1268                         vnfr_params[k] = yaml.safe_load(v[7:])
1269
1270                 # Check if this VNF has a charm configuration
1271                 vnf_config = vnfd.get("vnf-configuration")
1272                 if vnf_config and vnf_config.get("juju"):
1273                     native_charm = vnf_config["juju"].get("proxy") is False
1274
1275                     if native_charm:
1276                         if not vca_model_name:
1277                             step = "creating VCA model name '{}'".format(nsr_id)
1278                             self.logger.debug(logging_text + step)
1279                             await self.n2vc.CreateNetworkService(nsr_id)
1280                             vca_model_name = nsr_id
1281                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1282                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1283                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1284                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1285                         charm_params = {
1286                             "user_values": vnfr_params,
1287                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1288                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1289                         }
1290
1291                         # get username
1292                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1293                         #  merged. Meanwhile let's get username from initial-config-primitive
1294                         if vnf_config.get("initial-config-primitive"):
1295                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1296                                 if param["name"] == "ssh-username":
1297                                     charm_params["username"] = param["value"]
1298                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1299                             if vnf_config["config-access"]["ssh-access"].get("required"):
1300                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1301
1302                         # Login to the VCA. If there are multiple calls to login(),
1303                         # subsequent calls will be a nop and return immediately.
1304                         await self.n2vc.login()
1305
1306                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1307                         number_to_configure += 1
1308
1309                 # Deploy charms for each VDU that supports one.
1310                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1311                     vdu_config = vdu.get('vdu-configuration')
1312                     native_charm = False
1313
1314                     if vdu_config and vdu_config.get("juju"):
1315                         native_charm = vdu_config["juju"].get("proxy") is False
1316
1317                         if native_charm:
1318                             if not vca_model_name:
1319                                 step = "creating VCA model name"
1320                                 await self.n2vc.CreateNetworkService(nsr_id)
1321                                 vca_model_name = nsr_id
1322                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1323                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1324                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1325                                                                                                      vdu["id"])
1326                             await self.n2vc.login()
1327                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1328                             # TODO for the moment only first vdu_id contains a charm deployed
1329                             if vdur["vdu-id-ref"] != vdu["id"]:
1330                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1331                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1332                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1333                             charm_params = {
1334                                 "user_values": vnfr_params,
1335                                 "rw_mgmt_ip": vdur["ip-address"],
1336                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1337                             }
1338
1339                             # get username
1340                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1341                             #  merged. Meanwhile let's get username from initial-config-primitive
1342                             if vdu_config.get("initial-config-primitive"):
1343                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1344                                     if param["name"] == "ssh-username":
1345                                         charm_params["username"] = param["value"]
1346                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1347                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1348                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1349                                         "default-user")
1350
1351                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1352                                          charm_params, n2vc_info, native_charm)
1353                             number_to_configure += 1
1354
1355             # Check if this NS has a charm configuration
1356
1357             ns_config = nsd.get("ns-configuration")
1358             if ns_config and ns_config.get("juju"):
1359                 native_charm = ns_config["juju"].get("proxy") is False
1360
1361                 if native_charm:
1362                     step = "deploying native charm to configure ns"
1363                     # TODO is NS magmt IP address needed?
1364
1365                     # Get additional parameters
1366                     additional_params = {}
1367                     if db_nsr.get("additionalParamsForNs"):
1368                         additional_params = db_nsr["additionalParamsForNs"].copy()
1369                     for k, v in additional_params.items():
1370                         if isinstance(v, str) and v.startswith("!!yaml "):
1371                             additional_params[k] = yaml.safe_load(v[7:])
1372
1373                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1374                     charm_params = {
1375                         "user_values": additional_params,
1376                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1377                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1378                     }
1379
1380                     # get username
1381                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1382                     #  merged. Meanwhile let's get username from initial-config-primitive
1383                     if ns_config.get("initial-config-primitive"):
1384                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1385                             if param["name"] == "ssh-username":
1386                                 charm_params["username"] = param["value"]
1387                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1388                         if ns_config["config-access"]["ssh-access"].get("required"):
1389                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1390
1391                     # Login to the VCA. If there are multiple calls to login(),
1392                     # subsequent calls will be a nop and return immediately.
1393                     await self.n2vc.login()
1394                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1395                     number_to_configure += 1
1396
1397             # waiting all charms are ok
1398             configuration_failed = False
1399             if number_to_configure:
1400                 step = "Waiting all charms are active"
1401                 old_status = "configuring: init: {}".format(number_to_configure)
1402                 db_nsr_update["config-status"] = old_status
1403                 db_nsr_update["detailed-status"] = old_status
1404                 db_nslcmop_update["detailed-status"] = old_status
1405
1406                 # wait until all are configured.
1407                 while time() <= start_deploy + self.total_deploy_timeout:
1408                     if db_nsr_update:
1409                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1410                     if db_nslcmop_update:
1411                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1412                     # TODO add a fake task that set n2vc_event after some time
1413                     await n2vc_info["n2vc_event"].wait()
1414                     n2vc_info["n2vc_event"].clear()
1415                     all_active = True
1416                     status_map = {}
1417                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1418                     now = time()
1419                     for vca_deployed in vca_deployed_list:
1420                         vca_status = vca_deployed["operational-status"]
1421                         if vca_status not in status_map:
1422                             # Initialize it
1423                             status_map[vca_status] = 0
1424                         status_map[vca_status] += 1
1425
1426                         if vca_status == "active":
1427                             vca_deployed.pop("time_first_error", None)
1428                             vca_deployed.pop("status_first_error", None)
1429                             continue
1430
1431                         all_active = False
1432                         if vca_status in ("error", "blocked"):
1433                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1434                             # if not first time in this status error
1435                             if not vca_deployed.get("time_first_error"):
1436                                 vca_deployed["time_first_error"] = now
1437                                 continue
1438                         if vca_deployed.get("time_first_error") and \
1439                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1440                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1441                                                    .format(vca_deployed["member-vnf-index"],
1442                                                            vca_deployed["vdu_id"], vca_status,
1443                                                            vca_deployed["detailed-status-error"]))
1444
1445                     if all_active:
1446                         break
1447                     elif n2vc_error_text:
1448                         db_nsr_update["config-status"] = "failed"
1449                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1450                         db_nsr_update["detailed-status"] = error_text
1451                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1452                         db_nslcmop_update["detailed-status"] = error_text
1453                         db_nslcmop_update["statusEnteredTime"] = time()
1454                         configuration_failed = True
1455                         break
1456                     else:
1457                         cs = "configuring: "
1458                         separator = ""
1459                         for status, num in status_map.items():
1460                             cs += separator + "{}: {}".format(status, num)
1461                             separator = ", "
1462                         if old_status != cs:
1463                             db_nsr_update["config-status"] = cs
1464                             db_nsr_update["detailed-status"] = cs
1465                             db_nslcmop_update["detailed-status"] = cs
1466                             old_status = cs
1467                 else:   # total_deploy_timeout
1468                     raise LcmException("Timeout waiting ns to be configured")
1469
1470             if not configuration_failed:
1471                 # all is done
1472                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1473                 db_nslcmop_update["statusEnteredTime"] = time()
1474                 db_nslcmop_update["detailed-status"] = "done"
1475                 db_nsr_update["config-status"] = "configured"
1476                 db_nsr_update["detailed-status"] = "done"
1477
1478             return
1479
1480         except (ROclient.ROClientException, DbException, LcmException) as e:
1481             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1482             exc = e
1483         except asyncio.CancelledError:
1484             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1485             exc = "Operation was cancelled"
1486         except Exception as e:
1487             exc = traceback.format_exc()
1488             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1489                                  exc_info=True)
1490         finally:
1491             if exc:
1492                 if db_nsr:
1493                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1494                     db_nsr_update["operational-status"] = "failed"
1495                 if db_nslcmop:
1496                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1497                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1498                     db_nslcmop_update["statusEnteredTime"] = time()
1499             try:
1500                 if db_nsr:
1501                     db_nsr_update["_admin.nslcmop"] = None
1502                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1503                 if db_nslcmop_update:
1504                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1505             except DbException as e:
1506                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1507             if nslcmop_operation_state:
1508                 try:
1509                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1510                                                                    "operationState": nslcmop_operation_state},
1511                                             loop=self.loop)
1512                 except Exception as e:
1513                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1514
1515             self.logger.debug(logging_text + "Exit")
1516             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1517
1518     async def _destroy_charm(self, model, application):
1519         """
1520         Order N2VC destroy a charm
1521         :param model:
1522         :param application:
1523         :return: True if charm does not exist. False if it exist
1524         """
1525         if not await self.n2vc.HasApplication(model, application):
1526             return True  # Already removed
1527         await self.n2vc.RemoveCharms(model, application)
1528         return False
1529
1530     async def _wait_charm_destroyed(self, model, application, timeout):
1531         """
1532         Wait until charm does not exist
1533         :param model:
1534         :param application:
1535         :param timeout:
1536         :return: True if not exist, False if timeout
1537         """
1538         while True:
1539             if not await self.n2vc.HasApplication(model, application):
1540                 return True
1541             if timeout < 0:
1542                 return False
1543             await asyncio.sleep(10)
1544             timeout -= 10
1545
1546     # Check if this VNFD has a configured terminate action
1547     def _has_terminate_config_primitive(self, vnfd):
1548         vnf_config = vnfd.get("vnf-configuration")
1549         if vnf_config and vnf_config.get("terminate-config-primitive"):
1550             return True
1551         else:
1552             return False
1553
1554     # Get a numerically sorted list of the sequences for this VNFD's terminate action
1555     def _get_terminate_config_primitive_seq_list(self, vnfd):
1556         # No need to check for existing primitive twice, already done before
1557         vnf_config = vnfd.get("vnf-configuration")
1558         seq_list = vnf_config.get("terminate-config-primitive")
1559         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1560         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1561         return seq_list_sorted
1562
1563     @staticmethod
1564     def _create_nslcmop(nsr_id, operation, params):
1565         """
1566         Creates a ns-lcm-opp content to be stored at database.
1567         :param nsr_id: internal id of the instance
1568         :param operation: instantiate, terminate, scale, action, ...
1569         :param params: user parameters for the operation
1570         :return: dictionary following SOL005 format
1571         """
1572         # Raise exception if invalid arguments
1573         if not (nsr_id and operation and params):
1574             raise LcmException(
1575                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1576         now = time()
1577         _id = str(uuid4())
1578         nslcmop = {
1579             "id": _id,
1580             "_id": _id,
1581             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1582             "operationState": "PROCESSING",
1583             "statusEnteredTime": now,
1584             "nsInstanceId": nsr_id,
1585             "lcmOperationType": operation,
1586             "startTime": now,
1587             "isAutomaticInvocation": False,
1588             "operationParams": params,
1589             "isCancelPending": False,
1590             "links": {
1591                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1592                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1593             }
1594         }
1595         return nslcmop
1596
1597     # Create a primitive with params from VNFD
1598     # - Called from terminate() before deleting instance
1599     # - Calls action() to execute the primitive
1600     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1601         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1602         db_vnfds = {}
1603         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1604         # Loop over VNFRs
1605         for vnfr in db_vnfrs_list:
1606             vnfd_id = vnfr["vnfd-id"]
1607             vnf_index = vnfr["member-vnf-index-ref"]
1608             if vnfd_id not in db_vnfds:
1609                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1610                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1611                 db_vnfds[vnfd_id] = vnfd
1612             vnfd = db_vnfds[vnfd_id]
1613             if not self._has_terminate_config_primitive(vnfd):
1614                 continue
1615             # Get the primitive's sorted sequence list
1616             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1617             for seq in seq_list:
1618                 # For each sequence in list, call terminate action
1619                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1620                     vnf_index, seq.get("name"))
1621                 self.logger.debug(logging_text + step)
1622                 # Create the primitive for each sequence
1623                 operation = "action"
1624                 # primitive, i.e. "primitive": "touch"
1625                 primitive = seq.get('name')
1626                 primitive_params = {}
1627                 params = {
1628                     "member_vnf_index": vnf_index,
1629                     "primitive": primitive,
1630                     "primitive_params": primitive_params,
1631                 }
1632                 nslcmop_primitive = self._create_nslcmop(nsr_id, operation, params)
1633                 # Get a copy of db_nslcmop 'admin' part
1634                 db_nslcmop_action = {"_admin": deepcopy(db_nslcmop["_admin"])}
1635                 # Update db_nslcmop with the primitive data
1636                 db_nslcmop_action.update(nslcmop_primitive)
1637                 # Create a new db entry for the created primitive, returns the new ID.
1638                 # (The ID is normally obtained from Kafka.)
1639                 nslcmop_terminate_action_id = self.db.create(
1640                     "nslcmops", db_nslcmop_action)
1641                 # Execute the primitive
1642                 nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1643                     nsr_id, nslcmop_terminate_action_id)
1644                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1645                 nslcmop_operation_states_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1646                 if nslcmop_operation_state not in nslcmop_operation_states_ok:
1647                     raise LcmException(
1648                         "terminate_primitive_action for vnf_member_index={}",
1649                         " primitive={} fails with error {}".format(
1650                             vnf_index, seq.get("name"), nslcmop_operation_state_detail))
1651
1652     async def terminate(self, nsr_id, nslcmop_id):
1653         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1654         self.logger.debug(logging_text + "Enter")
1655         db_nsr = None
1656         db_nslcmop = None
1657         exc = None
1658         failed_detail = []   # annotates all failed error messages
1659         vca_time_destroy = None   # time of where destroy charm order
1660         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1661         db_nslcmop_update = {}
1662         nslcmop_operation_state = None
1663         autoremove = False  # autoremove after terminated
1664         try:
1665             step = "Getting nslcmop={} from db".format(nslcmop_id)
1666             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1667             step = "Getting nsr={} from db".format(nsr_id)
1668             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1669             # nsd = db_nsr["nsd"]
1670             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1671             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1672                 return
1673             # #TODO check if VIM is creating and wait
1674             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1675             # Call internal terminate action
1676             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1677
1678             db_nsr_update["operational-status"] = "terminating"
1679             db_nsr_update["config-status"] = "terminating"
1680
1681             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1682                 vca_model_name = nsr_deployed["VCA-model-name"]
1683                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1684                 self.logger.debug(logging_text + step)
1685                 try:
1686                     await self.n2vc.DestroyNetworkService(vca_model_name)
1687                 except NetworkServiceDoesNotExist:
1688                     pass
1689                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1690                 if nsr_deployed.get("VCA"):
1691                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1692                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1693                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1694             # for backward compatibility if charm have been created with "default"  model name delete one by one
1695             elif nsr_deployed and nsr_deployed.get("VCA"):
1696                 try:
1697                     step = "Scheduling configuration charms removing"
1698                     db_nsr_update["detailed-status"] = "Deleting charms"
1699                     self.logger.debug(logging_text + step)
1700                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1701                     # for backward compatibility
1702                     if isinstance(nsr_deployed["VCA"], dict):
1703                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1704                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1705                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1706
1707                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1708                         if vca_deployed:
1709                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1710                                 vca_deployed.clear()
1711                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1712                             else:
1713                                 vca_time_destroy = time()
1714                 except Exception as e:
1715                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1716
1717             # remove from RO
1718             RO_fail = False
1719             RO = ROclient.ROClient(self.loop, **self.ro_config)
1720
1721             # Delete ns
1722             RO_nsr_id = RO_delete_action = None
1723             if nsr_deployed and nsr_deployed.get("RO"):
1724                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1725                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1726             try:
1727                 if RO_nsr_id:
1728                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1729                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1730                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1731                     self.logger.debug(logging_text + step)
1732                     desc = await RO.delete("ns", RO_nsr_id)
1733                     RO_delete_action = desc["action_id"]
1734                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1735                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1736                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1737                 if RO_delete_action:
1738                     # wait until NS is deleted from VIM
1739                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1740                         format(RO_nsr_id, RO_delete_action)
1741                     detailed_status_old = None
1742                     self.logger.debug(logging_text + step)
1743
1744                     delete_timeout = 20 * 60   # 20 minutes
1745                     while delete_timeout > 0:
1746                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1747                                              extra_item_id=RO_delete_action)
1748                         ns_status, ns_status_info = RO.check_action_status(desc)
1749                         if ns_status == "ERROR":
1750                             raise ROclient.ROClientException(ns_status_info)
1751                         elif ns_status == "BUILD":
1752                             detailed_status = step + "; {}".format(ns_status_info)
1753                         elif ns_status == "ACTIVE":
1754                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1755                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1756                             break
1757                         else:
1758                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1759                         if detailed_status != detailed_status_old:
1760                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1761                                 db_nsr_update["detailed-status"] = detailed_status
1762                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1763                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1764                         await asyncio.sleep(5, loop=self.loop)
1765                         delete_timeout -= 5
1766                     else:  # delete_timeout <= 0:
1767                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1768
1769             except ROclient.ROClientException as e:
1770                 if e.http_code == 404:  # not found
1771                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1772                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1773                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1774                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1775                 elif e.http_code == 409:   # conflict
1776                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1777                     self.logger.debug(logging_text + failed_detail[-1])
1778                     RO_fail = True
1779                 else:
1780                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1781                     self.logger.error(logging_text + failed_detail[-1])
1782                     RO_fail = True
1783
1784             # Delete nsd
1785             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1786                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1787                 try:
1788                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1789                         "Deleting nsd at RO"
1790                     await RO.delete("nsd", RO_nsd_id)
1791                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1792                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1793                 except ROclient.ROClientException as e:
1794                     if e.http_code == 404:  # not found
1795                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1796                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1797                     elif e.http_code == 409:   # conflict
1798                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1799                         self.logger.debug(logging_text + failed_detail[-1])
1800                         RO_fail = True
1801                     else:
1802                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1803                         self.logger.error(logging_text + failed_detail[-1])
1804                         RO_fail = True
1805
1806             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1807                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1808                     if not vnf_deployed or not vnf_deployed["id"]:
1809                         continue
1810                     try:
1811                         RO_vnfd_id = vnf_deployed["id"]
1812                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1813                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1814                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1815                         await RO.delete("vnfd", RO_vnfd_id)
1816                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1817                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1818                     except ROclient.ROClientException as e:
1819                         if e.http_code == 404:  # not found
1820                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1821                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1822                         elif e.http_code == 409:   # conflict
1823                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1824                             self.logger.debug(logging_text + failed_detail[-1])
1825                         else:
1826                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1827                             self.logger.error(logging_text + failed_detail[-1])
1828
1829             # wait until charm deleted
1830             if vca_time_destroy:
1831                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1832                     "Waiting for deletion of configuration charms"
1833                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1834                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1835                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1836                     if not vca_deployed:
1837                         continue
1838                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1839                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1840                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1841                                                             timeout):
1842                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1843                             vca_deployed["application"]))
1844                     else:
1845                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1846
1847             if failed_detail:
1848                 self.logger.error(logging_text + " ;".join(failed_detail))
1849                 db_nsr_update["operational-status"] = "failed"
1850                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1851                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1852                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1853                 db_nslcmop_update["statusEnteredTime"] = time()
1854             else:
1855                 db_nsr_update["operational-status"] = "terminated"
1856                 db_nsr_update["detailed-status"] = "Done"
1857                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1858                 db_nslcmop_update["detailed-status"] = "Done"
1859                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1860                 db_nslcmop_update["statusEnteredTime"] = time()
1861                 if db_nslcmop["operationParams"].get("autoremove"):
1862                     autoremove = True
1863
1864         except (ROclient.ROClientException, DbException, LcmException) as e:
1865             self.logger.error(logging_text + "Exit Exception {}".format(e))
1866             exc = e
1867         except asyncio.CancelledError:
1868             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1869             exc = "Operation was cancelled"
1870         except Exception as e:
1871             exc = traceback.format_exc()
1872             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1873         finally:
1874             if exc and db_nslcmop:
1875                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1876                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1877                 db_nslcmop_update["statusEnteredTime"] = time()
1878             try:
1879                 if db_nslcmop and db_nslcmop_update:
1880                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1881                 if db_nsr:
1882                     db_nsr_update["_admin.nslcmop"] = None
1883                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1884             except DbException as e:
1885                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1886             if nslcmop_operation_state:
1887                 try:
1888                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1889                                                                  "operationState": nslcmop_operation_state,
1890                                                                  "autoremove": autoremove},
1891                                             loop=self.loop)
1892                 except Exception as e:
1893                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1894             self.logger.debug(logging_text + "Exit")
1895             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1896
1897     @staticmethod
1898     def _map_primitive_params(primitive_desc, params, instantiation_params):
1899         """
1900         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1901         The default-value is used. If it is between < > it look for a value at instantiation_params
1902         :param primitive_desc: portion of VNFD/NSD that describes primitive
1903         :param params: Params provided by user
1904         :param instantiation_params: Instantiation params provided by user
1905         :return: a dictionary with the calculated params
1906         """
1907         calculated_params = {}
1908         for parameter in primitive_desc.get("parameter", ()):
1909             param_name = parameter["name"]
1910             if param_name in params:
1911                 calculated_params[param_name] = params[param_name]
1912             elif "default-value" in parameter or "value" in parameter:
1913                 if "value" in parameter:
1914                     calculated_params[param_name] = parameter["value"]
1915                 else:
1916                     calculated_params[param_name] = parameter["default-value"]
1917                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1918                         and calculated_params[param_name].endswith(">"):
1919                     if calculated_params[param_name][1:-1] in instantiation_params:
1920                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1921                     else:
1922                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1923                                            format(parameter["default-value"], primitive_desc["name"]))
1924             else:
1925                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1926                                    format(param_name, primitive_desc["name"]))
1927
1928             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1929                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1930                                                                width=256)
1931             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1932                 calculated_params[param_name] = calculated_params[param_name][7:]
1933         return calculated_params
1934
1935     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1936                                     primitive, primitive_params, retries=0, retries_interval=30):
1937         start_primitive_time = time()
1938         try:
1939             for vca_deployed in db_deployed["VCA"]:
1940                 if not vca_deployed:
1941                     continue
1942                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1943                     continue
1944                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1945                     continue
1946                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1947                     continue
1948                 break
1949             else:
1950                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1951                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1952             model_name = vca_deployed.get("model")
1953             application_name = vca_deployed.get("application")
1954             if not model_name or not application_name:
1955                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1956                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1957                                                                        vdu_count_index))
1958             # if vca_deployed["operational-status"] != "active":
1959             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1960             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1961             callback = None  # self.n2vc_callback
1962             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1963             await self.n2vc.login()
1964             if primitive == "config":
1965                 primitive_params = {"params": primitive_params}
1966             while retries >= 0:
1967                 primitive_id = await self.n2vc.ExecutePrimitive(
1968                     model_name,
1969                     application_name,
1970                     primitive,
1971                     callback,
1972                     *callback_args,
1973                     **primitive_params
1974                 )
1975                 while time() - start_primitive_time < self.timeout_primitive:
1976                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1977                     if primitive_result_ in ("completed", "failed"):
1978                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1979                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1980                         break
1981                     elif primitive_result_ is None and primitive == "config":
1982                         primitive_result = "COMPLETED"
1983                         detailed_result = None
1984                         break
1985                     else:  # ("running", "pending", None):
1986                         pass
1987                     await asyncio.sleep(5)
1988                 else:
1989                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1990                 if primitive_result == "COMPLETED":
1991                     break
1992                 retries -= 1
1993                 if retries >= 0:
1994                     await asyncio.sleep(retries_interval)
1995
1996             return primitive_result, detailed_result
1997         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
1998             return "FAILED", str(e)
1999
2000     async def action(self, nsr_id, nslcmop_id):
2001         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2002         self.logger.debug(logging_text + "Enter")
2003         # get all needed from database
2004         db_nsr = None
2005         db_nslcmop = None
2006         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2007         db_nslcmop_update = {}
2008         nslcmop_operation_state = None
2009         nslcmop_operation_state_detail = None
2010         exc = None
2011         try:
2012             step = "Getting information from database"
2013             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2014             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2015
2016             nsr_deployed = db_nsr["_admin"].get("deployed")
2017             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2018             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2019             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2020             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2021
2022             if vnf_index:
2023                 step = "Getting vnfr from database"
2024                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2025                 step = "Getting vnfd from database"
2026                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2027             else:
2028                 if db_nsr.get("nsd"):
2029                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2030                 else:
2031                     step = "Getting nsd from database"
2032                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2033
2034             # look if previous tasks in process
2035             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
2036             if task_dependency:
2037                 step = db_nslcmop_update["detailed-status"] = \
2038                     "Waiting for related tasks to be completed: {}".format(task_name)
2039                 self.logger.debug(logging_text + step)
2040                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2041                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
2042                 if pending:
2043                     raise LcmException("Timeout waiting related tasks to be completed")
2044
2045             # for backward compatibility
2046             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2047                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2048                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2049                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2050
2051             primitive = db_nslcmop["operationParams"]["primitive"]
2052             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2053
2054             # look for primitive
2055             config_primitive_desc = None
2056             if vdu_id:
2057                 for vdu in get_iterable(db_vnfd, "vdu"):
2058                     if vdu_id == vdu["id"]:
2059                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2060                             if config_primitive["name"] == primitive:
2061                                 config_primitive_desc = config_primitive
2062                                 break
2063             elif vnf_index:
2064                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2065                     if config_primitive["name"] == primitive:
2066                         config_primitive_desc = config_primitive
2067                         break
2068             else:
2069                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2070                     if config_primitive["name"] == primitive:
2071                         config_primitive_desc = config_primitive
2072                         break
2073
2074             if not config_primitive_desc:
2075                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2076                                    format(primitive))
2077
2078             desc_params = {}
2079             if vnf_index:
2080                 if db_vnfr.get("additionalParamsForVnf"):
2081                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2082             else:
2083                 if db_nsr.get("additionalParamsForVnf"):
2084                     desc_params.update(db_nsr["additionalParamsForNs"])
2085
2086             # TODO check if ns is in a proper status
2087             result, result_detail = await self._ns_execute_primitive(
2088                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2089                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2090             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2091             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2092             db_nslcmop_update["statusEnteredTime"] = time()
2093             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2094             return  # database update is called inside finally
2095
2096         except (DbException, LcmException) as e:
2097             self.logger.error(logging_text + "Exit Exception {}".format(e))
2098             exc = e
2099         except asyncio.CancelledError:
2100             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2101             exc = "Operation was cancelled"
2102         except Exception as e:
2103             exc = traceback.format_exc()
2104             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2105         finally:
2106             if exc and db_nslcmop:
2107                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2108                     "FAILED {}: {}".format(step, exc)
2109                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2110                 db_nslcmop_update["statusEnteredTime"] = time()
2111             try:
2112                 if db_nslcmop_update:
2113                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2114                 if db_nsr:
2115                     db_nsr_update["_admin.nslcmop"] = None
2116                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2117             except DbException as e:
2118                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2119             self.logger.debug(logging_text + "Exit")
2120             if nslcmop_operation_state:
2121                 try:
2122                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2123                                                                "operationState": nslcmop_operation_state},
2124                                             loop=self.loop)
2125                 except Exception as e:
2126                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2127             self.logger.debug(logging_text + "Exit")
2128             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2129             return nslcmop_operation_state, nslcmop_operation_state_detail
2130
2131     async def scale(self, nsr_id, nslcmop_id):
2132         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2133         self.logger.debug(logging_text + "Enter")
2134         # get all needed from database
2135         db_nsr = None
2136         db_nslcmop = None
2137         db_nslcmop_update = {}
2138         nslcmop_operation_state = None
2139         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2140         exc = None
2141         # in case of error, indicates what part of scale was failed to put nsr at error status
2142         scale_process = None
2143         old_operational_status = ""
2144         old_config_status = ""
2145         vnfr_scaled = False
2146         try:
2147             # look if previous tasks in process
2148             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
2149             if task_dependency:
2150                 step = db_nslcmop_update["detailed-status"] = \
2151                     "Waiting for related tasks to be completed: {}".format(task_name)
2152                 self.logger.debug(logging_text + step)
2153                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2154                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
2155                 if pending:
2156                     raise LcmException("Timeout waiting related tasks to be completed")
2157
2158             step = "Getting nslcmop from database"
2159             self.logger.debug(step + " after having waited for previous tasks to be completed")
2160             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2161             step = "Getting nsr from database"
2162             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2163
2164             old_operational_status = db_nsr["operational-status"]
2165             old_config_status = db_nsr["config-status"]
2166             step = "Parsing scaling parameters"
2167             # self.logger.debug(step)
2168             db_nsr_update["operational-status"] = "scaling"
2169             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2170             nsr_deployed = db_nsr["_admin"].get("deployed")
2171             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2172             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2173             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2174             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2175             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2176
2177             # for backward compatibility
2178             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2179                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2180                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2181                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2182
2183             step = "Getting vnfr from database"
2184             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2185             step = "Getting vnfd from database"
2186             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2187
2188             step = "Getting scaling-group-descriptor"
2189             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2190                 if scaling_descriptor["name"] == scaling_group:
2191                     break
2192             else:
2193                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2194                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2195
2196             # cooldown_time = 0
2197             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2198             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2199             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2200             #         break
2201
2202             # TODO check if ns is in a proper status
2203             step = "Sending scale order to RO"
2204             nb_scale_op = 0
2205             if not db_nsr["_admin"].get("scaling-group"):
2206                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2207                 admin_scale_index = 0
2208             else:
2209                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2210                     if admin_scale_info["name"] == scaling_group:
2211                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2212                         break
2213                 else:  # not found, set index one plus last element and add new entry with the name
2214                     admin_scale_index += 1
2215                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2216             RO_scaling_info = []
2217             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2218             if scaling_type == "SCALE_OUT":
2219                 # count if max-instance-count is reached
2220                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
2221                     max_instance_count = int(scaling_descriptor["max-instance-count"])
2222
2223                     # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(scaling_descriptor["max-instance-count"]))
2224                     if nb_scale_op >= max_instance_count:
2225                         raise LcmException("reached the limit of {} (max-instance-count) "
2226                                            "scaling-out operations for the "
2227                                            "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2228
2229                 nb_scale_op += 1
2230                 vdu_scaling_info["scaling_direction"] = "OUT"
2231                 vdu_scaling_info["vdu-create"] = {}
2232                 for vdu_scale_info in scaling_descriptor["vdu"]:
2233                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2234                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2235                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2236
2237             elif scaling_type == "SCALE_IN":
2238                 # count if min-instance-count is reached
2239                 min_instance_count = 0
2240                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2241                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2242                 if nb_scale_op <= min_instance_count:
2243                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2244                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2245                 nb_scale_op -= 1
2246                 vdu_scaling_info["scaling_direction"] = "IN"
2247                 vdu_scaling_info["vdu-delete"] = {}
2248                 for vdu_scale_info in scaling_descriptor["vdu"]:
2249                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2250                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2251                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2252
2253             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2254             vdu_create = vdu_scaling_info.get("vdu-create")
2255             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2256             if vdu_scaling_info["scaling_direction"] == "IN":
2257                 for vdur in reversed(db_vnfr["vdur"]):
2258                     if vdu_delete.get(vdur["vdu-id-ref"]):
2259                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2260                         vdu_scaling_info["vdu"].append({
2261                             "name": vdur["name"],
2262                             "vdu_id": vdur["vdu-id-ref"],
2263                             "interface": []
2264                         })
2265                         for interface in vdur["interfaces"]:
2266                             vdu_scaling_info["vdu"][-1]["interface"].append({
2267                                 "name": interface["name"],
2268                                 "ip_address": interface["ip-address"],
2269                                 "mac_address": interface.get("mac-address"),
2270                             })
2271                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2272
2273             # execute primitive service PRE-SCALING
2274             step = "Executing pre-scale vnf-config-primitive"
2275             if scaling_descriptor.get("scaling-config-action"):
2276                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2277                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2278                             and scaling_type == "SCALE_IN":
2279                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2280                         step = db_nslcmop_update["detailed-status"] = \
2281                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2282
2283                         # look for primitive
2284                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2285                             if config_primitive["name"] == vnf_config_primitive:
2286                                 break
2287                         else:
2288                             raise LcmException(
2289                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2290                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2291                                 "primitive".format(scaling_group, config_primitive))
2292
2293                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2294                         if db_vnfr.get("additionalParamsForVnf"):
2295                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2296
2297                         scale_process = "VCA"
2298                         db_nsr_update["config-status"] = "configuring pre-scaling"
2299                         result, result_detail = await self._ns_execute_primitive(
2300                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2301                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2302                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2303                             vnf_config_primitive, result, result_detail))
2304                         if result == "FAILED":
2305                             raise LcmException(result_detail)
2306                         db_nsr_update["config-status"] = old_config_status
2307                         scale_process = None
2308
2309             if RO_scaling_info:
2310                 scale_process = "RO"
2311                 RO = ROclient.ROClient(self.loop, **self.ro_config)
2312                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2313                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2314                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2315                 # wait until ready
2316                 RO_nslcmop_id = RO_desc["instance_action_id"]
2317                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2318
2319                 RO_task_done = False
2320                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2321                 detailed_status_old = None
2322                 self.logger.debug(logging_text + step)
2323
2324                 deployment_timeout = 1 * 3600   # One hour
2325                 while deployment_timeout > 0:
2326                     if not RO_task_done:
2327                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2328                                              extra_item_id=RO_nslcmop_id)
2329                         ns_status, ns_status_info = RO.check_action_status(desc)
2330                         if ns_status == "ERROR":
2331                             raise ROclient.ROClientException(ns_status_info)
2332                         elif ns_status == "BUILD":
2333                             detailed_status = step + "; {}".format(ns_status_info)
2334                         elif ns_status == "ACTIVE":
2335                             RO_task_done = True
2336                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2337                             self.logger.debug(logging_text + step)
2338                         else:
2339                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2340                     else:
2341                         desc = await RO.show("ns", RO_nsr_id)
2342                         ns_status, ns_status_info = RO.check_ns_status(desc)
2343                         if ns_status == "ERROR":
2344                             raise ROclient.ROClientException(ns_status_info)
2345                         elif ns_status == "BUILD":
2346                             detailed_status = step + "; {}".format(ns_status_info)
2347                         elif ns_status == "ACTIVE":
2348                             step = detailed_status = \
2349                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2350                             if not vnfr_scaled:
2351                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2352                                 vnfr_scaled = True
2353                             try:
2354                                 desc = await RO.show("ns", RO_nsr_id)
2355                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2356                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2357                                 break
2358                             except LcmExceptionNoMgmtIP:
2359                                 pass
2360                         else:
2361                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2362                     if detailed_status != detailed_status_old:
2363                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2364                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2365
2366                     await asyncio.sleep(5, loop=self.loop)
2367                     deployment_timeout -= 5
2368                 if deployment_timeout <= 0:
2369                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2370
2371                 # update VDU_SCALING_INFO with the obtained ip_addresses
2372                 if vdu_scaling_info["scaling_direction"] == "OUT":
2373                     for vdur in reversed(db_vnfr["vdur"]):
2374                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2375                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2376                             vdu_scaling_info["vdu"].append({
2377                                 "name": vdur["name"],
2378                                 "vdu_id": vdur["vdu-id-ref"],
2379                                 "interface": []
2380                             })
2381                             for interface in vdur["interfaces"]:
2382                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2383                                     "name": interface["name"],
2384                                     "ip_address": interface["ip-address"],
2385                                     "mac_address": interface.get("mac-address"),
2386                                 })
2387                     del vdu_scaling_info["vdu-create"]
2388
2389             scale_process = None
2390             if db_nsr_update:
2391                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2392
2393             # execute primitive service POST-SCALING
2394             step = "Executing post-scale vnf-config-primitive"
2395             if scaling_descriptor.get("scaling-config-action"):
2396                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2397                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2398                             and scaling_type == "SCALE_OUT":
2399                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2400                         step = db_nslcmop_update["detailed-status"] = \
2401                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2402
2403                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2404                         if db_vnfr.get("additionalParamsForVnf"):
2405                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2406
2407                         # look for primitive
2408                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2409                             if config_primitive["name"] == vnf_config_primitive:
2410                                 break
2411                         else:
2412                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2413                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2414                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2415                                                                                                      config_primitive))
2416                         scale_process = "VCA"
2417                         db_nsr_update["config-status"] = "configuring post-scaling"
2418
2419                         result, result_detail = await self._ns_execute_primitive(
2420                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2421                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2422                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2423                             vnf_config_primitive, result, result_detail))
2424                         if result == "FAILED":
2425                             raise LcmException(result_detail)
2426                         db_nsr_update["config-status"] = old_config_status
2427                         scale_process = None
2428
2429             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2430             db_nslcmop_update["statusEnteredTime"] = time()
2431             db_nslcmop_update["detailed-status"] = "done"
2432             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2433             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2434                 else old_operational_status
2435             db_nsr_update["config-status"] = old_config_status
2436             return
2437         except (ROclient.ROClientException, DbException, LcmException) as e:
2438             self.logger.error(logging_text + "Exit Exception {}".format(e))
2439             exc = e
2440         except asyncio.CancelledError:
2441             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2442             exc = "Operation was cancelled"
2443         except Exception as e:
2444             exc = traceback.format_exc()
2445             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2446         finally:
2447             if exc:
2448                 if db_nslcmop:
2449                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2450                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2451                     db_nslcmop_update["statusEnteredTime"] = time()
2452                 if db_nsr:
2453                     db_nsr_update["operational-status"] = old_operational_status
2454                     db_nsr_update["config-status"] = old_config_status
2455                     db_nsr_update["detailed-status"] = ""
2456                     db_nsr_update["_admin.nslcmop"] = None
2457                     if scale_process:
2458                         if "VCA" in scale_process:
2459                             db_nsr_update["config-status"] = "failed"
2460                         if "RO" in scale_process:
2461                             db_nsr_update["operational-status"] = "failed"
2462                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2463                                                                                                      exc)
2464             try:
2465                 if db_nslcmop and db_nslcmop_update:
2466                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2467                 if db_nsr:
2468                     db_nsr_update["_admin.nslcmop"] = None
2469                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2470             except DbException as e:
2471                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2472             if nslcmop_operation_state:
2473                 try:
2474                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2475                                                              "operationState": nslcmop_operation_state},
2476                                             loop=self.loop)
2477                     # if cooldown_time:
2478                     #     await asyncio.sleep(cooldown_time)
2479                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2480                 except Exception as e:
2481                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2482             self.logger.debug(logging_text + "Exit")
2483             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")