osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 import ROclient
  28 from lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105
 106     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 107         """
 108         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 109         :param vnfd: input vnfd
 110         :param new_id: overrides vnf id if provided
 111         :param additionalParams: Instantiation params for VNFs provided
 112         :param nsrId: Id of the NSR
 113         :return: copy of vnfd
 114         """
 115         try:
 116             vnfd_RO = deepcopy(vnfd)
 117             # remove unused by RO configuration, monitoring, scaling and internal keys
 118             vnfd_RO.pop("_id", None)
 119             vnfd_RO.pop("_admin", None)
 120             vnfd_RO.pop("vnf-configuration", None)
 121             vnfd_RO.pop("monitoring-param", None)
 122             vnfd_RO.pop("scaling-group-descriptor", None)
 123             if new_id:
 124                 vnfd_RO["id"] = new_id
 125
 126             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 127             for vdu in get_iterable(vnfd_RO, "vdu"):
 128                 cloud_init_file = None
 129                 if vdu.get("cloud-init-file"):
 130                     base_folder = vnfd["_admin"]["storage"]
 131                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 132                                                                    vdu["cloud-init-file"])
 133                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 134                         cloud_init_content = ci_file.read()
 135                     vdu.pop("cloud-init-file", None)
 136                 elif vdu.get("cloud-init"):
 137                     cloud_init_content = vdu["cloud-init"]
 138                 else:
 139                     continue
 140
 141                 env = Environment()
 142                 ast = env.parse(cloud_init_content)
 143                 mandatory_vars = meta.find_undeclared_variables(ast)
 144                 if mandatory_vars:
 145                     for var in mandatory_vars:
 146                         if not additionalParams or var not in additionalParams.keys():
 147                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 148                                                "file, must be provided in the instantiation parameters inside the "
 149                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 150                 template = Template(cloud_init_content)
 151                 cloud_init_content = template.render(additionalParams or {})
 152                 vdu["cloud-init"] = cloud_init_content
 153
 154             return vnfd_RO
 155         except FsException as e:
 156             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 157                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 158         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 159             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 160                                format(vnfd["id"], vdu["id"], e))
 161
 162     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 163         """
 164         Callback both for charm status change and task completion
 165         :param model_name: Charm model name
 166         :param application_name: Charm application name
 167         :param status: Can be
 168             - blocked: The unit needs manual intervention
 169             - maintenance: The unit is actively deploying/configuring
 170             - waiting: The unit is waiting for another charm to be ready
 171             - active: The unit is deployed, configured, and ready
 172             - error: The charm has failed and needs attention.
 173             - terminated: The charm has been destroyed
 174             - removing,
 175             - removed
 176         :param message: detailed message error
 177         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 178             nsr_id:
 179             nslcmop_id:
 180             lcmOperationType: currently "instantiate"
 181             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 182             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 183             n2vc_event: event used to notify instantiation task that some change has been produced
 184         :param task: None for charm status change, or task for completion task callback
 185         :return:
 186         """
 187         try:
 188             nsr_id = n2vc_info["nsr_id"]
 189             deployed = n2vc_info["deployed"]
 190             db_nsr_update = n2vc_info["db_update"]
 191             nslcmop_id = n2vc_info["nslcmop_id"]
 192             ns_operation = n2vc_info["lcmOperationType"]
 193             n2vc_event = n2vc_info["n2vc_event"]
 194             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 195                                                                                     application_name)
 196             for vca_index, vca_deployed in enumerate(deployed):
 197                 if not vca_deployed:
 198                     continue
 199                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 200                     break
 201             else:
 202                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 203                                   format(model_name))
 204                 return
 205             if task:
 206                 if task.cancelled():
 207                     self.logger.debug(logging_text + " task Cancelled")
 208                     vca_deployed['operational-status'] = "error"
 209                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 210                     vca_deployed['detailed-status'] = "Task Cancelled"
 211                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 212
 213                 elif task.done():
 214                     exc = task.exception()
 215                     if exc:
 216                         self.logger.error(logging_text + " task Exception={}".format(exc))
 217                         vca_deployed['operational-status'] = "error"
 218                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 219                         vca_deployed['detailed-status'] = str(exc)
 220                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 221                     else:
 222                         self.logger.debug(logging_text + " task Done")
 223                         # task is Done, but callback is still ongoing. So ignore
 224                         return
 225             elif status:
 226                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 227                 if vca_deployed['operational-status'] == status:
 228                     return  # same status, ignore
 229                 vca_deployed['operational-status'] = status
 230                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 231                 vca_deployed['detailed-status'] = str(message)
 232                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 233             else:
 234                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 235                 return
 236             # wake up instantiate task
 237             n2vc_event.set()
 238         except Exception as e:
 239             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 240
 241     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 242         """
 243         Creates a RO ns descriptor from OSM ns_instantiate params
 244         :param ns_params: OSM instantiate params
 245         :return: The RO ns descriptor
 246         """
 247         vim_2_RO = {}
 248         wim_2_RO = {}
 249         # TODO feature 1417: Check that no instantiation is set over PDU
 250         # check if PDU forces a concrete vim-network-id and add it
 251         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 252
 253         def vim_account_2_RO(vim_account):
 254             if vim_account in vim_2_RO:
 255                 return vim_2_RO[vim_account]
 256
 257             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 258             if db_vim["_admin"]["operationalState"] != "ENABLED":
 259                 raise LcmException("VIM={} is not available. operationalState={}".format(
 260                     vim_account, db_vim["_admin"]["operationalState"]))
 261             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 262             vim_2_RO[vim_account] = RO_vim_id
 263             return RO_vim_id
 264
 265         def wim_account_2_RO(wim_account):
 266             if isinstance(wim_account, str):
 267                 if wim_account in wim_2_RO:
 268                     return wim_2_RO[wim_account]
 269
 270                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 271                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 272                     raise LcmException("WIM={} is not available. operationalState={}".format(
 273                         wim_account, db_wim["_admin"]["operationalState"]))
 274                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 275                 wim_2_RO[wim_account] = RO_wim_id
 276                 return RO_wim_id
 277             else:
 278                 return wim_account
 279
 280         def ip_profile_2_RO(ip_profile):
 281             RO_ip_profile = deepcopy((ip_profile))
 282             if "dns-server" in RO_ip_profile:
 283                 if isinstance(RO_ip_profile["dns-server"], list):
 284                     RO_ip_profile["dns-address"] = []
 285                     for ds in RO_ip_profile.pop("dns-server"):
 286                         RO_ip_profile["dns-address"].append(ds['address'])
 287                 else:
 288                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 289             if RO_ip_profile.get("ip-version") == "ipv4":
 290                 RO_ip_profile["ip-version"] = "IPv4"
 291             if RO_ip_profile.get("ip-version") == "ipv6":
 292                 RO_ip_profile["ip-version"] = "IPv6"
 293             if "dhcp-params" in RO_ip_profile:
 294                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 295             return RO_ip_profile
 296
 297         if not ns_params:
 298             return None
 299         RO_ns_params = {
 300             # "name": ns_params["nsName"],
 301             # "description": ns_params.get("nsDescription"),
 302             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 303             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 304             # "scenario": ns_params["nsdId"],
 305         }
 306         if n2vc_key_list:
 307             for vnfd_ref, vnfd in vnfd_dict.items():
 308                 vdu_needed_access = []
 309                 mgmt_cp = None
 310                 if vnfd.get("vnf-configuration"):
 311                     if vnfd.get("mgmt-interface"):
 312                         if vnfd["mgmt-interface"].get("vdu-id"):
 313                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 314                         elif vnfd["mgmt-interface"].get("cp"):
 315                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 316
 317                 for vdu in vnfd.get("vdu", ()):
 318                     if vdu.get("vdu-configuration"):
 319                         vdu_needed_access.append(vdu["id"])
 320                     elif mgmt_cp:
 321                         for vdu_interface in vdu.get("interface"):
 322                             if vdu_interface.get("external-connection-point-ref") and \
 323                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 324                                 vdu_needed_access.append(vdu["id"])
 325                                 mgmt_cp = None
 326                                 break
 327
 328                 if vdu_needed_access:
 329                     for vnf_member in nsd.get("constituent-vnfd"):
 330                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 331                             continue
 332                         for vdu in vdu_needed_access:
 333                             populate_dict(RO_ns_params,
 334                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 335                                           n2vc_key_list)
 336
 337         if ns_params.get("vduImage"):
 338             RO_ns_params["vduImage"] = ns_params["vduImage"]
 339
 340         if ns_params.get("ssh_keys"):
 341             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 342         for vnf_params in get_iterable(ns_params, "vnf"):
 343             for constituent_vnfd in nsd["constituent-vnfd"]:
 344                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 345                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 346                     break
 347             else:
 348                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 349                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 350             if vnf_params.get("vimAccountId"):
 351                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 352                               vim_account_2_RO(vnf_params["vimAccountId"]))
 353
 354             for vdu_params in get_iterable(vnf_params, "vdu"):
 355                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 356                 if vdu_params.get("volume"):
 357                     for volume_params in vdu_params["volume"]:
 358                         if volume_params.get("vim-volume-id"):
 359                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 360                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 361                                           volume_params["vim-volume-id"])
 362                 if vdu_params.get("interface"):
 363                     for interface_params in vdu_params["interface"]:
 364                         if interface_params.get("ip-address"):
 365                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 366                                                          vdu_params["id"], "interfaces", interface_params["name"],
 367                                                          "ip_address"),
 368                                           interface_params["ip-address"])
 369                         if interface_params.get("mac-address"):
 370                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 371                                                          vdu_params["id"], "interfaces", interface_params["name"],
 372                                                          "mac_address"),
 373                                           interface_params["mac-address"])
 374                         if interface_params.get("floating-ip-required"):
 375                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 376                                                          vdu_params["id"], "interfaces", interface_params["name"],
 377                                                          "floating-ip"),
 378                                           interface_params["floating-ip-required"])
 379
 380             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 381                 if internal_vld_params.get("vim-network-name"):
 382                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 383                                                  internal_vld_params["name"], "vim-network-name"),
 384                                   internal_vld_params["vim-network-name"])
 385                 if internal_vld_params.get("vim-network-id"):
 386                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 387                                                  internal_vld_params["name"], "vim-network-id"),
 388                                   internal_vld_params["vim-network-id"])
 389                 if internal_vld_params.get("ip-profile"):
 390                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 391                                                  internal_vld_params["name"], "ip-profile"),
 392                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 393
 394                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 395                     # look for interface
 396                     iface_found = False
 397                     for vdu_descriptor in vnf_descriptor["vdu"]:
 398                         for vdu_interface in vdu_descriptor["interface"]:
 399                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 400                                 if icp_params.get("ip-address"):
 401                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 402                                                                  vdu_descriptor["id"], "interfaces",
 403                                                                  vdu_interface["name"], "ip_address"),
 404                                                   icp_params["ip-address"])
 405
 406                                 if icp_params.get("mac-address"):
 407                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 408                                                                  vdu_descriptor["id"], "interfaces",
 409                                                                  vdu_interface["name"], "mac_address"),
 410                                                   icp_params["mac-address"])
 411                                 iface_found = True
 412                                 break
 413                         if iface_found:
 414                             break
 415                     else:
 416                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 417                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 418                                            "connection-point".format(vnf_params["member-vnf-index"],
 419                                                                      icp_params["id-ref"]))
 420
 421         for vld_params in get_iterable(ns_params, "vld"):
 422             if "ip-profile" in vld_params:
 423                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 424                               ip_profile_2_RO(vld_params["ip-profile"]))
 425
 426             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 427                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 428                               wim_account_2_RO(vld_params["wimAccountId"])),
 429             if vld_params.get("vim-network-name"):
 430                 RO_vld_sites = []
 431                 if isinstance(vld_params["vim-network-name"], dict):
 432                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 433                         RO_vld_sites.append({
 434                             "netmap-use": vim_net,
 435                             "datacenter": vim_account_2_RO(vim_account)
 436                         })
 437                 else:  # isinstance str
 438                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 439                 if RO_vld_sites:
 440                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 441             if vld_params.get("vim-network-id"):
 442                 RO_vld_sites = []
 443                 if isinstance(vld_params["vim-network-id"], dict):
 444                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 445                         RO_vld_sites.append({
 446                             "netmap-use": vim_net,
 447                             "datacenter": vim_account_2_RO(vim_account)
 448                         })
 449                 else:  # isinstance str
 450                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 451                 if RO_vld_sites:
 452                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 453             if vld_params.get("ns-net"):
 454                 if isinstance(vld_params["ns-net"], dict):
 455                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 456                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 457                 if RO_vld_ns_net:
 458                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 459             if "vnfd-connection-point-ref" in vld_params:
 460                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 461                     # look for interface
 462                     for constituent_vnfd in nsd["constituent-vnfd"]:
 463                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 464                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 465                             break
 466                     else:
 467                         raise LcmException(
 468                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 469                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 470                     match_cp = False
 471                     for vdu_descriptor in vnf_descriptor["vdu"]:
 472                         for interface_descriptor in vdu_descriptor["interface"]:
 473                             if interface_descriptor.get("external-connection-point-ref") == \
 474                                     cp_params["vnfd-connection-point-ref"]:
 475                                 match_cp = True
 476                                 break
 477                         if match_cp:
 478                             break
 479                     else:
 480                         raise LcmException(
 481                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 482                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 483                                 cp_params["member-vnf-index-ref"],
 484                                 cp_params["vnfd-connection-point-ref"],
 485                                 vnf_descriptor["id"]))
 486                     if cp_params.get("ip-address"):
 487                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 488                                                      vdu_descriptor["id"], "interfaces",
 489                                                      interface_descriptor["name"], "ip_address"),
 490                                       cp_params["ip-address"])
 491                     if cp_params.get("mac-address"):
 492                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 493                                                      vdu_descriptor["id"], "interfaces",
 494                                                      interface_descriptor["name"], "mac_address"),
 495                                       cp_params["mac-address"])
 496         return RO_ns_params
 497
 498     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 499         # make a copy to do not change
 500         vdu_create = copy(vdu_create)
 501         vdu_delete = copy(vdu_delete)
 502
 503         vdurs = db_vnfr.get("vdur")
 504         if vdurs is None:
 505             vdurs = []
 506         vdu_index = len(vdurs)
 507         while vdu_index:
 508             vdu_index -= 1
 509             vdur = vdurs[vdu_index]
 510             if vdur.get("pdu-type"):
 511                 continue
 512             vdu_id_ref = vdur["vdu-id-ref"]
 513             if vdu_create and vdu_create.get(vdu_id_ref):
 514                 for index in range(0, vdu_create[vdu_id_ref]):
 515                     vdur = deepcopy(vdur)
 516                     vdur["_id"] = str(uuid4())
 517                     vdur["count-index"] += 1
 518                     vdurs.insert(vdu_index+1+index, vdur)
 519                 del vdu_create[vdu_id_ref]
 520             if vdu_delete and vdu_delete.get(vdu_id_ref):
 521                 del vdurs[vdu_index]
 522                 vdu_delete[vdu_id_ref] -= 1
 523                 if not vdu_delete[vdu_id_ref]:
 524                     del vdu_delete[vdu_id_ref]
 525         # check all operations are done
 526         if vdu_create or vdu_delete:
 527             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 528                 vdu_create))
 529         if vdu_delete:
 530             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 531                 vdu_delete))
 532
 533         vnfr_update = {"vdur": vdurs}
 534         db_vnfr["vdur"] = vdurs
 535         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 536
 537     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 538         """
 539         Updates database nsr with the RO info for the created vld
 540         :param ns_update_nsr: dictionary to be filled with the updated info
 541         :param db_nsr: content of db_nsr. This is also modified
 542         :param nsr_desc_RO: nsr descriptor from RO
 543         :return: Nothing, LcmException is raised on errors
 544         """
 545
 546         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 547             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 548                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 549                     continue
 550                 vld["vim-id"] = net_RO.get("vim_net_id")
 551                 vld["name"] = net_RO.get("vim_name")
 552                 vld["status"] = net_RO.get("status")
 553                 vld["status-detailed"] = net_RO.get("error_msg")
 554                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 555                 break
 556             else:
 557                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 558
 559     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 560         """
 561         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 562         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 563         :param nsr_desc_RO: nsr descriptor from RO
 564         :return: Nothing, LcmException is raised on errors
 565         """
 566         for vnf_index, db_vnfr in db_vnfrs.items():
 567             for vnf_RO in nsr_desc_RO["vnfs"]:
 568                 if vnf_RO["member_vnf_index"] != vnf_index:
 569                     continue
 570                 vnfr_update = {}
 571                 if vnf_RO.get("ip_address"):
 572                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 573                 elif not db_vnfr.get("ip-address"):
 574                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 575
 576                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 577                     vdur_RO_count_index = 0
 578                     if vdur.get("pdu-type"):
 579                         continue
 580                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 581                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 582                             continue
 583                         if vdur["count-index"] != vdur_RO_count_index:
 584                             vdur_RO_count_index += 1
 585                             continue
 586                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 587                         if vdur_RO.get("ip_address"):
 588                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 589                         else:
 590                             vdur["ip-address"] = None
 591                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 592                         vdur["name"] = vdur_RO.get("vim_name")
 593                         vdur["status"] = vdur_RO.get("status")
 594                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 595                         for ifacer in get_iterable(vdur, "interfaces"):
 596                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 597                                 if ifacer["name"] == interface_RO.get("internal_name"):
 598                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 599                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 600                                     break
 601                             else:
 602                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 603                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 604                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 605                         break
 606                     else:
 607                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 608                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 609
 610                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 611                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 612                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 613                             continue
 614                         vld["vim-id"] = net_RO.get("vim_net_id")
 615                         vld["name"] = net_RO.get("vim_name")
 616                         vld["status"] = net_RO.get("status")
 617                         vld["status-detailed"] = net_RO.get("error_msg")
 618                         vnfr_update["vld.{}".format(vld_index)] = vld
 619                         break
 620                     else:
 621                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 622                             vnf_index, vld["id"]))
 623
 624                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 625                 break
 626
 627             else:
 628                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 629
 630     async def instantiate(self, nsr_id, nslcmop_id):
 631         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 632         self.logger.debug(logging_text + "Enter")
 633         # get all needed from database
 634         start_deploy = time()
 635         db_nsr = None
 636         db_nslcmop = None
 637         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 638         db_nslcmop_update = {}
 639         nslcmop_operation_state = None
 640         db_vnfrs = {}
 641         RO_descriptor_number = 0   # number of descriptors created at RO
 642         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 643         n2vc_info = {}
 644         exc = None
 645         try:
 646             step = "Getting nslcmop={} from db".format(nslcmop_id)
 647             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 648             step = "Getting nsr={} from db".format(nsr_id)
 649             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 650             ns_params = db_nslcmop.get("operationParams")
 651             nsd = db_nsr["nsd"]
 652             nsr_name = db_nsr["name"]   # TODO short-name??
 653
 654             # look if previous tasks in process
 655             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
 656             if task_dependency:
 657                 step = db_nslcmop_update["detailed-status"] = \
 658                     "Waiting for related tasks to be completed: {}".format(task_name)
 659                 self.logger.debug(logging_text + step)
 660                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 661                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 662                 if pending:
 663                     raise LcmException("Timeout waiting related tasks to be completed")
 664
 665             step = "Getting vnfrs from db"
 666             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 667             db_vnfds_ref = {}
 668             db_vnfds = {}
 669             for vnfr in db_vnfrs_list:
 670                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 671                 vnfd_id = vnfr["vnfd-id"]
 672                 vnfd_ref = vnfr["vnfd-ref"]
 673                 if vnfd_id not in db_vnfds:
 674                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 675                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 676                     db_vnfds_ref[vnfd_ref] = vnfd
 677                     db_vnfds[vnfd_id] = vnfd
 678
 679             # Get or generates the _admin.deployed,VCA list
 680             vca_deployed_list = None
 681             vca_model_name = None
 682             if db_nsr["_admin"].get("deployed"):
 683                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 684                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 685             if vca_deployed_list is None:
 686                 vca_deployed_list = []
 687                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 688             elif isinstance(vca_deployed_list, dict):
 689                 # maintain backward compatibility. Change a dict to list at database
 690                 vca_deployed_list = list(vca_deployed_list.values())
 691                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 692
 693             db_nsr_update["detailed-status"] = "creating"
 694             db_nsr_update["operational-status"] = "init"
 695             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 696                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 697                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 698                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 699
 700             RO = ROclient.ROClient(self.loop, **self.ro_config)
 701
 702             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 703             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 704             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 705
 706             # get vnfds, instantiate at RO
 707             for c_vnf in nsd.get("constituent-vnfd", ()):
 708                 member_vnf_index = c_vnf["member-vnf-index"]
 709                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
 710                 vnfd_ref = vnfd["id"]
 711                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member-vnf-index='{}' at RO".format(
 712                     vnfd_ref, member_vnf_index)
 713                 # self.logger.debug(logging_text + step)
 714                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
 715                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
 716                 RO_descriptor_number += 1
 717
 718                 # look position at deployed.RO.vnfd if not present it will be appended at the end
 719                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
 720                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
 721                         break
 722                 else:
 723                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
 724                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
 725
 726                 # look if present
 727                 RO_update = {"member-vnf-index": member_vnf_index}
 728                 vnfd_list = await RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
 729                 if vnfd_list:
 730                     RO_update["id"] = vnfd_list[0]["uuid"]
 731                     self.logger.debug(logging_text + "vnfd='{}'  member-vnf-index='{}' exists at RO. Using RO_id={}".
 732                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
 733                 else:
 734                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
 735                                            get("additionalParamsForVnf"), nsr_id)
 736                     desc = await RO.create("vnfd", descriptor=vnfd_RO)
 737                     RO_update["id"] = desc["uuid"]
 738                     self.logger.debug(logging_text + "vnfd='{}' member-vnf-index='{}' created at RO. RO_id={}".format(
 739                         vnfd_ref, member_vnf_index, desc["uuid"]))
 740                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
 741                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
 742                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 743
 744             # create nsd at RO
 745             nsd_ref = nsd["id"]
 746             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
 747             # self.logger.debug(logging_text + step)
 748
 749             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
 750             RO_descriptor_number += 1
 751             nsd_list = await RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
 752             if nsd_list:
 753                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
 754                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
 755                     nsd_ref, RO_nsd_uuid))
 756             else:
 757                 nsd_RO = deepcopy(nsd)
 758                 nsd_RO["id"] = RO_osm_nsd_id
 759                 nsd_RO.pop("_id", None)
 760                 nsd_RO.pop("_admin", None)
 761                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
 762                     member_vnf_index = c_vnf["member-vnf-index"]
 763                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 764                 for c_vld in nsd_RO.get("vld", ()):
 765                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
 766                         member_vnf_index = cp["member-vnf-index-ref"]
 767                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
 768
 769                 desc = await RO.create("nsd", descriptor=nsd_RO)
 770                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 771                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
 772                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
 773             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 774
 775             # Crate ns at RO
 776             # if present use it unless in error status
 777             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
 778             if RO_nsr_id:
 779                 try:
 780                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
 781                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
 782                     desc = await RO.show("ns", RO_nsr_id)
 783                 except ROclient.ROClientException as e:
 784                     if e.http_code != HTTPStatus.NOT_FOUND:
 785                         raise
 786                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 787                 if RO_nsr_id:
 788                     ns_status, ns_status_info = RO.check_ns_status(desc)
 789                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 790                     if ns_status == "ERROR":
 791                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
 792                         self.logger.debug(logging_text + step)
 793                         await RO.delete("ns", RO_nsr_id)
 794                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
 795             if not RO_nsr_id:
 796                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
 797                 # self.logger.debug(logging_text + step)
 798
 799                 # check if VIM is creating and wait  look if previous tasks in process
 800                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
 801                 if task_dependency:
 802                     step = "Waiting for related tasks to be completed: {}".format(task_name)
 803                     self.logger.debug(logging_text + step)
 804                     await asyncio.wait(task_dependency, timeout=3600)
 805                 if ns_params.get("vnf"):
 806                     for vnf in ns_params["vnf"]:
 807                         if "vimAccountId" in vnf:
 808                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 809                                                                                         vnf["vimAccountId"])
 810                         if task_dependency:
 811                             step = "Waiting for related tasks to be completed: {}".format(task_name)
 812                             self.logger.debug(logging_text + step)
 813                             await asyncio.wait(task_dependency, timeout=3600)
 814
 815                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
 816
 817                 # feature 1429. Add n2vc public key to needed VMs
 818                 n2vc_key = await self.n2vc.GetPublicKey()
 819                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, [n2vc_key])
 820
 821                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
 822                 desc = await RO.create("ns", descriptor=RO_ns_params,
 823                                        name=db_nsr["name"],
 824                                        scenario=RO_nsd_uuid)
 825                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
 826                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 827                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
 828                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 829             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 830
 831             # wait until NS is ready
 832             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
 833             detailed_status_old = None
 834             self.logger.debug(logging_text + step)
 835
 836             while time() <= start_deploy + self.total_deploy_timeout:
 837                 desc = await RO.show("ns", RO_nsr_id)
 838                 ns_status, ns_status_info = RO.check_ns_status(desc)
 839                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
 840                 if ns_status == "ERROR":
 841                     raise ROclient.ROClientException(ns_status_info)
 842                 elif ns_status == "BUILD":
 843                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 844                 elif ns_status == "ACTIVE":
 845                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
 846                     try:
 847                         self.ns_update_vnfr(db_vnfrs, desc)
 848                         break
 849                     except LcmExceptionNoMgmtIP:
 850                         pass
 851                 else:
 852                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 853                 if detailed_status != detailed_status_old:
 854                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
 855                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
 856                 await asyncio.sleep(5, loop=self.loop)
 857             else:   # total_deploy_timeout
 858                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 859
 860             step = "Updating NSR"
 861             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
 862
 863             db_nsr["detailed-status"] = "Configuring vnfr"
 864             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 865
 866             # The parameters we'll need to deploy a charm
 867             number_to_configure = 0
 868
 869             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info):
 870                 """An inner function to deploy the charm from either ns, vnf or vdu
 871                 For ns both vnf_index and vdu_id are None.
 872                 For vnf only vdu_id is None
 873                 For vdu both vnf_index and vdu_id contain a value
 874                 """
 875                 if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 876                     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 877                 # Login to the VCA.
 878                 # if number_to_configure == 0:
 879                 #     self.logger.debug("Logging into N2VC...")
 880                 #     task = asyncio.ensure_future(self.n2vc.login())
 881                 #     yield from asyncio.wait_for(task, 30.0)
 882                 #     self.logger.debug("Logged into N2VC!")
 883
 884                 # # await self.n2vc.login()
 885
 886                 # Note: The charm needs to exist on disk at the location
 887                 # specified by charm_path.
 888                 descriptor = vnfd if vnf_index else nsd
 889                 base_folder = descriptor["_admin"]["storage"]
 890                 storage_params = self.fs.get_params()
 891                 charm_path = "{}{}/{}/charms/{}".format(
 892                     storage_params["path"],
 893                     base_folder["folder"],
 894                     base_folder["pkg-dir"],
 895                     proxy_charm
 896                 )
 897
 898                 # ns_name will be ignored in the current version of N2VC
 899                 # but will be implemented for the next point release.
 900                 model_name = nsr_id
 901                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 902                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 903                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 904
 905                 vca_index = len(vca_deployed_list)
 906                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 907                 # 26*26 charm in the same NS
 908                 application_name = application_name[0:48]
 909                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 910                 vca_deployed_ = {
 911                     "member-vnf-index": vnf_index,
 912                     "vdu_id": vdu_id,
 913                     "model": model_name,
 914                     "application": application_name,
 915                     "operational-status": "init",
 916                     "detailed-status": "",
 917                     "vnfd_id": vnfd_id,
 918                     "vdu_name": vdu_name,
 919                     "vdu_count_index": vdu_count_index,
 920                 }
 921                 vca_deployed_list.append(vca_deployed_)
 922                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 923                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 924
 925                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 926                                                                                                 proxy_charm))
 927                 if not n2vc_info:
 928                     n2vc_info["nsr_id"] = nsr_id
 929                     n2vc_info["nslcmop_id"] = nslcmop_id
 930                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 931                     n2vc_info["lcmOperationType"] = "instantiate"
 932                     n2vc_info["deployed"] = vca_deployed_list
 933                     n2vc_info["db_update"] = db_nsr_update
 934                 task = asyncio.ensure_future(
 935                     self.n2vc.DeployCharms(
 936                         model_name,          # The network service name
 937                         application_name,    # The application name
 938                         descriptor,          # The vnf/nsd descriptor
 939                         charm_path,          # Path to charm
 940                         charm_params,        # Runtime params, like mgmt ip
 941                         {},                  # for native charms only
 942                         self.n2vc_callback,  # Callback for status changes
 943                         n2vc_info,           # Callback parameter
 944                         None,                # Callback parameter (task)
 945                     )
 946                 )
 947                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 948                                                          n2vc_info))
 949                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 950
 951             step = "Looking for needed vnfd to configure"
 952             self.logger.debug(logging_text + step)
 953
 954             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 955                 vnfd_id = c_vnf["vnfd-id-ref"]
 956                 vnf_index = str(c_vnf["member-vnf-index"])
 957                 vnfd = db_vnfds_ref[vnfd_id]
 958
 959                 # Get additional parameters
 960                 vnfr_params = {}
 961                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 962                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 963                 for k, v in vnfr_params.items():
 964                     if isinstance(v, str) and v.startswith("!!yaml "):
 965                         vnfr_params[k] = yaml.safe_load(v[7:])
 966
 967                 # Check if this VNF has a charm configuration
 968                 vnf_config = vnfd.get("vnf-configuration")
 969                 if vnf_config and vnf_config.get("juju"):
 970                     proxy_charm = vnf_config["juju"]["charm"]
 971
 972                     if proxy_charm:
 973                         if not vca_model_name:
 974                             step = "creating VCA model name '{}'".format(nsr_id)
 975                             self.logger.debug(logging_text + step)
 976                             await self.n2vc.CreateNetworkService(nsr_id)
 977                             vca_model_name = nsr_id
 978                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 979                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 980                         step = "connecting to N2VC to configure vnf {}".format(vnf_index)
 981                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 982                         charm_params = {
 983                             "user_values": vnfr_params,
 984                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 985                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {}
 986                         }
 987
 988                         # Login to the VCA. If there are multiple calls to login(),
 989                         # subsequent calls will be a nop and return immediately.
 990                         await self.n2vc.login()
 991
 992                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 993                         number_to_configure += 1
 994
 995                 # Deploy charms for each VDU that supports one.
 996                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 997                     vdu_config = vdu.get('vdu-configuration')
 998                     proxy_charm = None
 999
1000                     if vdu_config and vdu_config.get("juju"):
1001                         proxy_charm = vdu_config["juju"]["charm"]
1002
1003                         if proxy_charm:
1004                             if not vca_model_name:
1005                                 step = "creating VCA model name"
1006                                 await self.n2vc.CreateNetworkService(nsr_id)
1007                                 vca_model_name = nsr_id
1008                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1009                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1010                             step = "connecting to N2VC to configure vdu {} from vnf {}".format(vdu["id"], vnf_index)
1011                             await self.n2vc.login()
1012                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1013                             # TODO for the moment only first vdu_id contains a charm deployed
1014                             if vdur["vdu-id-ref"] != vdu["id"]:
1015                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1016                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1017                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1018                             charm_params = {
1019                                 "user_values": vnfr_params,
1020                                 "rw_mgmt_ip": vdur["ip-address"],
1021                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1022                             }
1023                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1024                                          charm_params, n2vc_info)
1025                             number_to_configure += 1
1026
1027             # Check if this NS has a charm configuration
1028
1029             ns_config = nsd.get("ns-configuration")
1030             if ns_config and ns_config.get("juju"):
1031                 proxy_charm = ns_config["juju"]["charm"]
1032
1033                 if proxy_charm:
1034                     step = "connecting to N2VC to configure ns"
1035                     # TODO is NS magmt IP address needed?
1036
1037                     # Get additional parameters
1038                     additional_params = {}
1039                     if db_nsr.get("additionalParamsForNs"):
1040                         additional_params = db_nsr["additionalParamsForNs"].copy()
1041                     for k, v in additional_params.items():
1042                         if isinstance(v, str) and v.startswith("!!yaml "):
1043                             additional_params[k] = yaml.safe_load(v[7:])
1044
1045                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1046                     charm_params = {
1047                         "user_values": additional_params,
1048                         # "rw_mgmt_ip": db_nsr["ip-address"],
1049                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1050                     }
1051
1052                     # Login to the VCA. If there are multiple calls to login(),
1053                     # subsequent calls will be a nop and return immediately.
1054                     await self.n2vc.login()
1055                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
1056                     number_to_configure += 1
1057
1058             db_nsr_update["operational-status"] = "running"
1059             configuration_failed = False
1060             if number_to_configure:
1061                 old_status = "configuring: init: {}".format(number_to_configure)
1062                 db_nsr_update["config-status"] = old_status
1063                 db_nsr_update["detailed-status"] = old_status
1064                 db_nslcmop_update["detailed-status"] = old_status
1065
1066                 # wait until all are configured.
1067                 while time() <= start_deploy + self.total_deploy_timeout:
1068                     if db_nsr_update:
1069                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1070                     if db_nslcmop_update:
1071                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1072                     # TODO add a fake task that set n2vc_event after some time
1073                     await n2vc_info["n2vc_event"].wait()
1074                     n2vc_info["n2vc_event"].clear()
1075                     all_active = True
1076                     status_map = {}
1077                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1078                     now = time()
1079                     for vca_deployed in vca_deployed_list:
1080                         vca_status = vca_deployed["operational-status"]
1081                         if vca_status not in status_map:
1082                             # Initialize it
1083                             status_map[vca_status] = 0
1084                         status_map[vca_status] += 1
1085
1086                         if vca_status == "active":
1087                             vca_deployed.pop("time_first_error", None)
1088                             vca_deployed.pop("status_first_error", None)
1089                             continue
1090
1091                         all_active = False
1092                         if vca_status in ("error", "blocked"):
1093                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1094                             # if not first time in this status error
1095                             if not vca_deployed.get("time_first_error"):
1096                                 vca_deployed["time_first_error"] = now
1097                                 continue
1098                         if vca_deployed.get("time_first_error") and \
1099                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1100                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1101                                                    .format(vca_deployed["member-vnf-index"],
1102                                                            vca_deployed["vdu_id"], vca_status,
1103                                                            vca_deployed["detailed-status-error"]))
1104
1105                     if all_active:
1106                         break
1107                     elif n2vc_error_text:
1108                         db_nsr_update["config-status"] = "failed"
1109                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1110                         db_nsr_update["detailed-status"] = error_text
1111                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1112                         db_nslcmop_update["detailed-status"] = error_text
1113                         db_nslcmop_update["statusEnteredTime"] = time()
1114                         configuration_failed = True
1115                         break
1116                     else:
1117                         cs = "configuring: "
1118                         separator = ""
1119                         for status, num in status_map.items():
1120                             cs += separator + "{}: {}".format(status, num)
1121                             separator = ", "
1122                         if old_status != cs:
1123                             db_nsr_update["config-status"] = cs
1124                             db_nsr_update["detailed-status"] = cs
1125                             db_nslcmop_update["detailed-status"] = cs
1126                             old_status = cs
1127                 else:   # total_deploy_timeout
1128                     raise LcmException("Timeout waiting ns to be configured")
1129
1130             if not configuration_failed:
1131                 # all is done
1132                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1133                 db_nslcmop_update["statusEnteredTime"] = time()
1134                 db_nslcmop_update["detailed-status"] = "done"
1135                 db_nsr_update["config-status"] = "configured"
1136                 db_nsr_update["detailed-status"] = "done"
1137
1138             return
1139
1140         except (ROclient.ROClientException, DbException, LcmException) as e:
1141             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1142             exc = e
1143         except asyncio.CancelledError:
1144             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1145             exc = "Operation was cancelled"
1146         except Exception as e:
1147             exc = traceback.format_exc()
1148             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1149                                  exc_info=True)
1150         finally:
1151             if exc:
1152                 if db_nsr:
1153                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1154                     db_nsr_update["operational-status"] = "failed"
1155                 if db_nslcmop:
1156                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1157                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1158                     db_nslcmop_update["statusEnteredTime"] = time()
1159             try:
1160                 if db_nsr:
1161                     db_nsr_update["_admin.nslcmop"] = None
1162                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1163                 if db_nslcmop_update:
1164                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1165             except DbException as e:
1166                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1167             if nslcmop_operation_state:
1168                 try:
1169                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1170                                                                    "operationState": nslcmop_operation_state},
1171                                             loop=self.loop)
1172                 except Exception as e:
1173                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1174
1175             self.logger.debug(logging_text + "Exit")
1176             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1177
1178     async def _destroy_charm(self, model, application):
1179         """
1180         Order N2VC destroy a charm
1181         :param model:
1182         :param application:
1183         :return: True if charm does not exist. False if it exist
1184         """
1185         if not await self.n2vc.HasApplication(model, application):
1186             return True  # Already removed
1187         await self.n2vc.RemoveCharms(model, application)
1188         return False
1189
1190     async def _wait_charm_destroyed(self, model, application, timeout):
1191         """
1192         Wait until charm does not exist
1193         :param model:
1194         :param application:
1195         :param timeout:
1196         :return: True if not exist, False if timeout
1197         """
1198         while True:
1199             if not await self.n2vc.HasApplication(model, application):
1200                 return True
1201             if timeout < 0:
1202                 return False
1203             await asyncio.sleep(10)
1204             timeout -= 10
1205
1206     # Check if this VNFD has a configured terminate action
1207     def _has_terminate_config_primitive(self, vnfd):
1208         vnf_config = vnfd.get("vnf-configuration")
1209         if vnf_config and vnf_config.get("terminate-config-primitive"):
1210             return True
1211         else:
1212             return False
1213
1214     # Get a numerically sorted list of the sequences for this VNFD's terminate action
1215     def _get_terminate_config_primitive_seq_list(self, vnfd):
1216         # No need to check for existing primitive twice, already done before
1217         vnf_config = vnfd.get("vnf-configuration")
1218         seq_list = vnf_config.get("terminate-config-primitive")
1219         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1220         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1221         return seq_list_sorted
1222
1223     @staticmethod
1224     def _create_nslcmop(nsr_id, operation, params):
1225         """
1226         Creates a ns-lcm-opp content to be stored at database.
1227         :param nsr_id: internal id of the instance
1228         :param operation: instantiate, terminate, scale, action, ...
1229         :param params: user parameters for the operation
1230         :return: dictionary following SOL005 format
1231         """
1232         # Raise exception if invalid arguments
1233         if not (nsr_id and operation and params):
1234             raise LcmException(
1235                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1236         now = time()
1237         _id = str(uuid4())
1238         nslcmop = {
1239             "id": _id,
1240             "_id": _id,
1241             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1242             "operationState": "PROCESSING",
1243             "statusEnteredTime": now,
1244             "nsInstanceId": nsr_id,
1245             "lcmOperationType": operation,
1246             "startTime": now,
1247             "isAutomaticInvocation": False,
1248             "operationParams": params,
1249             "isCancelPending": False,
1250             "links": {
1251                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1252                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1253             }
1254         }
1255         return nslcmop
1256
1257     # Create a primitive with params from VNFD
1258     # - Called from terminate() before deleting instance
1259     # - Calls action() to execute the primitive
1260     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1261         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1262         db_vnfds = {}
1263         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1264         # Loop over VNFRs
1265         for vnfr in db_vnfrs_list:
1266             vnfd_id = vnfr["vnfd-id"]
1267             vnf_index = vnfr["member-vnf-index-ref"]
1268             if vnfd_id not in db_vnfds:
1269                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1270                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1271                 db_vnfds[vnfd_id] = vnfd
1272             vnfd = db_vnfds[vnfd_id]
1273             if not self._has_terminate_config_primitive(vnfd):
1274                 continue
1275             # Get the primitive's sorted sequence list
1276             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1277             for seq in seq_list:
1278                 # For each sequence in list, call terminate action
1279                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1280                     vnf_index, seq.get("name"))
1281                 self.logger.debug(logging_text + step)
1282                 # Create the primitive for each sequence
1283                 operation = "action"
1284                 # primitive, i.e. "primitive": "touch"
1285                 primitive = seq.get('name')
1286                 primitive_params = {}
1287                 params = {
1288                     "member_vnf_index": vnf_index,
1289                     "primitive": primitive,
1290                     "primitive_params": primitive_params,
1291                 }
1292                 nslcmop_primitive = self._create_nslcmop(nsr_id, operation, params)
1293                 # Get a copy of db_nslcmop 'admin' part
1294                 db_nslcmop_action = {"_admin": deepcopy(db_nslcmop["_admin"])}
1295                 # Update db_nslcmop with the primitive data
1296                 db_nslcmop_action.update(nslcmop_primitive)
1297                 # Create a new db entry for the created primitive, returns the new ID.
1298                 # (The ID is normally obtained from Kafka.)
1299                 nslcmop_terminate_action_id = self.db.create(
1300                     "nslcmops", db_nslcmop_action)
1301                 # Execute the primitive
1302                 nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1303                     nsr_id, nslcmop_terminate_action_id)
1304                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1305                 nslcmop_operation_states_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1306                 if (nslcmop_operation_state not in nslcmop_operation_states_ok):
1307                     raise LcmException(
1308                         "terminate_primitive_action for vnf_member_index={}",
1309                         " primitive={} fails with error {}".format(
1310                             vnf_index, seq.get("name"), nslcmop_operation_state_detail))
1311
1312     async def terminate(self, nsr_id, nslcmop_id):
1313         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1314         self.logger.debug(logging_text + "Enter")
1315         db_nsr = None
1316         db_nslcmop = None
1317         exc = None
1318         failed_detail = []   # annotates all failed error messages
1319         vca_time_destroy = None   # time of where destroy charm order
1320         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1321         db_nslcmop_update = {}
1322         nslcmop_operation_state = None
1323         autoremove = False  # autoremove after terminated
1324         try:
1325             step = "Getting nslcmop={} from db".format(nslcmop_id)
1326             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1327             step = "Getting nsr={} from db".format(nsr_id)
1328             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1329             # nsd = db_nsr["nsd"]
1330             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1331             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1332                 return
1333             # #TODO check if VIM is creating and wait
1334             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1335             # Call internal terminate action
1336             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1337
1338             db_nsr_update["operational-status"] = "terminating"
1339             db_nsr_update["config-status"] = "terminating"
1340
1341             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1342                 vca_model_name = nsr_deployed["VCA-model-name"]
1343                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1344                 self.logger.debug(logging_text + step)
1345                 try:
1346                     await self.n2vc.DestroyNetworkService(vca_model_name)
1347                 except NetworkServiceDoesNotExist:
1348                     pass
1349                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1350                 if nsr_deployed.get("VCA"):
1351                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1352                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1353                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1354             # for backward compatibility if charm have been created with "default"  model name delete one by one
1355             elif nsr_deployed and nsr_deployed.get("VCA"):
1356                 try:
1357                     step = "Scheduling configuration charms removing"
1358                     db_nsr_update["detailed-status"] = "Deleting charms"
1359                     self.logger.debug(logging_text + step)
1360                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1361                     # for backward compatibility
1362                     if isinstance(nsr_deployed["VCA"], dict):
1363                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1364                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1365                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1366
1367                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1368                         if vca_deployed:
1369                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1370                                 vca_deployed.clear()
1371                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1372                             else:
1373                                 vca_time_destroy = time()
1374                 except Exception as e:
1375                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1376
1377             # remove from RO
1378             RO_fail = False
1379             RO = ROclient.ROClient(self.loop, **self.ro_config)
1380
1381             # Delete ns
1382             RO_nsr_id = RO_delete_action = None
1383             if nsr_deployed and nsr_deployed.get("RO"):
1384                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1385                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1386             try:
1387                 if RO_nsr_id:
1388                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1389                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1390                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1391                     self.logger.debug(logging_text + step)
1392                     desc = await RO.delete("ns", RO_nsr_id)
1393                     RO_delete_action = desc["action_id"]
1394                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1395                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1396                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1397                 if RO_delete_action:
1398                     # wait until NS is deleted from VIM
1399                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1400                         format(RO_nsr_id, RO_delete_action)
1401                     detailed_status_old = None
1402                     self.logger.debug(logging_text + step)
1403
1404                     delete_timeout = 20 * 60   # 20 minutes
1405                     while delete_timeout > 0:
1406                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1407                                              extra_item_id=RO_delete_action)
1408                         ns_status, ns_status_info = RO.check_action_status(desc)
1409                         if ns_status == "ERROR":
1410                             raise ROclient.ROClientException(ns_status_info)
1411                         elif ns_status == "BUILD":
1412                             detailed_status = step + "; {}".format(ns_status_info)
1413                         elif ns_status == "ACTIVE":
1414                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1415                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1416                             break
1417                         else:
1418                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1419                         if detailed_status != detailed_status_old:
1420                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1421                                 db_nsr_update["detailed-status"] = detailed_status
1422                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1423                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1424                         await asyncio.sleep(5, loop=self.loop)
1425                         delete_timeout -= 5
1426                     else:  # delete_timeout <= 0:
1427                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1428
1429             except ROclient.ROClientException as e:
1430                 if e.http_code == 404:  # not found
1431                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1432                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1433                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1434                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1435                 elif e.http_code == 409:   # conflict
1436                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1437                     self.logger.debug(logging_text + failed_detail[-1])
1438                     RO_fail = True
1439                 else:
1440                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1441                     self.logger.error(logging_text + failed_detail[-1])
1442                     RO_fail = True
1443
1444             # Delete nsd
1445             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1446                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1447                 try:
1448                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1449                         "Deleting nsd at RO"
1450                     await RO.delete("nsd", RO_nsd_id)
1451                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1452                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1453                 except ROclient.ROClientException as e:
1454                     if e.http_code == 404:  # not found
1455                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1456                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1457                     elif e.http_code == 409:   # conflict
1458                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1459                         self.logger.debug(logging_text + failed_detail[-1])
1460                         RO_fail = True
1461                     else:
1462                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1463                         self.logger.error(logging_text + failed_detail[-1])
1464                         RO_fail = True
1465
1466             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1467                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1468                     if not vnf_deployed or not vnf_deployed["id"]:
1469                         continue
1470                     try:
1471                         RO_vnfd_id = vnf_deployed["id"]
1472                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1473                             "Deleting member-vnf-index={} RO_vnfd_id={} from RO".format(
1474                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1475                         await RO.delete("vnfd", RO_vnfd_id)
1476                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1477                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1478                     except ROclient.ROClientException as e:
1479                         if e.http_code == 404:  # not found
1480                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1481                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1482                         elif e.http_code == 409:   # conflict
1483                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1484                             self.logger.debug(logging_text + failed_detail[-1])
1485                         else:
1486                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1487                             self.logger.error(logging_text + failed_detail[-1])
1488
1489             # wait until charm deleted
1490             if vca_time_destroy:
1491                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1492                     "Waiting for deletion of configuration charms"
1493                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1494                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1495                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1496                     if not vca_deployed:
1497                         continue
1498                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1499                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1500                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1501                                                             timeout):
1502                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1503                             vca_deployed["application"]))
1504                     else:
1505                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1506
1507             if failed_detail:
1508                 self.logger.error(logging_text + " ;".join(failed_detail))
1509                 db_nsr_update["operational-status"] = "failed"
1510                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1511                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1512                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1513                 db_nslcmop_update["statusEnteredTime"] = time()
1514             else:
1515                 db_nsr_update["operational-status"] = "terminated"
1516                 db_nsr_update["detailed-status"] = "Done"
1517                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1518                 db_nslcmop_update["detailed-status"] = "Done"
1519                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1520                 db_nslcmop_update["statusEnteredTime"] = time()
1521                 if db_nslcmop["operationParams"].get("autoremove"):
1522                     autoremove = True
1523
1524         except (ROclient.ROClientException, DbException, LcmException) as e:
1525             self.logger.error(logging_text + "Exit Exception {}".format(e))
1526             exc = e
1527         except asyncio.CancelledError:
1528             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1529             exc = "Operation was cancelled"
1530         except Exception as e:
1531             exc = traceback.format_exc()
1532             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1533         finally:
1534             if exc and db_nslcmop:
1535                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1536                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1537                 db_nslcmop_update["statusEnteredTime"] = time()
1538             try:
1539                 if db_nslcmop and db_nslcmop_update:
1540                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1541                 if db_nsr:
1542                     db_nsr_update["_admin.nslcmop"] = None
1543                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1544             except DbException as e:
1545                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1546             if nslcmop_operation_state:
1547                 try:
1548                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1549                                                                  "operationState": nslcmop_operation_state,
1550                                                                  "autoremove": autoremove},
1551                                             loop=self.loop)
1552                 except Exception as e:
1553                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1554             self.logger.debug(logging_text + "Exit")
1555             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1556
1557     @staticmethod
1558     def _map_primitive_params(primitive_desc, params, instantiation_params):
1559         """
1560         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1561         The default-value is used. If it is between < > it look for a value at instantiation_params
1562         :param primitive_desc: portion of VNFD/NSD that describes primitive
1563         :param params: Params provided by user
1564         :param instantiation_params: Instantiation params provided by user
1565         :return: a dictionary with the calculated params
1566         """
1567         calculated_params = {}
1568         for parameter in primitive_desc.get("parameter", ()):
1569             param_name = parameter["name"]
1570             if param_name in params:
1571                 calculated_params[param_name] = params[param_name]
1572             elif "default-value" in parameter:
1573                 calculated_params[param_name] = parameter["default-value"]
1574                 if isinstance(parameter["default-value"], str) and parameter["default-value"].startswith("<") and \
1575                         parameter["default-value"].endswith(">"):
1576                     if parameter["default-value"][1:-1] in instantiation_params:
1577                         calculated_params[param_name] = instantiation_params[parameter["default-value"][1:-1]]
1578                     else:
1579                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1580                                            format(parameter["default-value"], primitive_desc["name"]))
1581             else:
1582                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1583                                    format(param_name, primitive_desc["name"]))
1584
1585             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1586                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1587                                                                width=256)
1588             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1589                 calculated_params[param_name] = calculated_params[param_name][7:]
1590         return calculated_params
1591
1592     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1593                                     primitive, primitive_params):
1594         start_primitive_time = time()
1595         try:
1596             for vca_deployed in db_deployed["VCA"]:
1597                 if not vca_deployed:
1598                     continue
1599                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
1600                     continue
1601                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
1602                     continue
1603                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
1604                     continue
1605                 break
1606             else:
1607                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
1608                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
1609             model_name = vca_deployed.get("model")
1610             application_name = vca_deployed.get("application")
1611             if not model_name or not application_name:
1612                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
1613                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
1614                                                                        vdu_count_index))
1615             if vca_deployed["operational-status"] != "active":
1616                 raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
1617                     member_vnf_index, vdu_id, vca_deployed["operational-status"]))
1618             callback = None  # self.n2vc_callback
1619             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
1620             await self.n2vc.login()
1621             primitive_id = await self.n2vc.ExecutePrimitive(
1622                 model_name,
1623                 application_name,
1624                 primitive,
1625                 callback,
1626                 *callback_args,
1627                 **primitive_params
1628             )
1629             while time() - start_primitive_time < self.timeout_primitive:
1630                 primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
1631                 if primitive_result_ in ("running", "pending"):
1632                     pass
1633                 elif primitive_result_ in ("completed", "failed"):
1634                     primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
1635                     detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
1636                     break
1637                 else:
1638                     detailed_result = "Invalid N2VC.GetPrimitiveStatus = {} obtained".format(primitive_result_)
1639                     primitive_result = "FAILED"
1640                     break
1641                 await asyncio.sleep(5)
1642             else:
1643                 raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
1644             return primitive_result, detailed_result
1645         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
1646             return "FAILED", str(e)
1647
1648     async def action(self, nsr_id, nslcmop_id):
1649         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
1650         self.logger.debug(logging_text + "Enter")
1651         # get all needed from database
1652         db_nsr = None
1653         db_nslcmop = None
1654         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1655         db_nslcmop_update = {}
1656         nslcmop_operation_state = None
1657         nslcmop_operation_state_detail = None
1658         exc = None
1659         try:
1660             step = "Getting information from database"
1661             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1662             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1663
1664             nsr_deployed = db_nsr["_admin"].get("deployed")
1665             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
1666             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1667             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1668             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1669
1670             if vnf_index:
1671                 step = "Getting vnfr from database"
1672                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1673                 step = "Getting vnfd from database"
1674                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1675             else:
1676                 if db_nsr.get("nsd"):
1677                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
1678                 else:
1679                     step = "Getting nsd from database"
1680                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
1681
1682             # look if previous tasks in process
1683             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1684             if task_dependency:
1685                 step = db_nslcmop_update["detailed-status"] = \
1686                     "Waiting for related tasks to be completed: {}".format(task_name)
1687                 self.logger.debug(logging_text + step)
1688                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1689                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1690                 if pending:
1691                     raise LcmException("Timeout waiting related tasks to be completed")
1692
1693             # for backward compatibility
1694             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1695                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1696                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1697                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1698
1699             primitive = db_nslcmop["operationParams"]["primitive"]
1700             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
1701
1702             # look for primitive
1703             config_primitive_desc = None
1704             if vdu_id:
1705                 for vdu in get_iterable(db_vnfd, "vdu"):
1706                     if vdu_id == vdu["id"]:
1707                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
1708                             if config_primitive["name"] == primitive:
1709                                 config_primitive_desc = config_primitive
1710                                 break
1711             elif vnf_index:
1712                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1713                     if config_primitive["name"] == primitive:
1714                         config_primitive_desc = config_primitive
1715                         break
1716             else:
1717                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
1718                     if config_primitive["name"] == primitive:
1719                         config_primitive_desc = config_primitive
1720                         break
1721
1722             if not config_primitive_desc:
1723                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
1724                                    format(primitive))
1725
1726             desc_params = {}
1727             if vnf_index:
1728                 if db_vnfr.get("additionalParamsForVnf"):
1729                     desc_params.update(db_vnfr["additionalParamsForVnf"])
1730             else:
1731                 if db_nsr.get("additionalParamsForVnf"):
1732                     desc_params.update(db_nsr["additionalParamsForNs"])
1733
1734             # TODO check if ns is in a proper status
1735             result, result_detail = await self._ns_execute_primitive(
1736                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1737                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
1738             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
1739             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
1740             db_nslcmop_update["statusEnteredTime"] = time()
1741             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
1742             return  # database update is called inside finally
1743
1744         except (DbException, LcmException) as e:
1745             self.logger.error(logging_text + "Exit Exception {}".format(e))
1746             exc = e
1747         except asyncio.CancelledError:
1748             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1749             exc = "Operation was cancelled"
1750         except Exception as e:
1751             exc = traceback.format_exc()
1752             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
1753         finally:
1754             if exc and db_nslcmop:
1755                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
1756                     "FAILED {}: {}".format(step, exc)
1757                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1758                 db_nslcmop_update["statusEnteredTime"] = time()
1759             try:
1760                 if db_nslcmop_update:
1761                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1762                 if db_nsr:
1763                     db_nsr_update["_admin.nslcmop"] = None
1764                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1765             except DbException as e:
1766                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1767             self.logger.debug(logging_text + "Exit")
1768             if nslcmop_operation_state:
1769                 try:
1770                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1771                                                                "operationState": nslcmop_operation_state},
1772                                             loop=self.loop)
1773                 except Exception as e:
1774                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1775             self.logger.debug(logging_text + "Exit")
1776             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
1777             return nslcmop_operation_state, nslcmop_operation_state_detail
1778
1779     async def scale(self, nsr_id, nslcmop_id):
1780         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
1781         self.logger.debug(logging_text + "Enter")
1782         # get all needed from database
1783         db_nsr = None
1784         db_nslcmop = None
1785         db_nslcmop_update = {}
1786         nslcmop_operation_state = None
1787         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1788         exc = None
1789         # in case of error, indicates what part of scale was failed to put nsr at error status
1790         scale_process = None
1791         old_operational_status = ""
1792         old_config_status = ""
1793         vnfr_scaled = False
1794         try:
1795             step = "Getting nslcmop from database"
1796             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1797             step = "Getting nsr from database"
1798             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1799
1800             old_operational_status = db_nsr["operational-status"]
1801             old_config_status = db_nsr["config-status"]
1802
1803             # look if previous tasks in process
1804             task_name, task_dependency = self.lcm_tasks.lookfor_related("ns", nsr_id, nslcmop_id)
1805             if task_dependency:
1806                 step = db_nslcmop_update["detailed-status"] = \
1807                     "Waiting for related tasks to be completed: {}".format(task_name)
1808                 self.logger.debug(logging_text + step)
1809                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1810                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
1811                 if pending:
1812                     raise LcmException("Timeout waiting related tasks to be completed")
1813
1814             step = "Parsing scaling parameters"
1815             db_nsr_update["operational-status"] = "scaling"
1816             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1817             nsr_deployed = db_nsr["_admin"].get("deployed")
1818             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
1819             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
1820             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
1821             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
1822             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
1823
1824             # for backward compatibility
1825             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
1826                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1827                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1828                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1829
1830             step = "Getting vnfr from database"
1831             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
1832             step = "Getting vnfd from database"
1833             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
1834             step = "Getting scaling-group-descriptor"
1835             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
1836                 if scaling_descriptor["name"] == scaling_group:
1837                     break
1838             else:
1839                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
1840                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
1841             # cooldown_time = 0
1842             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
1843             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
1844             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
1845             #         break
1846
1847             # TODO check if ns is in a proper status
1848             step = "Sending scale order to RO"
1849             nb_scale_op = 0
1850             if not db_nsr["_admin"].get("scaling-group"):
1851                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
1852                 admin_scale_index = 0
1853             else:
1854                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
1855                     if admin_scale_info["name"] == scaling_group:
1856                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
1857                         break
1858                 else:  # not found, set index one plus last element and add new entry with the name
1859                     admin_scale_index += 1
1860                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
1861             RO_scaling_info = []
1862             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
1863             if scaling_type == "SCALE_OUT":
1864                 # count if max-instance-count is reached
1865                 if "max-instance-count" in scaling_descriptor and scaling_descriptor["max-instance-count"] is not None:
1866                     max_instance_count = int(scaling_descriptor["max-instance-count"])
1867                     if nb_scale_op >= max_instance_count:
1868                         raise LcmException("reached the limit of {} (max-instance-count) scaling-out operations for the"
1869                                            " scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1870                 nb_scale_op = nb_scale_op + 1
1871                 vdu_scaling_info["scaling_direction"] = "OUT"
1872                 vdu_scaling_info["vdu-create"] = {}
1873                 for vdu_scale_info in scaling_descriptor["vdu"]:
1874                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1875                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
1876                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1877             elif scaling_type == "SCALE_IN":
1878                 # count if min-instance-count is reached
1879                 min_instance_count = 0
1880                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
1881                     min_instance_count = int(scaling_descriptor["min-instance-count"])
1882                 if nb_scale_op <= min_instance_count:
1883                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
1884                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
1885                 nb_scale_op = nb_scale_op - 1
1886                 vdu_scaling_info["scaling_direction"] = "IN"
1887                 vdu_scaling_info["vdu-delete"] = {}
1888                 for vdu_scale_info in scaling_descriptor["vdu"]:
1889                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
1890                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
1891                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
1892
1893             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
1894             vdu_create = vdu_scaling_info.get("vdu-create")
1895             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
1896             if vdu_scaling_info["scaling_direction"] == "IN":
1897                 for vdur in reversed(db_vnfr["vdur"]):
1898                     if vdu_delete.get(vdur["vdu-id-ref"]):
1899                         vdu_delete[vdur["vdu-id-ref"]] -= 1
1900                         vdu_scaling_info["vdu"].append({
1901                             "name": vdur["name"],
1902                             "vdu_id": vdur["vdu-id-ref"],
1903                             "interface": []
1904                         })
1905                         for interface in vdur["interfaces"]:
1906                             vdu_scaling_info["vdu"][-1]["interface"].append({
1907                                 "name": interface["name"],
1908                                 "ip_address": interface["ip-address"],
1909                                 "mac_address": interface.get("mac-address"),
1910                             })
1911                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
1912
1913             # execute primitive service PRE-SCALING
1914             step = "Executing pre-scale vnf-config-primitive"
1915             if scaling_descriptor.get("scaling-config-action"):
1916                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
1917                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
1918                             and scaling_type == "SCALE_IN":
1919                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
1920                         step = db_nslcmop_update["detailed-status"] = \
1921                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
1922
1923                         # look for primitive
1924                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
1925                             if config_primitive["name"] == vnf_config_primitive:
1926                                 break
1927                         else:
1928                             raise LcmException(
1929                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
1930                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
1931                                 "primitive".format(scaling_group, config_primitive))
1932
1933                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
1934                         if db_vnfr.get("additionalParamsForVnf"):
1935                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
1936
1937                         scale_process = "VCA"
1938                         db_nsr_update["config-status"] = "configuring pre-scaling"
1939                         result, result_detail = await self._ns_execute_primitive(
1940                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
1941                             self._map_primitive_params(config_primitive, {}, vnfr_params))
1942                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
1943                             vnf_config_primitive, result, result_detail))
1944                         if result == "FAILED":
1945                             raise LcmException(result_detail)
1946                         db_nsr_update["config-status"] = old_config_status
1947                         scale_process = None
1948
1949             if RO_scaling_info:
1950                 scale_process = "RO"
1951                 RO = ROclient.ROClient(self.loop, **self.ro_config)
1952                 RO_desc = await RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
1953                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
1954                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
1955                 # wait until ready
1956                 RO_nslcmop_id = RO_desc["instance_action_id"]
1957                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
1958
1959                 RO_task_done = False
1960                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
1961                 detailed_status_old = None
1962                 self.logger.debug(logging_text + step)
1963
1964                 deployment_timeout = 1 * 3600   # One hour
1965                 while deployment_timeout > 0:
1966                     if not RO_task_done:
1967                         desc = await RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1968                                              extra_item_id=RO_nslcmop_id)
1969                         ns_status, ns_status_info = RO.check_action_status(desc)
1970                         if ns_status == "ERROR":
1971                             raise ROclient.ROClientException(ns_status_info)
1972                         elif ns_status == "BUILD":
1973                             detailed_status = step + "; {}".format(ns_status_info)
1974                         elif ns_status == "ACTIVE":
1975                             RO_task_done = True
1976                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1977                             self.logger.debug(logging_text + step)
1978                         else:
1979                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1980                     else:
1981                         desc = await RO.show("ns", RO_nsr_id)
1982                         ns_status, ns_status_info = RO.check_ns_status(desc)
1983                         if ns_status == "ERROR":
1984                             raise ROclient.ROClientException(ns_status_info)
1985                         elif ns_status == "BUILD":
1986                             detailed_status = step + "; {}".format(ns_status_info)
1987                         elif ns_status == "ACTIVE":
1988                             step = detailed_status = \
1989                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
1990                             if not vnfr_scaled:
1991                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
1992                                 vnfr_scaled = True
1993                             try:
1994                                 desc = await RO.show("ns", RO_nsr_id)
1995                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
1996                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
1997                                 break
1998                             except LcmExceptionNoMgmtIP:
1999                                 pass
2000                         else:
2001                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2002                     if detailed_status != detailed_status_old:
2003                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2004                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2005
2006                     await asyncio.sleep(5, loop=self.loop)
2007                     deployment_timeout -= 5
2008                 if deployment_timeout <= 0:
2009                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2010
2011                 # update VDU_SCALING_INFO with the obtained ip_addresses
2012                 if vdu_scaling_info["scaling_direction"] == "OUT":
2013                     for vdur in reversed(db_vnfr["vdur"]):
2014                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2015                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2016                             vdu_scaling_info["vdu"].append({
2017                                 "name": vdur["name"],
2018                                 "vdu_id": vdur["vdu-id-ref"],
2019                                 "interface": []
2020                             })
2021                             for interface in vdur["interfaces"]:
2022                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2023                                     "name": interface["name"],
2024                                     "ip_address": interface["ip-address"],
2025                                     "mac_address": interface.get("mac-address"),
2026                                 })
2027                     del vdu_scaling_info["vdu-create"]
2028
2029             scale_process = None
2030             if db_nsr_update:
2031                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2032
2033             # execute primitive service POST-SCALING
2034             step = "Executing post-scale vnf-config-primitive"
2035             if scaling_descriptor.get("scaling-config-action"):
2036                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2037                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2038                             and scaling_type == "SCALE_OUT":
2039                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2040                         step = db_nslcmop_update["detailed-status"] = \
2041                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2042
2043                         vnfr_params = {"<VDU_SCALE_INFO>": vdu_scaling_info}
2044                         if db_vnfr.get("additionalParamsForVnf"):
2045                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2046
2047                         # look for primitive
2048                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2049                             if config_primitive["name"] == vnf_config_primitive:
2050                                 break
2051                         else:
2052                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2053                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2054                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2055                                                                                                      config_primitive))
2056                         scale_process = "VCA"
2057                         db_nsr_update["config-status"] = "configuring post-scaling"
2058
2059                         result, result_detail = await self._ns_execute_primitive(
2060                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2061                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2062                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2063                             vnf_config_primitive, result, result_detail))
2064                         if result == "FAILED":
2065                             raise LcmException(result_detail)
2066                         db_nsr_update["config-status"] = old_config_status
2067                         scale_process = None
2068
2069             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2070             db_nslcmop_update["statusEnteredTime"] = time()
2071             db_nslcmop_update["detailed-status"] = "done"
2072             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2073             db_nsr_update["operational-status"] = old_operational_status
2074             db_nsr_update["config-status"] = old_config_status
2075             return
2076         except (ROclient.ROClientException, DbException, LcmException) as e:
2077             self.logger.error(logging_text + "Exit Exception {}".format(e))
2078             exc = e
2079         except asyncio.CancelledError:
2080             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2081             exc = "Operation was cancelled"
2082         except Exception as e:
2083             exc = traceback.format_exc()
2084             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2085         finally:
2086             if exc:
2087                 if db_nslcmop:
2088                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2089                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2090                     db_nslcmop_update["statusEnteredTime"] = time()
2091                 if db_nsr:
2092                     db_nsr_update["operational-status"] = old_operational_status
2093                     db_nsr_update["config-status"] = old_config_status
2094                     db_nsr_update["detailed-status"] = ""
2095                     db_nsr_update["_admin.nslcmop"] = None
2096                     if scale_process:
2097                         if "VCA" in scale_process:
2098                             db_nsr_update["config-status"] = "failed"
2099                         if "RO" in scale_process:
2100                             db_nsr_update["operational-status"] = "failed"
2101                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2102                                                                                                      exc)
2103             try:
2104                 if db_nslcmop and db_nslcmop_update:
2105                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2106                 if db_nsr:
2107                     db_nsr_update["_admin.nslcmop"] = None
2108                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2109             except DbException as e:
2110                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2111             if nslcmop_operation_state:
2112                 try:
2113                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2114                                                              "operationState": nslcmop_operation_state},
2115                                             loop=self.loop)
2116                     # if cooldown_time:
2117                     #     await asyncio.sleep(cooldown_time)
2118                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2119                 except Exception as e:
2120                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2121             self.logger.debug(logging_text + "Exit")
2122             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")