osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 106
 107     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 108         """
 109         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 110         :param vnfd: input vnfd
 111         :param new_id: overrides vnf id if provided
 112         :param additionalParams: Instantiation params for VNFs provided
 113         :param nsrId: Id of the NSR
 114         :return: copy of vnfd
 115         """
 116         try:
 117             vnfd_RO = deepcopy(vnfd)
 118             # remove unused by RO configuration, monitoring, scaling and internal keys
 119             vnfd_RO.pop("_id", None)
 120             vnfd_RO.pop("_admin", None)
 121             vnfd_RO.pop("vnf-configuration", None)
 122             vnfd_RO.pop("monitoring-param", None)
 123             vnfd_RO.pop("scaling-group-descriptor", None)
 124             if new_id:
 125                 vnfd_RO["id"] = new_id
 126
 127             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 128             for vdu in get_iterable(vnfd_RO, "vdu"):
 129                 cloud_init_file = None
 130                 if vdu.get("cloud-init-file"):
 131                     base_folder = vnfd["_admin"]["storage"]
 132                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 133                                                                    vdu["cloud-init-file"])
 134                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 135                         cloud_init_content = ci_file.read()
 136                     vdu.pop("cloud-init-file", None)
 137                 elif vdu.get("cloud-init"):
 138                     cloud_init_content = vdu["cloud-init"]
 139                 else:
 140                     continue
 141
 142                 env = Environment()
 143                 ast = env.parse(cloud_init_content)
 144                 mandatory_vars = meta.find_undeclared_variables(ast)
 145                 if mandatory_vars:
 146                     for var in mandatory_vars:
 147                         if not additionalParams or var not in additionalParams.keys():
 148                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 149                                                "file, must be provided in the instantiation parameters inside the "
 150                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 151                 template = Template(cloud_init_content)
 152                 cloud_init_content = template.render(additionalParams or {})
 153                 vdu["cloud-init"] = cloud_init_content
 154
 155             return vnfd_RO
 156         except FsException as e:
 157             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 158                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 159         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 160             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 161                                format(vnfd["id"], vdu["id"], e))
 162
 163     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 164         """
 165         Callback both for charm status change and task completion
 166         :param model_name: Charm model name
 167         :param application_name: Charm application name
 168         :param status: Can be
 169             - blocked: The unit needs manual intervention
 170             - maintenance: The unit is actively deploying/configuring
 171             - waiting: The unit is waiting for another charm to be ready
 172             - active: The unit is deployed, configured, and ready
 173             - error: The charm has failed and needs attention.
 174             - terminated: The charm has been destroyed
 175             - removing,
 176             - removed
 177         :param message: detailed message error
 178         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 179             nsr_id:
 180             nslcmop_id:
 181             lcmOperationType: currently "instantiate"
 182             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 183             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 184             n2vc_event: event used to notify instantiation task that some change has been produced
 185         :param task: None for charm status change, or task for completion task callback
 186         :return:
 187         """
 188         try:
 189             nsr_id = n2vc_info["nsr_id"]
 190             deployed = n2vc_info["deployed"]
 191             db_nsr_update = n2vc_info["db_update"]
 192             nslcmop_id = n2vc_info["nslcmop_id"]
 193             ns_operation = n2vc_info["lcmOperationType"]
 194             n2vc_event = n2vc_info["n2vc_event"]
 195             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 196                                                                                     application_name)
 197             for vca_index, vca_deployed in enumerate(deployed):
 198                 if not vca_deployed:
 199                     continue
 200                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 201                     break
 202             else:
 203                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 204                                   format(model_name))
 205                 return
 206             if task:
 207                 if task.cancelled():
 208                     self.logger.debug(logging_text + " task Cancelled")
 209                     vca_deployed['operational-status'] = "error"
 210                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 211                     vca_deployed['detailed-status'] = "Task Cancelled"
 212                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 213
 214                 elif task.done():
 215                     exc = task.exception()
 216                     if exc:
 217                         self.logger.error(logging_text + " task Exception={}".format(exc))
 218                         vca_deployed['operational-status'] = "error"
 219                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 220                         vca_deployed['detailed-status'] = str(exc)
 221                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 222                     else:
 223                         self.logger.debug(logging_text + " task Done")
 224                         # task is Done, but callback is still ongoing. So ignore
 225                         return
 226             elif status:
 227                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 228                 if vca_deployed['operational-status'] == status:
 229                     return  # same status, ignore
 230                 vca_deployed['operational-status'] = status
 231                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 232                 vca_deployed['detailed-status'] = str(message)
 233                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 234             else:
 235                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 236                 return
 237             # wake up instantiate task
 238             n2vc_event.set()
 239         except Exception as e:
 240             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 241
 242     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 243         """
 244         Creates a RO ns descriptor from OSM ns_instantiate params
 245         :param ns_params: OSM instantiate params
 246         :return: The RO ns descriptor
 247         """
 248         vim_2_RO = {}
 249         wim_2_RO = {}
 250         # TODO feature 1417: Check that no instantiation is set over PDU
 251         # check if PDU forces a concrete vim-network-id and add it
 252         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 253
 254         def vim_account_2_RO(vim_account):
 255             if vim_account in vim_2_RO:
 256                 return vim_2_RO[vim_account]
 257
 258             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 259             if db_vim["_admin"]["operationalState"] != "ENABLED":
 260                 raise LcmException("VIM={} is not available. operationalState={}".format(
 261                     vim_account, db_vim["_admin"]["operationalState"]))
 262             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 263             vim_2_RO[vim_account] = RO_vim_id
 264             return RO_vim_id
 265
 266         def wim_account_2_RO(wim_account):
 267             if isinstance(wim_account, str):
 268                 if wim_account in wim_2_RO:
 269                     return wim_2_RO[wim_account]
 270
 271                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 272                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 273                     raise LcmException("WIM={} is not available. operationalState={}".format(
 274                         wim_account, db_wim["_admin"]["operationalState"]))
 275                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 276                 wim_2_RO[wim_account] = RO_wim_id
 277                 return RO_wim_id
 278             else:
 279                 return wim_account
 280
 281         def ip_profile_2_RO(ip_profile):
 282             RO_ip_profile = deepcopy((ip_profile))
 283             if "dns-server" in RO_ip_profile:
 284                 if isinstance(RO_ip_profile["dns-server"], list):
 285                     RO_ip_profile["dns-address"] = []
 286                     for ds in RO_ip_profile.pop("dns-server"):
 287                         RO_ip_profile["dns-address"].append(ds['address'])
 288                 else:
 289                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 290             if RO_ip_profile.get("ip-version") == "ipv4":
 291                 RO_ip_profile["ip-version"] = "IPv4"
 292             if RO_ip_profile.get("ip-version") == "ipv6":
 293                 RO_ip_profile["ip-version"] = "IPv6"
 294             if "dhcp-params" in RO_ip_profile:
 295                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 296             return RO_ip_profile
 297
 298         if not ns_params:
 299             return None
 300         RO_ns_params = {
 301             # "name": ns_params["nsName"],
 302             # "description": ns_params.get("nsDescription"),
 303             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 304             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 305             # "scenario": ns_params["nsdId"],
 306         }
 307         if n2vc_key_list:
 308             for vnfd_ref, vnfd in vnfd_dict.items():
 309                 vdu_needed_access = []
 310                 mgmt_cp = None
 311                 if vnfd.get("vnf-configuration"):
 312                     if vnfd.get("mgmt-interface"):
 313                         if vnfd["mgmt-interface"].get("vdu-id"):
 314                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 315                         elif vnfd["mgmt-interface"].get("cp"):
 316                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 317
 318                 for vdu in vnfd.get("vdu", ()):
 319                     if vdu.get("vdu-configuration"):
 320                         vdu_needed_access.append(vdu["id"])
 321                     elif mgmt_cp:
 322                         for vdu_interface in vdu.get("interface"):
 323                             if vdu_interface.get("external-connection-point-ref") and \
 324                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 325                                 vdu_needed_access.append(vdu["id"])
 326                                 mgmt_cp = None
 327                                 break
 328
 329                 if vdu_needed_access:
 330                     for vnf_member in nsd.get("constituent-vnfd"):
 331                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 332                             continue
 333                         for vdu in vdu_needed_access:
 334                             populate_dict(RO_ns_params,
 335                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 336                                           n2vc_key_list)
 337
 338         if ns_params.get("vduImage"):
 339             RO_ns_params["vduImage"] = ns_params["vduImage"]
 340
 341         if ns_params.get("ssh_keys"):
 342             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 343         for vnf_params in get_iterable(ns_params, "vnf"):
 344             for constituent_vnfd in nsd["constituent-vnfd"]:
 345                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 346                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 347                     break
 348             else:
 349                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 350                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 351             if vnf_params.get("vimAccountId"):
 352                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 353                               vim_account_2_RO(vnf_params["vimAccountId"]))
 354
 355             for vdu_params in get_iterable(vnf_params, "vdu"):
 356                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 357                 if vdu_params.get("volume"):
 358                     for volume_params in vdu_params["volume"]:
 359                         if volume_params.get("vim-volume-id"):
 360                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 361                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 362                                           volume_params["vim-volume-id"])
 363                 if vdu_params.get("interface"):
 364                     for interface_params in vdu_params["interface"]:
 365                         if interface_params.get("ip-address"):
 366                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 367                                                          vdu_params["id"], "interfaces", interface_params["name"],
 368                                                          "ip_address"),
 369                                           interface_params["ip-address"])
 370                         if interface_params.get("mac-address"):
 371                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 372                                                          vdu_params["id"], "interfaces", interface_params["name"],
 373                                                          "mac_address"),
 374                                           interface_params["mac-address"])
 375                         if interface_params.get("floating-ip-required"):
 376                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 377                                                          vdu_params["id"], "interfaces", interface_params["name"],
 378                                                          "floating-ip"),
 379                                           interface_params["floating-ip-required"])
 380
 381             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 382                 if internal_vld_params.get("vim-network-name"):
 383                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 384                                                  internal_vld_params["name"], "vim-network-name"),
 385                                   internal_vld_params["vim-network-name"])
 386                 if internal_vld_params.get("vim-network-id"):
 387                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 388                                                  internal_vld_params["name"], "vim-network-id"),
 389                                   internal_vld_params["vim-network-id"])
 390                 if internal_vld_params.get("ip-profile"):
 391                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 392                                                  internal_vld_params["name"], "ip-profile"),
 393                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 394
 395                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 396                     # look for interface
 397                     iface_found = False
 398                     for vdu_descriptor in vnf_descriptor["vdu"]:
 399                         for vdu_interface in vdu_descriptor["interface"]:
 400                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 401                                 if icp_params.get("ip-address"):
 402                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 403                                                                  vdu_descriptor["id"], "interfaces",
 404                                                                  vdu_interface["name"], "ip_address"),
 405                                                   icp_params["ip-address"])
 406
 407                                 if icp_params.get("mac-address"):
 408                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 409                                                                  vdu_descriptor["id"], "interfaces",
 410                                                                  vdu_interface["name"], "mac_address"),
 411                                                   icp_params["mac-address"])
 412                                 iface_found = True
 413                                 break
 414                         if iface_found:
 415                             break
 416                     else:
 417                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 418                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 419                                            "connection-point".format(vnf_params["member-vnf-index"],
 420                                                                      icp_params["id-ref"]))
 421
 422         for vld_params in get_iterable(ns_params, "vld"):
 423             if "ip-profile" in vld_params:
 424                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 425                               ip_profile_2_RO(vld_params["ip-profile"]))
 426
 427             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 428                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 429                               wim_account_2_RO(vld_params["wimAccountId"])),
 430             if vld_params.get("vim-network-name"):
 431                 RO_vld_sites = []
 432                 if isinstance(vld_params["vim-network-name"], dict):
 433                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 434                         RO_vld_sites.append({
 435                             "netmap-use": vim_net,
 436                             "datacenter": vim_account_2_RO(vim_account)
 437                         })
 438                 else:  # isinstance str
 439                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 440                 if RO_vld_sites:
 441                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 442             if vld_params.get("vim-network-id"):
 443                 RO_vld_sites = []
 444                 if isinstance(vld_params["vim-network-id"], dict):
 445                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 446                         RO_vld_sites.append({
 447                             "netmap-use": vim_net,
 448                             "datacenter": vim_account_2_RO(vim_account)
 449                         })
 450                 else:  # isinstance str
 451                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 452                 if RO_vld_sites:
 453                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 454             if vld_params.get("ns-net"):
 455                 if isinstance(vld_params["ns-net"], dict):
 456                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 457                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 458                 if RO_vld_ns_net:
 459                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 460             if "vnfd-connection-point-ref" in vld_params:
 461                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 462                     # look for interface
 463                     for constituent_vnfd in nsd["constituent-vnfd"]:
 464                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 465                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 466                             break
 467                     else:
 468                         raise LcmException(
 469                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 470                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 471                     match_cp = False
 472                     for vdu_descriptor in vnf_descriptor["vdu"]:
 473                         for interface_descriptor in vdu_descriptor["interface"]:
 474                             if interface_descriptor.get("external-connection-point-ref") == \
 475                                     cp_params["vnfd-connection-point-ref"]:
 476                                 match_cp = True
 477                                 break
 478                         if match_cp:
 479                             break
 480                     else:
 481                         raise LcmException(
 482                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 483                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 484                                 cp_params["member-vnf-index-ref"],
 485                                 cp_params["vnfd-connection-point-ref"],
 486                                 vnf_descriptor["id"]))
 487                     if cp_params.get("ip-address"):
 488                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 489                                                      vdu_descriptor["id"], "interfaces",
 490                                                      interface_descriptor["name"], "ip_address"),
 491                                       cp_params["ip-address"])
 492                     if cp_params.get("mac-address"):
 493                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 494                                                      vdu_descriptor["id"], "interfaces",
 495                                                      interface_descriptor["name"], "mac_address"),
 496                                       cp_params["mac-address"])
 497         return RO_ns_params
 498
 499     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 500         # make a copy to do not change
 501         vdu_create = copy(vdu_create)
 502         vdu_delete = copy(vdu_delete)
 503
 504         vdurs = db_vnfr.get("vdur")
 505         if vdurs is None:
 506             vdurs = []
 507         vdu_index = len(vdurs)
 508         while vdu_index:
 509             vdu_index -= 1
 510             vdur = vdurs[vdu_index]
 511             if vdur.get("pdu-type"):
 512                 continue
 513             vdu_id_ref = vdur["vdu-id-ref"]
 514             if vdu_create and vdu_create.get(vdu_id_ref):
 515                 for index in range(0, vdu_create[vdu_id_ref]):
 516                     vdur = deepcopy(vdur)
 517                     vdur["_id"] = str(uuid4())
 518                     vdur["count-index"] += 1
 519                     vdurs.insert(vdu_index+1+index, vdur)
 520                 del vdu_create[vdu_id_ref]
 521             if vdu_delete and vdu_delete.get(vdu_id_ref):
 522                 del vdurs[vdu_index]
 523                 vdu_delete[vdu_id_ref] -= 1
 524                 if not vdu_delete[vdu_id_ref]:
 525                     del vdu_delete[vdu_id_ref]
 526         # check all operations are done
 527         if vdu_create or vdu_delete:
 528             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 529                 vdu_create))
 530         if vdu_delete:
 531             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 532                 vdu_delete))
 533
 534         vnfr_update = {"vdur": vdurs}
 535         db_vnfr["vdur"] = vdurs
 536         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 537
 538     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 539         """
 540         Updates database nsr with the RO info for the created vld
 541         :param ns_update_nsr: dictionary to be filled with the updated info
 542         :param db_nsr: content of db_nsr. This is also modified
 543         :param nsr_desc_RO: nsr descriptor from RO
 544         :return: Nothing, LcmException is raised on errors
 545         """
 546
 547         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 548             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 549                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 550                     continue
 551                 vld["vim-id"] = net_RO.get("vim_net_id")
 552                 vld["name"] = net_RO.get("vim_name")
 553                 vld["status"] = net_RO.get("status")
 554                 vld["status-detailed"] = net_RO.get("error_msg")
 555                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 556                 break
 557             else:
 558                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 559
 560     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 561         """
 562         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 563         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 564         :param nsr_desc_RO: nsr descriptor from RO
 565         :return: Nothing, LcmException is raised on errors
 566         """
 567         for vnf_index, db_vnfr in db_vnfrs.items():
 568             for vnf_RO in nsr_desc_RO["vnfs"]:
 569                 if vnf_RO["member_vnf_index"] != vnf_index:
 570                     continue
 571                 vnfr_update = {}
 572                 if vnf_RO.get("ip_address"):
 573                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 574                 elif not db_vnfr.get("ip-address"):
 575                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 576
 577                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 578                     vdur_RO_count_index = 0
 579                     if vdur.get("pdu-type"):
 580                         continue
 581                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 582                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 583                             continue
 584                         if vdur["count-index"] != vdur_RO_count_index:
 585                             vdur_RO_count_index += 1
 586                             continue
 587                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 588                         if vdur_RO.get("ip_address"):
 589                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 590                         else:
 591                             vdur["ip-address"] = None
 592                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 593                         vdur["name"] = vdur_RO.get("vim_name")
 594                         vdur["status"] = vdur_RO.get("status")
 595                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 596                         for ifacer in get_iterable(vdur, "interfaces"):
 597                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 598                                 if ifacer["name"] == interface_RO.get("internal_name"):
 599                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 600                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 601                                     break
 602                             else:
 603                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 604                                                    "at RO info".format(vnf_index, vdur["vdu-id-ref"], ifacer["name"]))
 605                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 606                         break
 607                     else:
 608                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} at "
 609                                            "RO info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 610
 611                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 612                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 613                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 614                             continue
 615                         vld["vim-id"] = net_RO.get("vim_net_id")
 616                         vld["name"] = net_RO.get("vim_name")
 617                         vld["status"] = net_RO.get("status")
 618                         vld["status-detailed"] = net_RO.get("error_msg")
 619                         vnfr_update["vld.{}".format(vld_index)] = vld
 620                         break
 621                     else:
 622                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} at RO info".format(
 623                             vnf_index, vld["id"]))
 624
 625                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 626                 break
 627
 628             else:
 629                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} at RO info".format(vnf_index))
 630
 631     async def instantiate(self, nsr_id, nslcmop_id):
 632
 633         # Try to lock HA task here
 634         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 635         if not task_is_locked_by_me:
 636             return
 637
 638         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 639         self.logger.debug(logging_text + "Enter")
 640         # get all needed from database
 641         start_deploy = time()
 642         db_nsr = None
 643         db_nslcmop = None
 644         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 645         db_nslcmop_update = {}
 646         nslcmop_operation_state = None
 647         db_vnfrs = {}
 648         RO_descriptor_number = 0   # number of descriptors created at RO
 649         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 650         n2vc_info = {}
 651         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 652         exc = None
 653         try:
 654             # wait for any previous tasks in process
 655             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 656
 657             step = "Getting nslcmop={} from db".format(nslcmop_id)
 658             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 659             step = "Getting nsr={} from db".format(nsr_id)
 660             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 661             ns_params = db_nslcmop.get("operationParams")
 662             nsd = db_nsr["nsd"]
 663             nsr_name = db_nsr["name"]   # TODO short-name??
 664
 665             step = "Getting vnfrs from db"
 666             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 667             db_vnfds_ref = {}
 668             db_vnfds = {}
 669             db_vnfds_index = {}
 670             for vnfr in db_vnfrs_list:
 671                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 672                 vnfd_id = vnfr["vnfd-id"]
 673                 vnfd_ref = vnfr["vnfd-ref"]
 674                 if vnfd_id not in db_vnfds:
 675                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 676                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 677                     db_vnfds_ref[vnfd_ref] = vnfd
 678                     db_vnfds[vnfd_id] = vnfd
 679                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 680
 681             # Get or generates the _admin.deployed,VCA list
 682             vca_deployed_list = None
 683             vca_model_name = None
 684             if db_nsr["_admin"].get("deployed"):
 685                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 686                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 687             if vca_deployed_list is None:
 688                 vca_deployed_list = []
 689                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 690                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 691             elif isinstance(vca_deployed_list, dict):
 692                 # maintain backward compatibility. Change a dict to list at database
 693                 vca_deployed_list = list(vca_deployed_list.values())
 694                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 695                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 696
 697             db_nsr_update["detailed-status"] = "creating"
 698             db_nsr_update["operational-status"] = "init"
 699             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 700                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 701                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 702                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 703
 704             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 705             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 706             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 707
 708             # Deploy charms
 709             # The parameters we'll need to deploy a charm
 710             number_to_configure = 0
 711
 712             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 713                 """An inner function to deploy the charm from either ns, vnf or vdu
 714                 For ns both vnf_index and vdu_id are None.
 715                 For vnf only vdu_id is None
 716                 For vdu both vnf_index and vdu_id contain a value
 717                 """
 718                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 719                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 720
 721                 machine_spec = {}
 722                 if native_charm:
 723                     machine_spec["username"] = charm_params.get("username"),
 724                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 725
 726                 # Note: The charm needs to exist on disk at the location
 727                 # specified by charm_path.
 728                 descriptor = vnfd if vnf_index else nsd
 729                 base_folder = descriptor["_admin"]["storage"]
 730                 storage_params = self.fs.get_params()
 731                 charm_path = "{}{}/{}/charms/{}".format(
 732                     storage_params["path"],
 733                     base_folder["folder"],
 734                     base_folder["pkg-dir"],
 735                     proxy_charm
 736                 )
 737
 738                 # ns_name will be ignored in the current version of N2VC
 739                 # but will be implemented for the next point release.
 740                 model_name = nsr_id
 741                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 742                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 743                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 744
 745                 vca_index = len(vca_deployed_list)
 746                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 747                 # 26*26 charm in the same NS
 748                 application_name = application_name[0:48]
 749                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 750                 vca_deployed_ = {
 751                     "member-vnf-index": vnf_index,
 752                     "vdu_id": vdu_id,
 753                     "model": model_name,
 754                     "application": application_name,
 755                     "operational-status": "init",
 756                     "detailed-status": "",
 757                     "step": "initial-deploy",
 758                     "vnfd_id": vnfd_id,
 759                     "vdu_name": vdu_name,
 760                     "vdu_count_index": vdu_count_index,
 761                 }
 762                 vca_deployed_list.append(vca_deployed_)
 763                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 764                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 765
 766                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 767                                                                                                 proxy_charm))
 768                 if not n2vc_info:
 769                     n2vc_info["nsr_id"] = nsr_id
 770                     n2vc_info["nslcmop_id"] = nslcmop_id
 771                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 772                     n2vc_info["lcmOperationType"] = "instantiate"
 773                     n2vc_info["deployed"] = vca_deployed_list
 774                     n2vc_info["db_update"] = db_nsr_update
 775                 task = asyncio.ensure_future(
 776                     self.n2vc.DeployCharms(
 777                         model_name,          # The network service name
 778                         application_name,    # The application name
 779                         descriptor,          # The vnf/nsd descriptor
 780                         charm_path,          # Path to charm
 781                         charm_params,        # Runtime params, like mgmt ip
 782                         machine_spec,        # for native charms only
 783                         self.n2vc_callback,  # Callback for status changes
 784                         n2vc_info,           # Callback parameter
 785                         None,                # Callback parameter (task)
 786                     )
 787                 )
 788                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 789                                                          n2vc_info))
 790                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 791
 792             step = "Looking for needed vnfd to configure with proxy charm"
 793             self.logger.debug(logging_text + step)
 794
 795             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 796                 vnfd_id = c_vnf["vnfd-id-ref"]
 797                 vnf_index = str(c_vnf["member-vnf-index"])
 798                 vnfd = db_vnfds_ref[vnfd_id]
 799
 800                 # Get additional parameters
 801                 vnfr_params = {}
 802                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 803                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 804                 for k, v in vnfr_params.items():
 805                     if isinstance(v, str) and v.startswith("!!yaml "):
 806                         vnfr_params[k] = yaml.safe_load(v[7:])
 807
 808                 step = "deploying proxy charms for configuration"
 809                 # Check if this VNF has a charm configuration
 810                 vnf_config = vnfd.get("vnf-configuration")
 811                 if vnf_config and vnf_config.get("juju"):
 812                     proxy_charm = vnf_config["juju"]["charm"]
 813                     if vnf_config["juju"].get("proxy") is False:
 814                         # native_charm, will be deployed after VM. Skip
 815                         proxy_charm = None
 816
 817                     if proxy_charm:
 818                         if not vca_model_name:
 819                             step = "creating VCA model name '{}'".format(nsr_id)
 820                             self.logger.debug(logging_text + step)
 821                             await self.n2vc.CreateNetworkService(nsr_id)
 822                             vca_model_name = nsr_id
 823                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 824                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 825                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 826                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 827                         charm_params = {
 828                             "user_values": vnfr_params,
 829                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 830                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 831                         }
 832
 833                         # Login to the VCA. If there are multiple calls to login(),
 834                         # subsequent calls will be a nop and return immediately.
 835                         await self.n2vc.login()
 836
 837                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 838                         number_to_configure += 1
 839
 840                 # Deploy charms for each VDU that supports one.
 841                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 842                     vdu_config = vdu.get('vdu-configuration')
 843                     proxy_charm = None
 844
 845                     if vdu_config and vdu_config.get("juju"):
 846                         proxy_charm = vdu_config["juju"]["charm"]
 847                         if vdu_config["juju"].get("proxy") is False:
 848                             # native_charm, will be deployed after VM. Skip
 849                             proxy_charm = None
 850
 851                         if proxy_charm:
 852                             if not vca_model_name:
 853                                 step = "creating VCA model name"
 854                                 await self.n2vc.CreateNetworkService(nsr_id)
 855                                 vca_model_name = nsr_id
 856                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 857                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 858                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 859                                                                                                           vdu["id"])
 860                             await self.n2vc.login()
 861                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 862                             # TODO for the moment only first vdu_id contains a charm deployed
 863                             if vdur["vdu-id-ref"] != vdu["id"]:
 864                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 865                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 866                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 867                             charm_params = {
 868                                 "user_values": vnfr_params,
 869                                 "rw_mgmt_ip": vdur["ip-address"],
 870                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 871                             }
 872                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 873                                          charm_params, n2vc_info)
 874                             number_to_configure += 1
 875
 876             # Check if this NS has a charm configuration
 877
 878             ns_config = nsd.get("ns-configuration")
 879             if ns_config and ns_config.get("juju"):
 880                 proxy_charm = ns_config["juju"]["charm"]
 881                 if ns_config["juju"].get("proxy") is False:
 882                     # native_charm, will be deployed after VM. Skip
 883                     proxy_charm = None
 884
 885                 if proxy_charm:
 886                     step = "deploying proxy charm to configure ns"
 887                     # TODO is NS magmt IP address needed?
 888
 889                     # Get additional parameters
 890                     additional_params = {}
 891                     if db_nsr.get("additionalParamsForNs"):
 892                         additional_params = db_nsr["additionalParamsForNs"].copy()
 893                     for k, v in additional_params.items():
 894                         if isinstance(v, str) and v.startswith("!!yaml "):
 895                             additional_params[k] = yaml.safe_load(v[7:])
 896
 897                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 898                     charm_params = {
 899                         "user_values": additional_params,
 900                         # "rw_mgmt_ip": db_nsr["ip-address"],
 901                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 902                     }
 903
 904                     # Login to the VCA. If there are multiple calls to login(),
 905                     # subsequent calls will be a nop and return immediately.
 906                     await self.n2vc.login()
 907                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 908                     number_to_configure += 1
 909
 910             db_nsr_update["operational-status"] = "running"
 911
 912             # Wait until all charms has reached blocked or active status
 913             step = "waiting proxy charms to be ready"
 914             if number_to_configure:
 915                 # wait until all charms are configured.
 916                 # steps are:
 917                 #       initial-deploy
 918                 #       get-ssh-public-key
 919                 #       generate-ssh-key
 920                 #       retry-get-ssh-public-key
 921                 #       ssh-public-key-obtained
 922                 while time() <= start_deploy + self.total_deploy_timeout:
 923                     if db_nsr_update:
 924                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 925                     if db_nslcmop_update:
 926                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 927
 928                     all_active = True
 929                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 930                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 931                         if vca_deployed["step"] == "initial-deploy":
 932                             if vca_deployed["operational-status"] in ("active", "blocked"):
 933                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 934                                        .format(vca_deployed["member-vnf-index"],
 935                                                vca_deployed["vdu_id"])
 936                                 self.logger.debug(logging_text + step)
 937                                 try:
 938                                     primitive_id = await self.n2vc.ExecutePrimitive(
 939                                         vca_deployed["model"],
 940                                         vca_deployed["application"],
 941                                         "get-ssh-public-key",
 942                                         None,
 943                                     )
 944                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 945                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 946                                         primitive_id
 947                                     db_nsr_update[database_entry + "operational-status"] =\
 948                                         vca_deployed["operational-status"]
 949                                 except PrimitiveDoesNotExist:
 950                                     ssh_public_key = None
 951                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 952                                         "ssh-public-key-obtained"
 953                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 954                                         ssh_public_key
 955                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 956                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 957                                     self.logger.debug(logging_text + step)
 958
 959                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 960                             primitive_id = vca_deployed["primitive_id"]
 961                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 962                                                                                   primitive_id)
 963                             if primitive_status in ("completed", "failed"):
 964                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 965                                                                                       primitive_id)
 966                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 967                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 968                                         primitive_result.get("pubkey"):
 969                                     ssh_public_key = primitive_result.get("pubkey")
 970                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 971                                         "ssh-public-key-obtained"
 972                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 973                                         ssh_public_key
 974                                     n2vc_key_list.append(ssh_public_key)
 975                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 976                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 977                                     self.logger.debug(logging_text + step)
 978                                 else:  # primitive_status == "failed":
 979                                     if vca_deployed["step"] == "get-ssh-public-key":
 980                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 981                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 982                                                                      vca_deployed["vdu_id"])
 983                                         self.logger.debug(logging_text + step)
 984                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 985                                             "generate-ssh-key"
 986                                         primitive_id = await self.n2vc.ExecutePrimitive(
 987                                             vca_deployed["model"],
 988                                             vca_deployed["application"],
 989                                             "generate-ssh-key",
 990                                             None,
 991                                         )
 992                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 993                                             primitive_id
 994                                     else:  # failed for second time
 995                                         raise LcmException(
 996                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 997
 998                         elif vca_deployed["step"] == "generate-ssh-key":
 999                             primitive_id = vca_deployed["primitive_id"]
1000                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1001                                                                                   primitive_id)
1002                             if primitive_status in ("completed", "failed"):
1003                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1004                                                                                       primitive_id)
1005                                 vca_deployed["primitive_id"] = db_nsr_update[
1006                                     database_entry + "primitive_id"] = None
1007                                 if primitive_status == "completed":
1008                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1009                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1010                                                               vca_deployed["vdu_id"])
1011                                     self.logger.debug(logging_text + step)
1012                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1013                                         "retry-get-ssh-public-key"
1014                                     primitive_id = await self.n2vc.ExecutePrimitive(
1015                                         vca_deployed["model"],
1016                                         vca_deployed["application"],
1017                                         "get-ssh-public-key",
1018                                         None,
1019                                     )
1020                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1021                                         primitive_id
1022
1023                                 else:  # primitive_status == "failed":
1024                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1025                                                        .format(primitive_result))
1026
1027                         if vca_deployed["step"] != "ssh-public-key-obtained":
1028                             all_active = False
1029
1030                     if all_active:
1031                         break
1032                     await asyncio.sleep(5)
1033                 else:   # total_deploy_timeout
1034                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1035                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1036
1037             # deploy RO
1038             # get vnfds, instantiate at RO
1039             for c_vnf in nsd.get("constituent-vnfd", ()):
1040                 member_vnf_index = c_vnf["member-vnf-index"]
1041                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1042                 vnfd_ref = vnfd["id"]
1043                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1044                     vnfd_ref, member_vnf_index)
1045                 # self.logger.debug(logging_text + step)
1046                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1047                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1048                 RO_descriptor_number += 1
1049
1050                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1051                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1052                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1053                         break
1054                 else:
1055                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1056                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1057
1058                 # look if present
1059                 RO_update = {"member-vnf-index": member_vnf_index}
1060                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1061                 if vnfd_list:
1062                     RO_update["id"] = vnfd_list[0]["uuid"]
1063                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1064                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1065                 else:
1066                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1067                                            get("additionalParamsForVnf"), nsr_id)
1068                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1069                     RO_update["id"] = desc["uuid"]
1070                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1071                         vnfd_ref, member_vnf_index, desc["uuid"]))
1072                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1073                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1074                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1075
1076             # create nsd at RO
1077             nsd_ref = nsd["id"]
1078             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1079             # self.logger.debug(logging_text + step)
1080
1081             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1082             RO_descriptor_number += 1
1083             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1084             if nsd_list:
1085                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1086                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1087                     nsd_ref, RO_nsd_uuid))
1088             else:
1089                 nsd_RO = deepcopy(nsd)
1090                 nsd_RO["id"] = RO_osm_nsd_id
1091                 nsd_RO.pop("_id", None)
1092                 nsd_RO.pop("_admin", None)
1093                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1094                     member_vnf_index = c_vnf["member-vnf-index"]
1095                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1096                 for c_vld in nsd_RO.get("vld", ()):
1097                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1098                         member_vnf_index = cp["member-vnf-index-ref"]
1099                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1100
1101                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1102                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1103                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1104                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1105             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1106
1107             # Crate ns at RO
1108             # if present use it unless in error status
1109             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1110             if RO_nsr_id:
1111                 try:
1112                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1113                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1114                     desc = await self.RO.show("ns", RO_nsr_id)
1115                 except ROclient.ROClientException as e:
1116                     if e.http_code != HTTPStatus.NOT_FOUND:
1117                         raise
1118                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1119                 if RO_nsr_id:
1120                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1121                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1122                     if ns_status == "ERROR":
1123                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1124                         self.logger.debug(logging_text + step)
1125                         await self.RO.delete("ns", RO_nsr_id)
1126                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1127             if not RO_nsr_id:
1128                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1129                 # self.logger.debug(logging_text + step)
1130
1131                 # check if VIM is creating and wait  look if previous tasks in process
1132                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1133                 if task_dependency:
1134                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1135                     self.logger.debug(logging_text + step)
1136                     await asyncio.wait(task_dependency, timeout=3600)
1137                 if ns_params.get("vnf"):
1138                     for vnf in ns_params["vnf"]:
1139                         if "vimAccountId" in vnf:
1140                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1141                                                                                         vnf["vimAccountId"])
1142                         if task_dependency:
1143                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1144                             self.logger.debug(logging_text + step)
1145                             await asyncio.wait(task_dependency, timeout=3600)
1146
1147                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1148
1149                 # feature 1429. Add n2vc public key to needed VMs
1150                 n2vc_key = await self.n2vc.GetPublicKey()
1151                 n2vc_key_list.append(n2vc_key)
1152                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1153
1154                 step = db_nsr_update["detailed-status"] = "Creating ns at RO"
1155                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1156                                             name=db_nsr["name"],
1157                                             scenario=RO_nsd_uuid)
1158                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1159                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1160                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1161                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1162             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1163
1164             # wait until NS is ready
1165             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
1166             detailed_status_old = None
1167             self.logger.debug(logging_text + step)
1168
1169             while time() <= start_deploy + self.total_deploy_timeout:
1170                 desc = await self.RO.show("ns", RO_nsr_id)
1171                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1172                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1173                 if ns_status == "ERROR":
1174                     raise ROclient.ROClientException(ns_status_info)
1175                 elif ns_status == "BUILD":
1176                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1177                 elif ns_status == "ACTIVE":
1178                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1179                     try:
1180                         self.ns_update_vnfr(db_vnfrs, desc)
1181                         break
1182                     except LcmExceptionNoMgmtIP:
1183                         pass
1184                 else:
1185                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1186                 if detailed_status != detailed_status_old:
1187                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1188                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1189                 await asyncio.sleep(5, loop=self.loop)
1190             else:   # total_deploy_timeout
1191                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1192
1193             step = "Updating NSR"
1194             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1195
1196             db_nsr_update["operational-status"] = "running"
1197             db_nsr["detailed-status"] = "Configuring vnfr"
1198             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1199
1200             # Configure proxy charms once VMs are up
1201             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1202                 vnf_index = vca_deployed.get("member-vnf-index")
1203                 vdu_id = vca_deployed.get("vdu_id")
1204                 vdu_name = None
1205                 vdu_count_index = None
1206
1207                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1208                                                                                                            vdu_id)
1209                 add_params = {}
1210                 initial_config_primitive_list = []
1211                 if vnf_index:
1212                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1213                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1214                     vnfd = db_vnfds_index[vnf_index]
1215
1216                     if vdu_id:
1217                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1218                             if vdu["id"] == vdu_id:
1219                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1220                                     'initial-config-primitive', [])
1221                                 break
1222                         else:
1223                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1224                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1225                         # TODO for the moment only first vdu_id contains a charm deployed
1226                         if vdur["vdu-id-ref"] != vdu["id"]:
1227                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1228                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1229                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1230                     else:
1231                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1232                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1233                 else:
1234                     if db_nsr.get("additionalParamsForNs"):
1235                         add_params = db_nsr["additionalParamsForNs"].copy()
1236                     for k, v in add_params.items():
1237                         if isinstance(v, str) and v.startswith("!!yaml "):
1238                             add_params[k] = yaml.safe_load(v[7:])
1239                     add_params["rw_mgmt_ip"] = None
1240                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1241
1242                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1243                 initial_config_primitive_list = initial_config_primitive_list.copy()
1244                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1245                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1246
1247                 for initial_config_primitive in initial_config_primitive_list:
1248                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1249                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1250                                       .format(initial_config_primitive["name"], primitive_params_))
1251                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1252                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1253                         initial_config_primitive["name"],
1254                         primitive_params_,
1255                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1256                         retries_interval=30)
1257                     if primitive_result != "COMPLETED":
1258                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1259                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1260                                                    vca_deployed["vdu_id"], primitive_detail))
1261
1262             # Deploy native charms
1263             step = "Looking for needed vnfd to configure with native charm"
1264             self.logger.debug(logging_text + step)
1265
1266             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1267                 vnfd_id = c_vnf["vnfd-id-ref"]
1268                 vnf_index = str(c_vnf["member-vnf-index"])
1269                 vnfd = db_vnfds_ref[vnfd_id]
1270
1271                 # Get additional parameters
1272                 vnfr_params = {}
1273                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1274                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1275                 for k, v in vnfr_params.items():
1276                     if isinstance(v, str) and v.startswith("!!yaml "):
1277                         vnfr_params[k] = yaml.safe_load(v[7:])
1278
1279                 # Check if this VNF has a charm configuration
1280                 vnf_config = vnfd.get("vnf-configuration")
1281                 if vnf_config and vnf_config.get("juju"):
1282                     native_charm = vnf_config["juju"].get("proxy") is False
1283
1284                     if native_charm:
1285                         if not vca_model_name:
1286                             step = "creating VCA model name '{}'".format(nsr_id)
1287                             self.logger.debug(logging_text + step)
1288                             await self.n2vc.CreateNetworkService(nsr_id)
1289                             vca_model_name = nsr_id
1290                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1291                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1292                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1293                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1294                         charm_params = {
1295                             "user_values": vnfr_params,
1296                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1297                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1298                         }
1299
1300                         # get username
1301                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1302                         #  merged. Meanwhile let's get username from initial-config-primitive
1303                         if vnf_config.get("initial-config-primitive"):
1304                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1305                                 if param["name"] == "ssh-username":
1306                                     charm_params["username"] = param["value"]
1307                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1308                             if vnf_config["config-access"]["ssh-access"].get("required"):
1309                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1310
1311                         # Login to the VCA. If there are multiple calls to login(),
1312                         # subsequent calls will be a nop and return immediately.
1313                         await self.n2vc.login()
1314
1315                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1316                         number_to_configure += 1
1317
1318                 # Deploy charms for each VDU that supports one.
1319                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1320                     vdu_config = vdu.get('vdu-configuration')
1321                     native_charm = False
1322
1323                     if vdu_config and vdu_config.get("juju"):
1324                         native_charm = vdu_config["juju"].get("proxy") is False
1325
1326                         if native_charm:
1327                             if not vca_model_name:
1328                                 step = "creating VCA model name"
1329                                 await self.n2vc.CreateNetworkService(nsr_id)
1330                                 vca_model_name = nsr_id
1331                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1332                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1333                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1334                                                                                                      vdu["id"])
1335                             await self.n2vc.login()
1336                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1337                             # TODO for the moment only first vdu_id contains a charm deployed
1338                             if vdur["vdu-id-ref"] != vdu["id"]:
1339                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1340                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1341                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1342                             charm_params = {
1343                                 "user_values": vnfr_params,
1344                                 "rw_mgmt_ip": vdur["ip-address"],
1345                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1346                             }
1347
1348                             # get username
1349                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1350                             #  merged. Meanwhile let's get username from initial-config-primitive
1351                             if vdu_config.get("initial-config-primitive"):
1352                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1353                                     if param["name"] == "ssh-username":
1354                                         charm_params["username"] = param["value"]
1355                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1356                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1357                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1358                                         "default-user")
1359
1360                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1361                                          charm_params, n2vc_info, native_charm)
1362                             number_to_configure += 1
1363
1364             # Check if this NS has a charm configuration
1365
1366             ns_config = nsd.get("ns-configuration")
1367             if ns_config and ns_config.get("juju"):
1368                 native_charm = ns_config["juju"].get("proxy") is False
1369
1370                 if native_charm:
1371                     step = "deploying native charm to configure ns"
1372                     # TODO is NS magmt IP address needed?
1373
1374                     # Get additional parameters
1375                     additional_params = {}
1376                     if db_nsr.get("additionalParamsForNs"):
1377                         additional_params = db_nsr["additionalParamsForNs"].copy()
1378                     for k, v in additional_params.items():
1379                         if isinstance(v, str) and v.startswith("!!yaml "):
1380                             additional_params[k] = yaml.safe_load(v[7:])
1381
1382                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1383                     charm_params = {
1384                         "user_values": additional_params,
1385                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1386                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1387                     }
1388
1389                     # get username
1390                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1391                     #  merged. Meanwhile let's get username from initial-config-primitive
1392                     if ns_config.get("initial-config-primitive"):
1393                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1394                             if param["name"] == "ssh-username":
1395                                 charm_params["username"] = param["value"]
1396                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1397                         if ns_config["config-access"]["ssh-access"].get("required"):
1398                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1399
1400                     # Login to the VCA. If there are multiple calls to login(),
1401                     # subsequent calls will be a nop and return immediately.
1402                     await self.n2vc.login()
1403                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1404                     number_to_configure += 1
1405
1406             # waiting all charms are ok
1407             configuration_failed = False
1408             if number_to_configure:
1409                 step = "Waiting all charms are active"
1410                 old_status = "configuring: init: {}".format(number_to_configure)
1411                 db_nsr_update["config-status"] = old_status
1412                 db_nsr_update["detailed-status"] = old_status
1413                 db_nslcmop_update["detailed-status"] = old_status
1414
1415                 # wait until all are configured.
1416                 while time() <= start_deploy + self.total_deploy_timeout:
1417                     if db_nsr_update:
1418                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1419                     if db_nslcmop_update:
1420                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1421                     # TODO add a fake task that set n2vc_event after some time
1422                     await n2vc_info["n2vc_event"].wait()
1423                     n2vc_info["n2vc_event"].clear()
1424                     all_active = True
1425                     status_map = {}
1426                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1427                     now = time()
1428                     for vca_deployed in vca_deployed_list:
1429                         vca_status = vca_deployed["operational-status"]
1430                         if vca_status not in status_map:
1431                             # Initialize it
1432                             status_map[vca_status] = 0
1433                         status_map[vca_status] += 1
1434
1435                         if vca_status == "active":
1436                             vca_deployed.pop("time_first_error", None)
1437                             vca_deployed.pop("status_first_error", None)
1438                             continue
1439
1440                         all_active = False
1441                         if vca_status in ("error", "blocked"):
1442                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1443                             # if not first time in this status error
1444                             if not vca_deployed.get("time_first_error"):
1445                                 vca_deployed["time_first_error"] = now
1446                                 continue
1447                         if vca_deployed.get("time_first_error") and \
1448                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1449                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1450                                                    .format(vca_deployed["member-vnf-index"],
1451                                                            vca_deployed["vdu_id"], vca_status,
1452                                                            vca_deployed["detailed-status-error"]))
1453
1454                     if all_active:
1455                         break
1456                     elif n2vc_error_text:
1457                         db_nsr_update["config-status"] = "failed"
1458                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1459                         db_nsr_update["detailed-status"] = error_text
1460                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1461                         db_nslcmop_update["detailed-status"] = error_text
1462                         db_nslcmop_update["statusEnteredTime"] = time()
1463                         configuration_failed = True
1464                         break
1465                     else:
1466                         cs = "configuring: "
1467                         separator = ""
1468                         for status, num in status_map.items():
1469                             cs += separator + "{}: {}".format(status, num)
1470                             separator = ", "
1471                         if old_status != cs:
1472                             db_nsr_update["config-status"] = cs
1473                             db_nsr_update["detailed-status"] = cs
1474                             db_nslcmop_update["detailed-status"] = cs
1475                             old_status = cs
1476                 else:   # total_deploy_timeout
1477                     raise LcmException("Timeout waiting ns to be configured")
1478
1479             if not configuration_failed:
1480                 # all is done
1481                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1482                 db_nslcmop_update["statusEnteredTime"] = time()
1483                 db_nslcmop_update["detailed-status"] = "done"
1484                 db_nsr_update["config-status"] = "configured"
1485                 db_nsr_update["detailed-status"] = "done"
1486
1487             return
1488
1489         except (ROclient.ROClientException, DbException, LcmException) as e:
1490             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1491             exc = e
1492         except asyncio.CancelledError:
1493             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1494             exc = "Operation was cancelled"
1495         except Exception as e:
1496             exc = traceback.format_exc()
1497             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1498                                  exc_info=True)
1499         finally:
1500             if exc:
1501                 if db_nsr:
1502                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1503                     db_nsr_update["operational-status"] = "failed"
1504                 if db_nslcmop:
1505                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1506                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1507                     db_nslcmop_update["statusEnteredTime"] = time()
1508             try:
1509                 if db_nsr:
1510                     db_nsr_update["_admin.nslcmop"] = None
1511                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1512                 if db_nslcmop_update:
1513                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1514             except DbException as e:
1515                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1516             if nslcmop_operation_state:
1517                 try:
1518                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1519                                                                    "operationState": nslcmop_operation_state},
1520                                             loop=self.loop)
1521                 except Exception as e:
1522                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1523
1524             self.logger.debug(logging_text + "Exit")
1525             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1526
1527     async def _destroy_charm(self, model, application):
1528         """
1529         Order N2VC destroy a charm
1530         :param model:
1531         :param application:
1532         :return: True if charm does not exist. False if it exist
1533         """
1534         if not await self.n2vc.HasApplication(model, application):
1535             return True  # Already removed
1536         await self.n2vc.RemoveCharms(model, application)
1537         return False
1538
1539     async def _wait_charm_destroyed(self, model, application, timeout):
1540         """
1541         Wait until charm does not exist
1542         :param model:
1543         :param application:
1544         :param timeout:
1545         :return: True if not exist, False if timeout
1546         """
1547         while True:
1548             if not await self.n2vc.HasApplication(model, application):
1549                 return True
1550             if timeout < 0:
1551                 return False
1552             await asyncio.sleep(10)
1553             timeout -= 10
1554
1555     # Check if this VNFD has a configured terminate action
1556     def _has_terminate_config_primitive(self, vnfd):
1557         vnf_config = vnfd.get("vnf-configuration")
1558         if vnf_config and vnf_config.get("terminate-config-primitive"):
1559             return True
1560         else:
1561             return False
1562
1563     @staticmethod
1564     def _get_terminate_config_primitive_seq_list(vnfd):
1565         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1566         # No need to check for existing primitive twice, already done before
1567         vnf_config = vnfd.get("vnf-configuration")
1568         seq_list = vnf_config.get("terminate-config-primitive")
1569         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1570         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1571         return seq_list_sorted
1572
1573     @staticmethod
1574     def _create_nslcmop(nsr_id, operation, params):
1575         """
1576         Creates a ns-lcm-opp content to be stored at database.
1577         :param nsr_id: internal id of the instance
1578         :param operation: instantiate, terminate, scale, action, ...
1579         :param params: user parameters for the operation
1580         :return: dictionary following SOL005 format
1581         """
1582         # Raise exception if invalid arguments
1583         if not (nsr_id and operation and params):
1584             raise LcmException(
1585                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1586         now = time()
1587         _id = str(uuid4())
1588         nslcmop = {
1589             "id": _id,
1590             "_id": _id,
1591             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1592             "operationState": "PROCESSING",
1593             "statusEnteredTime": now,
1594             "nsInstanceId": nsr_id,
1595             "lcmOperationType": operation,
1596             "startTime": now,
1597             "isAutomaticInvocation": False,
1598             "operationParams": params,
1599             "isCancelPending": False,
1600             "links": {
1601                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1602                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1603             }
1604         }
1605         return nslcmop
1606
1607     def _get_terminate_primitive_params(self, seq, vnf_index):
1608         primitive = seq.get('name')
1609         primitive_params = {}
1610         params = {
1611             "member_vnf_index": vnf_index,
1612             "primitive": primitive,
1613             "primitive_params": primitive_params,
1614         }
1615         desc_params = {}
1616         return self._map_primitive_params(seq, params, desc_params)
1617
1618     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1619                           vdu_name, primitive, mapped_primitive_params):
1620         # Create the "_admin.operations" array, or append operation if array already exists
1621         key_admin = '_admin'
1622         key_operations = 'operations'
1623         db_nslcmop_update = {}
1624         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1625         op_list = db_nslcmop_admin.get(key_operations)
1626         new_op = {'member_vnf_index': vnf_index,
1627                   'vdu_id': vdu_id,
1628                   'vdu_count_index': vdu_count_index,
1629                   'primitive': primitive,
1630                   'primitive_params': mapped_primitive_params}
1631         if db_nslcmop_admin:
1632             if not op_list:
1633                 # First operation, create key 'operations' with current operation as first list element
1634                 db_nslcmop_admin.update({key_operations: [new_op]})
1635                 op_list = db_nslcmop_admin.get(key_operations)
1636             else:
1637                 # Not first operation, append operation to existing list
1638                 op_list.append(new_op)
1639
1640         db_nslcmop_update['_admin.operations'] = op_list
1641         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1642
1643     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1644         """ Create a primitive with params from VNFD
1645             Called from terminate() before deleting instance
1646             Calls action() to execute the primitive """
1647         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1648         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1649         db_vnfds = {}
1650         # Loop over VNFRs
1651         for vnfr in db_vnfrs_list:
1652             vnfd_id = vnfr["vnfd-id"]
1653             vnf_index = vnfr["member-vnf-index-ref"]
1654             if vnfd_id not in db_vnfds:
1655                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1656                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1657                 db_vnfds[vnfd_id] = vnfd
1658             vnfd = db_vnfds[vnfd_id]
1659             if not self._has_terminate_config_primitive(vnfd):
1660                 continue
1661             # Get the primitive's sorted sequence list
1662             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1663             for seq in seq_list:
1664                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1665                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1666                     vnf_index, seq.get("name"))
1667                 self.logger.debug(logging_text + step)
1668                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1669                 primitive = seq.get('name')
1670                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1671                 # The following 3 parameters are currently set to None for 'terminate':
1672                 # vdu_id, vdu_count_index, vdu_name
1673                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1674                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1675                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1676                 # Add suboperation
1677                 self._add_suboperation(db_nslcmop,
1678                                        nslcmop_id,
1679                                        vnf_index,
1680                                        vdu_id,
1681                                        vdu_count_index,
1682                                        vdu_name,
1683                                        primitive,
1684                                        mapped_primitive_params)
1685                 # Suboperations: Call _ns_execute_primitive() instead of action()
1686                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1687                 nsr_deployed = db_nsr["_admin"]["deployed"]
1688                 result, result_detail = await self._ns_execute_primitive(
1689                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1690                     mapped_primitive_params)
1691
1692                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1693                 #    nsr_id, nslcmop_terminate_action_id)
1694                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1695                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1696                 if result not in result_ok:
1697                     raise LcmException(
1698                         "terminate_primitive_action for vnf_member_index={}",
1699                         " primitive={} fails with error {}".format(
1700                             vnf_index, seq.get("name"), result_detail))
1701
1702     async def terminate(self, nsr_id, nslcmop_id):
1703
1704         # Try to lock HA task here
1705         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1706         if not task_is_locked_by_me:
1707             return
1708
1709         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1710         self.logger.debug(logging_text + "Enter")
1711         db_nsr = None
1712         db_nslcmop = None
1713         exc = None
1714         failed_detail = []   # annotates all failed error messages
1715         vca_time_destroy = None   # time of where destroy charm order
1716         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1717         db_nslcmop_update = {}
1718         nslcmop_operation_state = None
1719         autoremove = False  # autoremove after terminated
1720         try:
1721             # wait for any previous tasks in process
1722             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1723
1724             step = "Getting nslcmop={} from db".format(nslcmop_id)
1725             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1726             step = "Getting nsr={} from db".format(nsr_id)
1727             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1728             # nsd = db_nsr["nsd"]
1729             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1730             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1731                 return
1732             # #TODO check if VIM is creating and wait
1733             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1734             # Call internal terminate action
1735             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1736
1737             db_nsr_update["operational-status"] = "terminating"
1738             db_nsr_update["config-status"] = "terminating"
1739
1740             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1741                 vca_model_name = nsr_deployed["VCA-model-name"]
1742                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1743                 self.logger.debug(logging_text + step)
1744                 try:
1745                     await self.n2vc.DestroyNetworkService(vca_model_name)
1746                 except NetworkServiceDoesNotExist:
1747                     pass
1748                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1749                 if nsr_deployed.get("VCA"):
1750                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1751                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1752                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1753             # for backward compatibility if charm have been created with "default"  model name delete one by one
1754             elif nsr_deployed and nsr_deployed.get("VCA"):
1755                 try:
1756                     step = "Scheduling configuration charms removing"
1757                     db_nsr_update["detailed-status"] = "Deleting charms"
1758                     self.logger.debug(logging_text + step)
1759                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1760                     # for backward compatibility
1761                     if isinstance(nsr_deployed["VCA"], dict):
1762                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1763                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1764                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1765
1766                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1767                         if vca_deployed:
1768                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1769                                 vca_deployed.clear()
1770                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1771                             else:
1772                                 vca_time_destroy = time()
1773                 except Exception as e:
1774                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1775
1776             # remove from RO
1777             RO_fail = False
1778
1779             # Delete ns
1780             RO_nsr_id = RO_delete_action = None
1781             if nsr_deployed and nsr_deployed.get("RO"):
1782                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1783                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1784             try:
1785                 if RO_nsr_id:
1786                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = "Deleting ns at RO"
1787                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1788                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1789                     self.logger.debug(logging_text + step)
1790                     desc = await self.RO.delete("ns", RO_nsr_id)
1791                     RO_delete_action = desc["action_id"]
1792                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1793                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1794                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1795                 if RO_delete_action:
1796                     # wait until NS is deleted from VIM
1797                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1798                         format(RO_nsr_id, RO_delete_action)
1799                     detailed_status_old = None
1800                     self.logger.debug(logging_text + step)
1801
1802                     delete_timeout = 20 * 60   # 20 minutes
1803                     while delete_timeout > 0:
1804                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1805                                                   extra_item_id=RO_delete_action)
1806                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1807                         if ns_status == "ERROR":
1808                             raise ROclient.ROClientException(ns_status_info)
1809                         elif ns_status == "BUILD":
1810                             detailed_status = step + "; {}".format(ns_status_info)
1811                         elif ns_status == "ACTIVE":
1812                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1813                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1814                             break
1815                         else:
1816                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1817                         if detailed_status != detailed_status_old:
1818                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1819                                 db_nsr_update["detailed-status"] = detailed_status
1820                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1821                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1822                         await asyncio.sleep(5, loop=self.loop)
1823                         delete_timeout -= 5
1824                     else:  # delete_timeout <= 0:
1825                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1826
1827             except ROclient.ROClientException as e:
1828                 if e.http_code == 404:  # not found
1829                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1830                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1831                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1832                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1833                 elif e.http_code == 409:   # conflict
1834                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1835                     self.logger.debug(logging_text + failed_detail[-1])
1836                     RO_fail = True
1837                 else:
1838                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1839                     self.logger.error(logging_text + failed_detail[-1])
1840                     RO_fail = True
1841
1842             # Delete nsd
1843             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1844                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1845                 try:
1846                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1847                         "Deleting nsd at RO"
1848                     await self.RO.delete("nsd", RO_nsd_id)
1849                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1850                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1851                 except ROclient.ROClientException as e:
1852                     if e.http_code == 404:  # not found
1853                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1854                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1855                     elif e.http_code == 409:   # conflict
1856                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1857                         self.logger.debug(logging_text + failed_detail[-1])
1858                         RO_fail = True
1859                     else:
1860                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1861                         self.logger.error(logging_text + failed_detail[-1])
1862                         RO_fail = True
1863
1864             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1865                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1866                     if not vnf_deployed or not vnf_deployed["id"]:
1867                         continue
1868                     try:
1869                         RO_vnfd_id = vnf_deployed["id"]
1870                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1871                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1872                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1873                         await self.RO.delete("vnfd", RO_vnfd_id)
1874                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1875                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1876                     except ROclient.ROClientException as e:
1877                         if e.http_code == 404:  # not found
1878                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1879                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1880                         elif e.http_code == 409:   # conflict
1881                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1882                             self.logger.debug(logging_text + failed_detail[-1])
1883                         else:
1884                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1885                             self.logger.error(logging_text + failed_detail[-1])
1886
1887             # wait until charm deleted
1888             if vca_time_destroy:
1889                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1890                     "Waiting for deletion of configuration charms"
1891                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1892                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1893                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1894                     if not vca_deployed:
1895                         continue
1896                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1897                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1898                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1899                                                             timeout):
1900                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1901                             vca_deployed["application"]))
1902                     else:
1903                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1904
1905             if failed_detail:
1906                 self.logger.error(logging_text + " ;".join(failed_detail))
1907                 db_nsr_update["operational-status"] = "failed"
1908                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1909                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1910                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1911                 db_nslcmop_update["statusEnteredTime"] = time()
1912             else:
1913                 db_nsr_update["operational-status"] = "terminated"
1914                 db_nsr_update["detailed-status"] = "Done"
1915                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1916                 db_nslcmop_update["detailed-status"] = "Done"
1917                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1918                 db_nslcmop_update["statusEnteredTime"] = time()
1919                 if db_nslcmop["operationParams"].get("autoremove"):
1920                     autoremove = True
1921
1922         except (ROclient.ROClientException, DbException, LcmException) as e:
1923             self.logger.error(logging_text + "Exit Exception {}".format(e))
1924             exc = e
1925         except asyncio.CancelledError:
1926             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1927             exc = "Operation was cancelled"
1928         except Exception as e:
1929             exc = traceback.format_exc()
1930             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1931         finally:
1932             if exc and db_nslcmop:
1933                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1934                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1935                 db_nslcmop_update["statusEnteredTime"] = time()
1936             try:
1937                 if db_nslcmop and db_nslcmop_update:
1938                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1939                 if db_nsr:
1940                     db_nsr_update["_admin.nslcmop"] = None
1941                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1942             except DbException as e:
1943                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1944             if nslcmop_operation_state:
1945                 try:
1946                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1947                                                                  "operationState": nslcmop_operation_state,
1948                                                                  "autoremove": autoremove},
1949                                             loop=self.loop)
1950                 except Exception as e:
1951                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1952             self.logger.debug(logging_text + "Exit")
1953             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1954
1955     @staticmethod
1956     def _map_primitive_params(primitive_desc, params, instantiation_params):
1957         """
1958         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1959         The default-value is used. If it is between < > it look for a value at instantiation_params
1960         :param primitive_desc: portion of VNFD/NSD that describes primitive
1961         :param params: Params provided by user
1962         :param instantiation_params: Instantiation params provided by user
1963         :return: a dictionary with the calculated params
1964         """
1965         calculated_params = {}
1966         for parameter in primitive_desc.get("parameter", ()):
1967             param_name = parameter["name"]
1968             if param_name in params:
1969                 calculated_params[param_name] = params[param_name]
1970             elif "default-value" in parameter or "value" in parameter:
1971                 if "value" in parameter:
1972                     calculated_params[param_name] = parameter["value"]
1973                 else:
1974                     calculated_params[param_name] = parameter["default-value"]
1975                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1976                         and calculated_params[param_name].endswith(">"):
1977                     if calculated_params[param_name][1:-1] in instantiation_params:
1978                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1979                     else:
1980                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1981                                            format(parameter["default-value"], primitive_desc["name"]))
1982             else:
1983                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1984                                    format(param_name, primitive_desc["name"]))
1985
1986             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1987                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1988                                                                width=256)
1989             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1990                 calculated_params[param_name] = calculated_params[param_name][7:]
1991         return calculated_params
1992
1993     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1994                                     primitive, primitive_params, retries=0, retries_interval=30):
1995         start_primitive_time = time()
1996         try:
1997             for vca_deployed in db_deployed["VCA"]:
1998                 if not vca_deployed:
1999                     continue
2000                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2001                     continue
2002                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2003                     continue
2004                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2005                     continue
2006                 break
2007             else:
2008                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2009                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2010             model_name = vca_deployed.get("model")
2011             application_name = vca_deployed.get("application")
2012             if not model_name or not application_name:
2013                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2014                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2015                                                                        vdu_count_index))
2016             # if vca_deployed["operational-status"] != "active":
2017             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2018             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2019             callback = None  # self.n2vc_callback
2020             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2021             await self.n2vc.login()
2022             if primitive == "config":
2023                 primitive_params = {"params": primitive_params}
2024             while retries >= 0:
2025                 primitive_id = await self.n2vc.ExecutePrimitive(
2026                     model_name,
2027                     application_name,
2028                     primitive,
2029                     callback,
2030                     *callback_args,
2031                     **primitive_params
2032                 )
2033                 while time() - start_primitive_time < self.timeout_primitive:
2034                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2035                     if primitive_result_ in ("completed", "failed"):
2036                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2037                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2038                         break
2039                     elif primitive_result_ is None and primitive == "config":
2040                         primitive_result = "COMPLETED"
2041                         detailed_result = None
2042                         break
2043                     else:  # ("running", "pending", None):
2044                         pass
2045                     await asyncio.sleep(5)
2046                 else:
2047                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2048                 if primitive_result == "COMPLETED":
2049                     break
2050                 retries -= 1
2051                 if retries >= 0:
2052                     await asyncio.sleep(retries_interval)
2053
2054             return primitive_result, detailed_result
2055         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2056             return "FAILED", str(e)
2057
2058     async def action(self, nsr_id, nslcmop_id):
2059
2060         # Try to lock HA task here
2061         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2062         if not task_is_locked_by_me:
2063             return
2064
2065         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2066         self.logger.debug(logging_text + "Enter")
2067         # get all needed from database
2068         db_nsr = None
2069         db_nslcmop = None
2070         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2071         db_nslcmop_update = {}
2072         nslcmop_operation_state = None
2073         nslcmop_operation_state_detail = None
2074         exc = None
2075         try:
2076             # wait for any previous tasks in process
2077             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2078
2079             step = "Getting information from database"
2080             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2081             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2082
2083             nsr_deployed = db_nsr["_admin"].get("deployed")
2084             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2085             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2086             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2087             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2088
2089             if vnf_index:
2090                 step = "Getting vnfr from database"
2091                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2092                 step = "Getting vnfd from database"
2093                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2094             else:
2095                 if db_nsr.get("nsd"):
2096                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2097                 else:
2098                     step = "Getting nsd from database"
2099                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2100
2101             # for backward compatibility
2102             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2103                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2104                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2105                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2106
2107             primitive = db_nslcmop["operationParams"]["primitive"]
2108             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2109
2110             # look for primitive
2111             config_primitive_desc = None
2112             if vdu_id:
2113                 for vdu in get_iterable(db_vnfd, "vdu"):
2114                     if vdu_id == vdu["id"]:
2115                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2116                             if config_primitive["name"] == primitive:
2117                                 config_primitive_desc = config_primitive
2118                                 break
2119             elif vnf_index:
2120                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2121                     if config_primitive["name"] == primitive:
2122                         config_primitive_desc = config_primitive
2123                         break
2124             else:
2125                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2126                     if config_primitive["name"] == primitive:
2127                         config_primitive_desc = config_primitive
2128                         break
2129
2130             if not config_primitive_desc:
2131                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2132                                    format(primitive))
2133
2134             desc_params = {}
2135             if vnf_index:
2136                 if db_vnfr.get("additionalParamsForVnf"):
2137                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2138             else:
2139                 if db_nsr.get("additionalParamsForVnf"):
2140                     desc_params.update(db_nsr["additionalParamsForNs"])
2141
2142             # TODO check if ns is in a proper status
2143             result, result_detail = await self._ns_execute_primitive(
2144                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2145                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2146             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2147             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2148             db_nslcmop_update["statusEnteredTime"] = time()
2149             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2150             return  # database update is called inside finally
2151
2152         except (DbException, LcmException) as e:
2153             self.logger.error(logging_text + "Exit Exception {}".format(e))
2154             exc = e
2155         except asyncio.CancelledError:
2156             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2157             exc = "Operation was cancelled"
2158         except Exception as e:
2159             exc = traceback.format_exc()
2160             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2161         finally:
2162             if exc and db_nslcmop:
2163                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2164                     "FAILED {}: {}".format(step, exc)
2165                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2166                 db_nslcmop_update["statusEnteredTime"] = time()
2167             try:
2168                 if db_nslcmop_update:
2169                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2170                 if db_nsr:
2171                     db_nsr_update["_admin.nslcmop"] = None
2172                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2173             except DbException as e:
2174                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2175             self.logger.debug(logging_text + "Exit")
2176             if nslcmop_operation_state:
2177                 try:
2178                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2179                                                                "operationState": nslcmop_operation_state},
2180                                             loop=self.loop)
2181                 except Exception as e:
2182                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2183             self.logger.debug(logging_text + "Exit")
2184             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2185             return nslcmop_operation_state, nslcmop_operation_state_detail
2186
2187     async def scale(self, nsr_id, nslcmop_id):
2188
2189         # Try to lock HA task here
2190         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2191         if not task_is_locked_by_me:
2192             return
2193
2194         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2195         self.logger.debug(logging_text + "Enter")
2196         # get all needed from database
2197         db_nsr = None
2198         db_nslcmop = None
2199         db_nslcmop_update = {}
2200         nslcmop_operation_state = None
2201         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2202         exc = None
2203         # in case of error, indicates what part of scale was failed to put nsr at error status
2204         scale_process = None
2205         old_operational_status = ""
2206         old_config_status = ""
2207         vnfr_scaled = False
2208         try:
2209             # wait for any previous tasks in process
2210             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2211
2212             step = "Getting nslcmop from database"
2213             self.logger.debug(step + " after having waited for previous tasks to be completed")
2214             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2215             step = "Getting nsr from database"
2216             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2217
2218             old_operational_status = db_nsr["operational-status"]
2219             old_config_status = db_nsr["config-status"]
2220             step = "Parsing scaling parameters"
2221             # self.logger.debug(step)
2222             db_nsr_update["operational-status"] = "scaling"
2223             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2224             nsr_deployed = db_nsr["_admin"].get("deployed")
2225             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2226             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2227             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2228             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2229             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2230
2231             # for backward compatibility
2232             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2233                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2234                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2235                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2236
2237             step = "Getting vnfr from database"
2238             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2239             step = "Getting vnfd from database"
2240             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2241
2242             step = "Getting scaling-group-descriptor"
2243             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2244                 if scaling_descriptor["name"] == scaling_group:
2245                     break
2246             else:
2247                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2248                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2249
2250             # cooldown_time = 0
2251             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2252             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2253             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2254             #         break
2255
2256             # TODO check if ns is in a proper status
2257             step = "Sending scale order to RO"
2258             nb_scale_op = 0
2259             if not db_nsr["_admin"].get("scaling-group"):
2260                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2261                 admin_scale_index = 0
2262             else:
2263                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2264                     if admin_scale_info["name"] == scaling_group:
2265                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2266                         break
2267                 else:  # not found, set index one plus last element and add new entry with the name
2268                     admin_scale_index += 1
2269                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2270             RO_scaling_info = []
2271             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2272             if scaling_type == "SCALE_OUT":
2273                 # count if max-instance-count is reached
2274                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2275                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2276                 if nb_scale_op >= max_instance_count:
2277                     raise LcmException("reached the limit of {} (max-instance-count) "
2278                                        "scaling-out operations for the "
2279                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2280
2281                 nb_scale_op += 1
2282                 vdu_scaling_info["scaling_direction"] = "OUT"
2283                 vdu_scaling_info["vdu-create"] = {}
2284                 for vdu_scale_info in scaling_descriptor["vdu"]:
2285                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2286                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2287                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2288
2289             elif scaling_type == "SCALE_IN":
2290                 # count if min-instance-count is reached
2291                 min_instance_count = 0
2292                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2293                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2294                 if nb_scale_op <= min_instance_count:
2295                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2296                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2297                 nb_scale_op -= 1
2298                 vdu_scaling_info["scaling_direction"] = "IN"
2299                 vdu_scaling_info["vdu-delete"] = {}
2300                 for vdu_scale_info in scaling_descriptor["vdu"]:
2301                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2302                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2303                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2304
2305             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2306             vdu_create = vdu_scaling_info.get("vdu-create")
2307             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2308             if vdu_scaling_info["scaling_direction"] == "IN":
2309                 for vdur in reversed(db_vnfr["vdur"]):
2310                     if vdu_delete.get(vdur["vdu-id-ref"]):
2311                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2312                         vdu_scaling_info["vdu"].append({
2313                             "name": vdur["name"],
2314                             "vdu_id": vdur["vdu-id-ref"],
2315                             "interface": []
2316                         })
2317                         for interface in vdur["interfaces"]:
2318                             vdu_scaling_info["vdu"][-1]["interface"].append({
2319                                 "name": interface["name"],
2320                                 "ip_address": interface["ip-address"],
2321                                 "mac_address": interface.get("mac-address"),
2322                             })
2323                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2324
2325             # execute primitive service PRE-SCALING
2326             step = "Executing pre-scale vnf-config-primitive"
2327             if scaling_descriptor.get("scaling-config-action"):
2328                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2329                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2330                             and scaling_type == "SCALE_IN":
2331                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2332                         step = db_nslcmop_update["detailed-status"] = \
2333                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2334
2335                         # look for primitive
2336                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2337                             if config_primitive["name"] == vnf_config_primitive:
2338                                 break
2339                         else:
2340                             raise LcmException(
2341                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2342                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2343                                 "primitive".format(scaling_group, config_primitive))
2344
2345                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2346                         if db_vnfr.get("additionalParamsForVnf"):
2347                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2348
2349                         scale_process = "VCA"
2350                         db_nsr_update["config-status"] = "configuring pre-scaling"
2351                         result, result_detail = await self._ns_execute_primitive(
2352                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2353                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2354                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2355                             vnf_config_primitive, result, result_detail))
2356                         if result == "FAILED":
2357                             raise LcmException(result_detail)
2358                         db_nsr_update["config-status"] = old_config_status
2359                         scale_process = None
2360
2361             if RO_scaling_info:
2362                 scale_process = "RO"
2363                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2364                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2365                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2366                 # wait until ready
2367                 RO_nslcmop_id = RO_desc["instance_action_id"]
2368                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2369
2370                 RO_task_done = False
2371                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2372                 detailed_status_old = None
2373                 self.logger.debug(logging_text + step)
2374
2375                 deployment_timeout = 1 * 3600   # One hour
2376                 while deployment_timeout > 0:
2377                     if not RO_task_done:
2378                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2379                                                   extra_item_id=RO_nslcmop_id)
2380                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2381                         if ns_status == "ERROR":
2382                             raise ROclient.ROClientException(ns_status_info)
2383                         elif ns_status == "BUILD":
2384                             detailed_status = step + "; {}".format(ns_status_info)
2385                         elif ns_status == "ACTIVE":
2386                             RO_task_done = True
2387                             step = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsr_id)
2388                             self.logger.debug(logging_text + step)
2389                         else:
2390                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2391                     else:
2392                         desc = await self.RO.show("ns", RO_nsr_id)
2393                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2394                         if ns_status == "ERROR":
2395                             raise ROclient.ROClientException(ns_status_info)
2396                         elif ns_status == "BUILD":
2397                             detailed_status = step + "; {}".format(ns_status_info)
2398                         elif ns_status == "ACTIVE":
2399                             step = detailed_status = \
2400                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2401                             if not vnfr_scaled:
2402                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2403                                 vnfr_scaled = True
2404                             try:
2405                                 desc = await self.RO.show("ns", RO_nsr_id)
2406                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2407                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2408                                 break
2409                             except LcmExceptionNoMgmtIP:
2410                                 pass
2411                         else:
2412                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2413                     if detailed_status != detailed_status_old:
2414                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2415                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2416
2417                     await asyncio.sleep(5, loop=self.loop)
2418                     deployment_timeout -= 5
2419                 if deployment_timeout <= 0:
2420                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2421
2422                 # update VDU_SCALING_INFO with the obtained ip_addresses
2423                 if vdu_scaling_info["scaling_direction"] == "OUT":
2424                     for vdur in reversed(db_vnfr["vdur"]):
2425                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2426                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2427                             vdu_scaling_info["vdu"].append({
2428                                 "name": vdur["name"],
2429                                 "vdu_id": vdur["vdu-id-ref"],
2430                                 "interface": []
2431                             })
2432                             for interface in vdur["interfaces"]:
2433                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2434                                     "name": interface["name"],
2435                                     "ip_address": interface["ip-address"],
2436                                     "mac_address": interface.get("mac-address"),
2437                                 })
2438                     del vdu_scaling_info["vdu-create"]
2439
2440             scale_process = None
2441             if db_nsr_update:
2442                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2443
2444             # execute primitive service POST-SCALING
2445             step = "Executing post-scale vnf-config-primitive"
2446             if scaling_descriptor.get("scaling-config-action"):
2447                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2448                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2449                             and scaling_type == "SCALE_OUT":
2450                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2451                         step = db_nslcmop_update["detailed-status"] = \
2452                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2453
2454                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2455                         if db_vnfr.get("additionalParamsForVnf"):
2456                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2457
2458                         # look for primitive
2459                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2460                             if config_primitive["name"] == vnf_config_primitive:
2461                                 break
2462                         else:
2463                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2464                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2465                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2466                                                                                                      config_primitive))
2467                         scale_process = "VCA"
2468                         db_nsr_update["config-status"] = "configuring post-scaling"
2469
2470                         result, result_detail = await self._ns_execute_primitive(
2471                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2472                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2473                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2474                             vnf_config_primitive, result, result_detail))
2475                         if result == "FAILED":
2476                             raise LcmException(result_detail)
2477                         db_nsr_update["config-status"] = old_config_status
2478                         scale_process = None
2479
2480             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2481             db_nslcmop_update["statusEnteredTime"] = time()
2482             db_nslcmop_update["detailed-status"] = "done"
2483             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2484             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2485                 else old_operational_status
2486             db_nsr_update["config-status"] = old_config_status
2487             return
2488         except (ROclient.ROClientException, DbException, LcmException) as e:
2489             self.logger.error(logging_text + "Exit Exception {}".format(e))
2490             exc = e
2491         except asyncio.CancelledError:
2492             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2493             exc = "Operation was cancelled"
2494         except Exception as e:
2495             exc = traceback.format_exc()
2496             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2497         finally:
2498             if exc:
2499                 if db_nslcmop:
2500                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2501                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2502                     db_nslcmop_update["statusEnteredTime"] = time()
2503                 if db_nsr:
2504                     db_nsr_update["operational-status"] = old_operational_status
2505                     db_nsr_update["config-status"] = old_config_status
2506                     db_nsr_update["detailed-status"] = ""
2507                     db_nsr_update["_admin.nslcmop"] = None
2508                     if scale_process:
2509                         if "VCA" in scale_process:
2510                             db_nsr_update["config-status"] = "failed"
2511                         if "RO" in scale_process:
2512                             db_nsr_update["operational-status"] = "failed"
2513                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2514                                                                                                      exc)
2515             try:
2516                 if db_nslcmop and db_nslcmop_update:
2517                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2518                 if db_nsr:
2519                     db_nsr_update["_admin.nslcmop"] = None
2520                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2521             except DbException as e:
2522                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2523             if nslcmop_operation_state:
2524                 try:
2525                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2526                                                              "operationState": nslcmop_operation_state},
2527                                             loop=self.loop)
2528                     # if cooldown_time:
2529                     #     await asyncio.sleep(cooldown_time)
2530                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2531                 except Exception as e:
2532                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2533             self.logger.debug(logging_text + "Exit")
2534             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")