osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import yaml
  21 import logging
  22 import logging.handlers
  23 import functools
  24 import traceback
  25 from jinja2 import Environment, Template, meta, TemplateError, TemplateNotFound, TemplateSyntaxError
  26
  27 from osm_lcm import ROclient
  28 from osm_lcm.lcm_utils import LcmException, LcmExceptionNoMgmtIP, LcmBase
  29
  30 from osm_common.dbbase import DbException
  31 from osm_common.fsbase import FsException
  32 from n2vc.vnf import N2VC, N2VCPrimitiveExecutionFailed, NetworkServiceDoesNotExist, PrimitiveDoesNotExist
  33
  34 from copy import copy, deepcopy
  35 from http import HTTPStatus
  36 from time import time
  37 from uuid import uuid4
  38
  39 __author__ = "Alfonso Tierno"
  40
  41
  42 def get_iterable(in_dict, in_key):
  43     """
  44     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  45     :param in_dict: a dictionary
  46     :param in_key: the key to look for at in_dict
  47     :return: in_dict[in_var] or () if it is None or not present
  48     """
  49     if not in_dict.get(in_key):
  50         return ()
  51     return in_dict[in_key]
  52
  53
  54 def populate_dict(target_dict, key_list, value):
  55     """
  56     Upate target_dict creating nested dictionaries with the key_list. Last key_list item is asigned the value.
  57     Example target_dict={K: J}; key_list=[a,b,c];  target_dict will be {K: J, a: {b: {c: value}}}
  58     :param target_dict: dictionary to be changed
  59     :param key_list: list of keys to insert at target_dict
  60     :param value:
  61     :return: None
  62     """
  63     for key in key_list[0:-1]:
  64         if key not in target_dict:
  65             target_dict[key] = {}
  66         target_dict = target_dict[key]
  67     target_dict[key_list[-1]] = value
  68
  69
  70 class NsLcm(LcmBase):
  71     timeout_vca_on_error = 5 * 60   # Time for charm from first time at blocked,error status to mark as failed
  72     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  73     timeout_charm_delete = 10 * 60
  74     timeout_primitive = 10 * 60  # timeout for primitive execution
  75
  76     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  77         """
  78         Init, Connect to database, filesystem storage, and messaging
  79         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  80         :return: None
  81         """
  82         # logging
  83         self.logger = logging.getLogger('lcm.ns')
  84         self.loop = loop
  85         self.lcm_tasks = lcm_tasks
  86
  87         super().__init__(db, msg, fs, self.logger)
  88
  89         self.ro_config = ro_config
  90
  91         self.n2vc = N2VC(
  92             log=self.logger,
  93             server=vca_config['host'],
  94             port=vca_config['port'],
  95             user=vca_config['user'],
  96             secret=vca_config['secret'],
  97             # TODO: This should point to the base folder where charms are stored,
  98             # if there is a common one (like object storage). Otherwise, leave
  99             # it unset and pass it via DeployCharms
 100             # artifacts=vca_config[''],
 101             artifacts=None,
 102             juju_public_key=vca_config.get('pubkey'),
 103             ca_cert=vca_config.get('cacert'),
 104         )
 105         self.RO = ROclient.ROClient(self.loop, **self.ro_config)
 106
 107     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 108         """
 109         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 110         :param vnfd: input vnfd
 111         :param new_id: overrides vnf id if provided
 112         :param additionalParams: Instantiation params for VNFs provided
 113         :param nsrId: Id of the NSR
 114         :return: copy of vnfd
 115         """
 116         try:
 117             vnfd_RO = deepcopy(vnfd)
 118             # remove unused by RO configuration, monitoring, scaling and internal keys
 119             vnfd_RO.pop("_id", None)
 120             vnfd_RO.pop("_admin", None)
 121             vnfd_RO.pop("vnf-configuration", None)
 122             vnfd_RO.pop("monitoring-param", None)
 123             vnfd_RO.pop("scaling-group-descriptor", None)
 124             if new_id:
 125                 vnfd_RO["id"] = new_id
 126
 127             # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 128             for vdu in get_iterable(vnfd_RO, "vdu"):
 129                 cloud_init_file = None
 130                 if vdu.get("cloud-init-file"):
 131                     base_folder = vnfd["_admin"]["storage"]
 132                     cloud_init_file = "{}/{}/cloud_init/{}".format(base_folder["folder"], base_folder["pkg-dir"],
 133                                                                    vdu["cloud-init-file"])
 134                     with self.fs.file_open(cloud_init_file, "r") as ci_file:
 135                         cloud_init_content = ci_file.read()
 136                     vdu.pop("cloud-init-file", None)
 137                 elif vdu.get("cloud-init"):
 138                     cloud_init_content = vdu["cloud-init"]
 139                 else:
 140                     continue
 141
 142                 env = Environment()
 143                 ast = env.parse(cloud_init_content)
 144                 mandatory_vars = meta.find_undeclared_variables(ast)
 145                 if mandatory_vars:
 146                     for var in mandatory_vars:
 147                         if not additionalParams or var not in additionalParams.keys():
 148                             raise LcmException("Variable '{}' defined at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 149                                                "file, must be provided in the instantiation parameters inside the "
 150                                                "'additionalParamsForVnf' block".format(var, vnfd["id"], vdu["id"]))
 151                 template = Template(cloud_init_content)
 152                 cloud_init_content = template.render(additionalParams or {})
 153                 vdu["cloud-init"] = cloud_init_content
 154
 155             return vnfd_RO
 156         except FsException as e:
 157             raise LcmException("Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".
 158                                format(vnfd["id"], vdu["id"], cloud_init_file, e))
 159         except (TemplateError, TemplateNotFound, TemplateSyntaxError) as e:
 160             raise LcmException("Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".
 161                                format(vnfd["id"], vdu["id"], e))
 162
 163     def n2vc_callback(self, model_name, application_name, status, message, n2vc_info, task=None):
 164         """
 165         Callback both for charm status change and task completion
 166         :param model_name: Charm model name
 167         :param application_name: Charm application name
 168         :param status: Can be
 169             - blocked: The unit needs manual intervention
 170             - maintenance: The unit is actively deploying/configuring
 171             - waiting: The unit is waiting for another charm to be ready
 172             - active: The unit is deployed, configured, and ready
 173             - error: The charm has failed and needs attention.
 174             - terminated: The charm has been destroyed
 175             - removing,
 176             - removed
 177         :param message: detailed message error
 178         :param n2vc_info: dictionary with information shared with instantiate task. It contains:
 179             nsr_id:
 180             nslcmop_id:
 181             lcmOperationType: currently "instantiate"
 182             deployed: dictionary with {<application>: {operational-status: <status>, detailed-status: <text>}}
 183             db_update: dictionary to be filled with the changes to be wrote to database with format key.key.key: value
 184             n2vc_event: event used to notify instantiation task that some change has been produced
 185         :param task: None for charm status change, or task for completion task callback
 186         :return:
 187         """
 188         try:
 189             nsr_id = n2vc_info["nsr_id"]
 190             deployed = n2vc_info["deployed"]
 191             db_nsr_update = n2vc_info["db_update"]
 192             nslcmop_id = n2vc_info["nslcmop_id"]
 193             ns_operation = n2vc_info["lcmOperationType"]
 194             n2vc_event = n2vc_info["n2vc_event"]
 195             logging_text = "Task ns={} {}={} [n2vc_callback] application={}".format(nsr_id, ns_operation, nslcmop_id,
 196                                                                                     application_name)
 197             for vca_index, vca_deployed in enumerate(deployed):
 198                 if not vca_deployed:
 199                     continue
 200                 if model_name == vca_deployed["model"] and application_name == vca_deployed["application"]:
 201                     break
 202             else:
 203                 self.logger.error(logging_text + " Not present at nsr._admin.deployed.VCA. Received model_name={}".
 204                                   format(model_name))
 205                 return
 206             if task:
 207                 if task.cancelled():
 208                     self.logger.debug(logging_text + " task Cancelled")
 209                     vca_deployed['operational-status'] = "error"
 210                     db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 211                     vca_deployed['detailed-status'] = "Task Cancelled"
 212                     db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = "Task Cancelled"
 213
 214                 elif task.done():
 215                     exc = task.exception()
 216                     if exc:
 217                         self.logger.error(logging_text + " task Exception={}".format(exc))
 218                         vca_deployed['operational-status'] = "error"
 219                         db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = "error"
 220                         vca_deployed['detailed-status'] = str(exc)
 221                         db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(exc)
 222                     else:
 223                         self.logger.debug(logging_text + " task Done")
 224                         # task is Done, but callback is still ongoing. So ignore
 225                         return
 226             elif status:
 227                 self.logger.debug(logging_text + " Enter status={} message={}".format(status, message))
 228                 if vca_deployed['operational-status'] == status:
 229                     return  # same status, ignore
 230                 vca_deployed['operational-status'] = status
 231                 db_nsr_update["_admin.deployed.VCA.{}.operational-status".format(vca_index)] = status
 232                 vca_deployed['detailed-status'] = str(message)
 233                 db_nsr_update["_admin.deployed.VCA.{}.detailed-status".format(vca_index)] = str(message)
 234             else:
 235                 self.logger.critical(logging_text + " Enter with bad parameters", exc_info=True)
 236                 return
 237             # wake up instantiate task
 238             n2vc_event.set()
 239         except Exception as e:
 240             self.logger.critical(logging_text + " Exception {}".format(e), exc_info=True)
 241
 242     def ns_params_2_RO(self, ns_params, nsd, vnfd_dict, n2vc_key_list):
 243         """
 244         Creates a RO ns descriptor from OSM ns_instantiate params
 245         :param ns_params: OSM instantiate params
 246         :return: The RO ns descriptor
 247         """
 248         vim_2_RO = {}
 249         wim_2_RO = {}
 250         # TODO feature 1417: Check that no instantiation is set over PDU
 251         # check if PDU forces a concrete vim-network-id and add it
 252         # check if PDU contains a SDN-assist info (dpid, switch, port) and pass it to RO
 253
 254         def vim_account_2_RO(vim_account):
 255             if vim_account in vim_2_RO:
 256                 return vim_2_RO[vim_account]
 257
 258             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 259             if db_vim["_admin"]["operationalState"] != "ENABLED":
 260                 raise LcmException("VIM={} is not available. operationalState={}".format(
 261                     vim_account, db_vim["_admin"]["operationalState"]))
 262             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 263             vim_2_RO[vim_account] = RO_vim_id
 264             return RO_vim_id
 265
 266         def wim_account_2_RO(wim_account):
 267             if isinstance(wim_account, str):
 268                 if wim_account in wim_2_RO:
 269                     return wim_2_RO[wim_account]
 270
 271                 db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 272                 if db_wim["_admin"]["operationalState"] != "ENABLED":
 273                     raise LcmException("WIM={} is not available. operationalState={}".format(
 274                         wim_account, db_wim["_admin"]["operationalState"]))
 275                 RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 276                 wim_2_RO[wim_account] = RO_wim_id
 277                 return RO_wim_id
 278             else:
 279                 return wim_account
 280
 281         def ip_profile_2_RO(ip_profile):
 282             RO_ip_profile = deepcopy((ip_profile))
 283             if "dns-server" in RO_ip_profile:
 284                 if isinstance(RO_ip_profile["dns-server"], list):
 285                     RO_ip_profile["dns-address"] = []
 286                     for ds in RO_ip_profile.pop("dns-server"):
 287                         RO_ip_profile["dns-address"].append(ds['address'])
 288                 else:
 289                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 290             if RO_ip_profile.get("ip-version") == "ipv4":
 291                 RO_ip_profile["ip-version"] = "IPv4"
 292             if RO_ip_profile.get("ip-version") == "ipv6":
 293                 RO_ip_profile["ip-version"] = "IPv6"
 294             if "dhcp-params" in RO_ip_profile:
 295                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 296             return RO_ip_profile
 297
 298         if not ns_params:
 299             return None
 300         RO_ns_params = {
 301             # "name": ns_params["nsName"],
 302             # "description": ns_params.get("nsDescription"),
 303             "datacenter": vim_account_2_RO(ns_params["vimAccountId"]),
 304             "wim_account": wim_account_2_RO(ns_params.get("wimAccountId")),
 305             # "scenario": ns_params["nsdId"],
 306         }
 307         if n2vc_key_list:
 308             for vnfd_ref, vnfd in vnfd_dict.items():
 309                 vdu_needed_access = []
 310                 mgmt_cp = None
 311                 if vnfd.get("vnf-configuration"):
 312                     if vnfd.get("mgmt-interface"):
 313                         if vnfd["mgmt-interface"].get("vdu-id"):
 314                             vdu_needed_access.append(vnfd["mgmt-interface"]["vdu-id"])
 315                         elif vnfd["mgmt-interface"].get("cp"):
 316                             mgmt_cp = vnfd["mgmt-interface"]["cp"]
 317
 318                 for vdu in vnfd.get("vdu", ()):
 319                     if vdu.get("vdu-configuration"):
 320                         vdu_needed_access.append(vdu["id"])
 321                     elif mgmt_cp:
 322                         for vdu_interface in vdu.get("interface"):
 323                             if vdu_interface.get("external-connection-point-ref") and \
 324                                     vdu_interface["external-connection-point-ref"] == mgmt_cp:
 325                                 vdu_needed_access.append(vdu["id"])
 326                                 mgmt_cp = None
 327                                 break
 328
 329                 if vdu_needed_access:
 330                     for vnf_member in nsd.get("constituent-vnfd"):
 331                         if vnf_member["vnfd-id-ref"] != vnfd_ref:
 332                             continue
 333                         for vdu in vdu_needed_access:
 334                             populate_dict(RO_ns_params,
 335                                           ("vnfs", vnf_member["member-vnf-index"], "vdus", vdu, "mgmt_keys"),
 336                                           n2vc_key_list)
 337
 338         if ns_params.get("vduImage"):
 339             RO_ns_params["vduImage"] = ns_params["vduImage"]
 340
 341         if ns_params.get("ssh_keys"):
 342             RO_ns_params["cloud-config"] = {"key-pairs": ns_params["ssh_keys"]}
 343         for vnf_params in get_iterable(ns_params, "vnf"):
 344             for constituent_vnfd in nsd["constituent-vnfd"]:
 345                 if constituent_vnfd["member-vnf-index"] == vnf_params["member-vnf-index"]:
 346                     vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 347                     break
 348             else:
 349                 raise LcmException("Invalid instantiate parameter vnf:member-vnf-index={} is not present at nsd:"
 350                                    "constituent-vnfd".format(vnf_params["member-vnf-index"]))
 351             if vnf_params.get("vimAccountId"):
 352                 populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "datacenter"),
 353                               vim_account_2_RO(vnf_params["vimAccountId"]))
 354
 355             for vdu_params in get_iterable(vnf_params, "vdu"):
 356                 # TODO feature 1417: check that this VDU exist and it is not a PDU
 357                 if vdu_params.get("volume"):
 358                     for volume_params in vdu_params["volume"]:
 359                         if volume_params.get("vim-volume-id"):
 360                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 361                                                          vdu_params["id"], "devices", volume_params["name"], "vim_id"),
 362                                           volume_params["vim-volume-id"])
 363                 if vdu_params.get("interface"):
 364                     for interface_params in vdu_params["interface"]:
 365                         if interface_params.get("ip-address"):
 366                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 367                                                          vdu_params["id"], "interfaces", interface_params["name"],
 368                                                          "ip_address"),
 369                                           interface_params["ip-address"])
 370                         if interface_params.get("mac-address"):
 371                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 372                                                          vdu_params["id"], "interfaces", interface_params["name"],
 373                                                          "mac_address"),
 374                                           interface_params["mac-address"])
 375                         if interface_params.get("floating-ip-required"):
 376                             populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 377                                                          vdu_params["id"], "interfaces", interface_params["name"],
 378                                                          "floating-ip"),
 379                                           interface_params["floating-ip-required"])
 380
 381             for internal_vld_params in get_iterable(vnf_params, "internal-vld"):
 382                 if internal_vld_params.get("vim-network-name"):
 383                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 384                                                  internal_vld_params["name"], "vim-network-name"),
 385                                   internal_vld_params["vim-network-name"])
 386                 if internal_vld_params.get("vim-network-id"):
 387                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 388                                                  internal_vld_params["name"], "vim-network-id"),
 389                                   internal_vld_params["vim-network-id"])
 390                 if internal_vld_params.get("ip-profile"):
 391                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "networks",
 392                                                  internal_vld_params["name"], "ip-profile"),
 393                                   ip_profile_2_RO(internal_vld_params["ip-profile"]))
 394
 395                 for icp_params in get_iterable(internal_vld_params, "internal-connection-point"):
 396                     # look for interface
 397                     iface_found = False
 398                     for vdu_descriptor in vnf_descriptor["vdu"]:
 399                         for vdu_interface in vdu_descriptor["interface"]:
 400                             if vdu_interface.get("internal-connection-point-ref") == icp_params["id-ref"]:
 401                                 if icp_params.get("ip-address"):
 402                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 403                                                                  vdu_descriptor["id"], "interfaces",
 404                                                                  vdu_interface["name"], "ip_address"),
 405                                                   icp_params["ip-address"])
 406
 407                                 if icp_params.get("mac-address"):
 408                                     populate_dict(RO_ns_params, ("vnfs", vnf_params["member-vnf-index"], "vdus",
 409                                                                  vdu_descriptor["id"], "interfaces",
 410                                                                  vdu_interface["name"], "mac_address"),
 411                                                   icp_params["mac-address"])
 412                                 iface_found = True
 413                                 break
 414                         if iface_found:
 415                             break
 416                     else:
 417                         raise LcmException("Invalid instantiate parameter vnf:member-vnf-index[{}]:"
 418                                            "internal-vld:id-ref={} is not present at vnfd:internal-"
 419                                            "connection-point".format(vnf_params["member-vnf-index"],
 420                                                                      icp_params["id-ref"]))
 421
 422         for vld_params in get_iterable(ns_params, "vld"):
 423             if "ip-profile" in vld_params:
 424                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 425                               ip_profile_2_RO(vld_params["ip-profile"]))
 426
 427             if "wimAccountId" in vld_params and vld_params["wimAccountId"] is not None:
 428                 populate_dict(RO_ns_params, ("networks", vld_params["name"], "wim_account"),
 429                               wim_account_2_RO(vld_params["wimAccountId"])),
 430             if vld_params.get("vim-network-name"):
 431                 RO_vld_sites = []
 432                 if isinstance(vld_params["vim-network-name"], dict):
 433                     for vim_account, vim_net in vld_params["vim-network-name"].items():
 434                         RO_vld_sites.append({
 435                             "netmap-use": vim_net,
 436                             "datacenter": vim_account_2_RO(vim_account)
 437                         })
 438                 else:  # isinstance str
 439                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-name"]})
 440                 if RO_vld_sites:
 441                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 442             if vld_params.get("vim-network-id"):
 443                 RO_vld_sites = []
 444                 if isinstance(vld_params["vim-network-id"], dict):
 445                     for vim_account, vim_net in vld_params["vim-network-id"].items():
 446                         RO_vld_sites.append({
 447                             "netmap-use": vim_net,
 448                             "datacenter": vim_account_2_RO(vim_account)
 449                         })
 450                 else:  # isinstance str
 451                     RO_vld_sites.append({"netmap-use": vld_params["vim-network-id"]})
 452                 if RO_vld_sites:
 453                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "sites"), RO_vld_sites)
 454             if vld_params.get("ns-net"):
 455                 if isinstance(vld_params["ns-net"], dict):
 456                     for vld_id, instance_scenario_id in vld_params["ns-net"].items():
 457                         RO_vld_ns_net = {"instance_scenario_id": instance_scenario_id, "osm_id": vld_id}
 458                 if RO_vld_ns_net:
 459                     populate_dict(RO_ns_params, ("networks", vld_params["name"], "use-network"), RO_vld_ns_net)
 460             if "vnfd-connection-point-ref" in vld_params:
 461                 for cp_params in vld_params["vnfd-connection-point-ref"]:
 462                     # look for interface
 463                     for constituent_vnfd in nsd["constituent-vnfd"]:
 464                         if constituent_vnfd["member-vnf-index"] == cp_params["member-vnf-index-ref"]:
 465                             vnf_descriptor = vnfd_dict[constituent_vnfd["vnfd-id-ref"]]
 466                             break
 467                     else:
 468                         raise LcmException(
 469                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={} "
 470                             "is not present at nsd:constituent-vnfd".format(cp_params["member-vnf-index-ref"]))
 471                     match_cp = False
 472                     for vdu_descriptor in vnf_descriptor["vdu"]:
 473                         for interface_descriptor in vdu_descriptor["interface"]:
 474                             if interface_descriptor.get("external-connection-point-ref") == \
 475                                     cp_params["vnfd-connection-point-ref"]:
 476                                 match_cp = True
 477                                 break
 478                         if match_cp:
 479                             break
 480                     else:
 481                         raise LcmException(
 482                             "Invalid instantiate parameter vld:vnfd-connection-point-ref:member-vnf-index-ref={}:"
 483                             "vnfd-connection-point-ref={} is not present at vnfd={}".format(
 484                                 cp_params["member-vnf-index-ref"],
 485                                 cp_params["vnfd-connection-point-ref"],
 486                                 vnf_descriptor["id"]))
 487                     if cp_params.get("ip-address"):
 488                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 489                                                      vdu_descriptor["id"], "interfaces",
 490                                                      interface_descriptor["name"], "ip_address"),
 491                                       cp_params["ip-address"])
 492                     if cp_params.get("mac-address"):
 493                         populate_dict(RO_ns_params, ("vnfs", cp_params["member-vnf-index-ref"], "vdus",
 494                                                      vdu_descriptor["id"], "interfaces",
 495                                                      interface_descriptor["name"], "mac_address"),
 496                                       cp_params["mac-address"])
 497         return RO_ns_params
 498
 499     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None):
 500         # make a copy to do not change
 501         vdu_create = copy(vdu_create)
 502         vdu_delete = copy(vdu_delete)
 503
 504         vdurs = db_vnfr.get("vdur")
 505         if vdurs is None:
 506             vdurs = []
 507         vdu_index = len(vdurs)
 508         while vdu_index:
 509             vdu_index -= 1
 510             vdur = vdurs[vdu_index]
 511             if vdur.get("pdu-type"):
 512                 continue
 513             vdu_id_ref = vdur["vdu-id-ref"]
 514             if vdu_create and vdu_create.get(vdu_id_ref):
 515                 for index in range(0, vdu_create[vdu_id_ref]):
 516                     vdur = deepcopy(vdur)
 517                     vdur["_id"] = str(uuid4())
 518                     vdur["count-index"] += 1
 519                     vdurs.insert(vdu_index+1+index, vdur)
 520                 del vdu_create[vdu_id_ref]
 521             if vdu_delete and vdu_delete.get(vdu_id_ref):
 522                 del vdurs[vdu_index]
 523                 vdu_delete[vdu_id_ref] -= 1
 524                 if not vdu_delete[vdu_id_ref]:
 525                     del vdu_delete[vdu_id_ref]
 526         # check all operations are done
 527         if vdu_create or vdu_delete:
 528             raise LcmException("Error scaling OUT VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 529                 vdu_create))
 530         if vdu_delete:
 531             raise LcmException("Error scaling IN VNFR for {}. There is not any existing vnfr. Scaled to 0?".format(
 532                 vdu_delete))
 533
 534         vnfr_update = {"vdur": vdurs}
 535         db_vnfr["vdur"] = vdurs
 536         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 537
 538     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 539         """
 540         Updates database nsr with the RO info for the created vld
 541         :param ns_update_nsr: dictionary to be filled with the updated info
 542         :param db_nsr: content of db_nsr. This is also modified
 543         :param nsr_desc_RO: nsr descriptor from RO
 544         :return: Nothing, LcmException is raised on errors
 545         """
 546
 547         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 548             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 549                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 550                     continue
 551                 vld["vim-id"] = net_RO.get("vim_net_id")
 552                 vld["name"] = net_RO.get("vim_name")
 553                 vld["status"] = net_RO.get("status")
 554                 vld["status-detailed"] = net_RO.get("error_msg")
 555                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 556                 break
 557             else:
 558                 raise LcmException("ns_update_nsr: Not found vld={} at RO info".format(vld["id"]))
 559
 560     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 561         """
 562         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 563         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 564         :param nsr_desc_RO: nsr descriptor from RO
 565         :return: Nothing, LcmException is raised on errors
 566         """
 567         for vnf_index, db_vnfr in db_vnfrs.items():
 568             for vnf_RO in nsr_desc_RO["vnfs"]:
 569                 if vnf_RO["member_vnf_index"] != vnf_index:
 570                     continue
 571                 vnfr_update = {}
 572                 if vnf_RO.get("ip_address"):
 573                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO["ip_address"].split(";")[0]
 574                 elif not db_vnfr.get("ip-address"):
 575                     raise LcmExceptionNoMgmtIP("ns member_vnf_index '{}' has no IP address".format(vnf_index))
 576
 577                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 578                     vdur_RO_count_index = 0
 579                     if vdur.get("pdu-type"):
 580                         continue
 581                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 582                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 583                             continue
 584                         if vdur["count-index"] != vdur_RO_count_index:
 585                             vdur_RO_count_index += 1
 586                             continue
 587                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 588                         if vdur_RO.get("ip_address"):
 589                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 590                         else:
 591                             vdur["ip-address"] = None
 592                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 593                         vdur["name"] = vdur_RO.get("vim_name")
 594                         vdur["status"] = vdur_RO.get("status")
 595                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 596                         for ifacer in get_iterable(vdur, "interfaces"):
 597                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 598                                 if ifacer["name"] == interface_RO.get("internal_name"):
 599                                     ifacer["ip-address"] = interface_RO.get("ip_address")
 600                                     ifacer["mac-address"] = interface_RO.get("mac_address")
 601                                     break
 602                             else:
 603                                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 604                                                    "from VIM info".format(vnf_index, vdur["vdu-id-ref"],
 605                                                                           ifacer["name"]))
 606                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 607                         break
 608                     else:
 609                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 610                                            "VIM info".format(vnf_index, vdur["vdu-id-ref"], vdur["count-index"]))
 611
 612                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 613                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 614                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 615                             continue
 616                         vld["vim-id"] = net_RO.get("vim_net_id")
 617                         vld["name"] = net_RO.get("vim_name")
 618                         vld["status"] = net_RO.get("status")
 619                         vld["status-detailed"] = net_RO.get("error_msg")
 620                         vnfr_update["vld.{}".format(vld_index)] = vld
 621                         break
 622                     else:
 623                         raise LcmException("ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 624                             vnf_index, vld["id"]))
 625
 626                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 627                 break
 628
 629             else:
 630                 raise LcmException("ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(vnf_index))
 631
 632     async def instantiate(self, nsr_id, nslcmop_id):
 633
 634         # Try to lock HA task here
 635         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
 636         if not task_is_locked_by_me:
 637             return
 638
 639         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
 640         self.logger.debug(logging_text + "Enter")
 641         # get all needed from database
 642         start_deploy = time()
 643         db_nsr = None
 644         db_nslcmop = None
 645         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
 646         db_nslcmop_update = {}
 647         nslcmop_operation_state = None
 648         db_vnfrs = {}
 649         RO_descriptor_number = 0   # number of descriptors created at RO
 650         vnf_index_2_RO_id = {}    # map between vnfd/nsd id to the id used at RO
 651         n2vc_info = {}
 652         n2vc_key_list = []  # list of public keys to be injected as authorized to VMs
 653         exc = None
 654         try:
 655             # wait for any previous tasks in process
 656             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
 657
 658             step = "Getting nslcmop={} from db".format(nslcmop_id)
 659             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 660             step = "Getting nsr={} from db".format(nsr_id)
 661             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 662             ns_params = db_nslcmop.get("operationParams")
 663             nsd = db_nsr["nsd"]
 664             nsr_name = db_nsr["name"]   # TODO short-name??
 665
 666             step = "Getting vnfrs from db"
 667             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
 668             db_vnfds_ref = {}
 669             db_vnfds = {}
 670             db_vnfds_index = {}
 671             for vnfr in db_vnfrs_list:
 672                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
 673                 vnfd_id = vnfr["vnfd-id"]
 674                 vnfd_ref = vnfr["vnfd-ref"]
 675                 if vnfd_id not in db_vnfds:
 676                     step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_ref)
 677                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
 678                     db_vnfds_ref[vnfd_ref] = vnfd
 679                     db_vnfds[vnfd_id] = vnfd
 680                 db_vnfds_index[vnfr["member-vnf-index-ref"]] = db_vnfds[vnfd_id]
 681
 682             # Get or generates the _admin.deployed,VCA list
 683             vca_deployed_list = None
 684             vca_model_name = None
 685             if db_nsr["_admin"].get("deployed"):
 686                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
 687                 vca_model_name = db_nsr["_admin"]["deployed"].get("VCA-model-name")
 688             if vca_deployed_list is None:
 689                 vca_deployed_list = []
 690                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 691                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 692             elif isinstance(vca_deployed_list, dict):
 693                 # maintain backward compatibility. Change a dict to list at database
 694                 vca_deployed_list = list(vca_deployed_list.values())
 695                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
 696                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
 697
 698             db_nsr_update["detailed-status"] = "creating"
 699             db_nsr_update["operational-status"] = "init"
 700             if not db_nsr["_admin"].get("deployed") or not db_nsr["_admin"]["deployed"].get("RO") or \
 701                     not db_nsr["_admin"]["deployed"]["RO"].get("vnfd"):
 702                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
 703                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
 704
 705             # set state to INSTANTIATED. When instantiated NBI will not delete directly
 706             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
 707             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 708
 709             # Deploy charms
 710             # The parameters we'll need to deploy a charm
 711             number_to_configure = 0
 712
 713             def deploy_charm(vnf_index, vdu_id, vdu_name, vdu_count_index, charm_params, n2vc_info, native_charm=False):
 714                 """An inner function to deploy the charm from either ns, vnf or vdu
 715                 For ns both vnf_index and vdu_id are None.
 716                 For vnf only vdu_id is None
 717                 For vdu both vnf_index and vdu_id contain a value
 718                 """
 719                 # if not charm_params.get("rw_mgmt_ip") and vnf_index:  # if NS skip mgmt_ip checking
 720                 #     raise LcmException("ns/vnfd/vdu has not management ip address to configure it")
 721
 722                 machine_spec = {}
 723                 if native_charm:
 724                     machine_spec["username"] = charm_params.get("username"),
 725                     machine_spec["hostname"] = charm_params.get("rw_mgmt_ip")
 726
 727                 # Note: The charm needs to exist on disk at the location
 728                 # specified by charm_path.
 729                 descriptor = vnfd if vnf_index else nsd
 730                 base_folder = descriptor["_admin"]["storage"]
 731                 storage_params = self.fs.get_params()
 732                 charm_path = "{}{}/{}/charms/{}".format(
 733                     storage_params["path"],
 734                     base_folder["folder"],
 735                     base_folder["pkg-dir"],
 736                     proxy_charm
 737                 )
 738
 739                 # ns_name will be ignored in the current version of N2VC
 740                 # but will be implemented for the next point release.
 741                 model_name = nsr_id
 742                 vdu_id_text = (str(vdu_id) if vdu_id else "") + "-"
 743                 vnf_index_text = (str(vnf_index) if vnf_index else "") + "-"
 744                 application_name = self.n2vc.FormatApplicationName(nsr_name, vnf_index_text, vdu_id_text)
 745
 746                 vca_index = len(vca_deployed_list)
 747                 # trunk name and add two char index at the end to ensure that it is unique. It is assumed no more than
 748                 # 26*26 charm in the same NS
 749                 application_name = application_name[0:48]
 750                 application_name += chr(97 + vca_index // 26) + chr(97 + vca_index % 26)
 751                 vca_deployed_ = {
 752                     "member-vnf-index": vnf_index,
 753                     "vdu_id": vdu_id,
 754                     "model": model_name,
 755                     "application": application_name,
 756                     "operational-status": "init",
 757                     "detailed-status": "",
 758                     "step": "initial-deploy",
 759                     "vnfd_id": vnfd_id,
 760                     "vdu_name": vdu_name,
 761                     "vdu_count_index": vdu_count_index,
 762                 }
 763                 vca_deployed_list.append(vca_deployed_)
 764                 db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = vca_deployed_
 765                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 766
 767                 self.logger.debug("Task create_ns={} Passing artifacts path '{}' for {}".format(nsr_id, charm_path,
 768                                                                                                 proxy_charm))
 769                 if not n2vc_info:
 770                     n2vc_info["nsr_id"] = nsr_id
 771                     n2vc_info["nslcmop_id"] = nslcmop_id
 772                     n2vc_info["n2vc_event"] = asyncio.Event(loop=self.loop)
 773                     n2vc_info["lcmOperationType"] = "instantiate"
 774                     n2vc_info["deployed"] = vca_deployed_list
 775                     n2vc_info["db_update"] = db_nsr_update
 776                 task = asyncio.ensure_future(
 777                     self.n2vc.DeployCharms(
 778                         model_name,          # The network service name
 779                         application_name,    # The application name
 780                         descriptor,          # The vnf/nsd descriptor
 781                         charm_path,          # Path to charm
 782                         charm_params,        # Runtime params, like mgmt ip
 783                         machine_spec,        # for native charms only
 784                         self.n2vc_callback,  # Callback for status changes
 785                         n2vc_info,           # Callback parameter
 786                         None,                # Callback parameter (task)
 787                     )
 788                 )
 789                 task.add_done_callback(functools.partial(self.n2vc_callback, model_name, application_name, None, None,
 790                                                          n2vc_info))
 791                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "create_charm:" + application_name, task)
 792
 793             step = "Looking for needed vnfd to configure with proxy charm"
 794             self.logger.debug(logging_text + step)
 795
 796             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
 797                 vnfd_id = c_vnf["vnfd-id-ref"]
 798                 vnf_index = str(c_vnf["member-vnf-index"])
 799                 vnfd = db_vnfds_ref[vnfd_id]
 800
 801                 # Get additional parameters
 802                 vnfr_params = {}
 803                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
 804                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
 805                 for k, v in vnfr_params.items():
 806                     if isinstance(v, str) and v.startswith("!!yaml "):
 807                         vnfr_params[k] = yaml.safe_load(v[7:])
 808
 809                 step = "deploying proxy charms for configuration"
 810                 # Check if this VNF has a charm configuration
 811                 vnf_config = vnfd.get("vnf-configuration")
 812                 if vnf_config and vnf_config.get("juju"):
 813                     proxy_charm = vnf_config["juju"]["charm"]
 814                     if vnf_config["juju"].get("proxy") is False:
 815                         # native_charm, will be deployed after VM. Skip
 816                         proxy_charm = None
 817
 818                     if proxy_charm:
 819                         if not vca_model_name:
 820                             step = "creating VCA model name '{}'".format(nsr_id)
 821                             self.logger.debug(logging_text + step)
 822                             await self.n2vc.CreateNetworkService(nsr_id)
 823                             vca_model_name = nsr_id
 824                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 825                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
 826                         step = "deploying proxy charm to configure vnf {}".format(vnf_index)
 827                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
 828                         charm_params = {
 829                             "user_values": vnfr_params,
 830                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
 831                             "initial-config-primitive": {}    # vnf_config.get('initial-config-primitive') or {}
 832                         }
 833
 834                         # Login to the VCA. If there are multiple calls to login(),
 835                         # subsequent calls will be a nop and return immediately.
 836                         await self.n2vc.login()
 837
 838                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info)
 839                         number_to_configure += 1
 840
 841                 # Deploy charms for each VDU that supports one.
 842                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
 843                     vdu_config = vdu.get('vdu-configuration')
 844                     proxy_charm = None
 845
 846                     if vdu_config and vdu_config.get("juju"):
 847                         proxy_charm = vdu_config["juju"]["charm"]
 848                         if vdu_config["juju"].get("proxy") is False:
 849                             # native_charm, will be deployed after VM. Skip
 850                             proxy_charm = None
 851
 852                         if proxy_charm:
 853                             if not vca_model_name:
 854                                 step = "creating VCA model name"
 855                                 await self.n2vc.CreateNetworkService(nsr_id)
 856                                 vca_model_name = nsr_id
 857                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
 858                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
 859                             step = "deploying proxy charm to configure member_vnf_index={} vdu={}".format(vnf_index,
 860                                                                                                           vdu["id"])
 861                             await self.n2vc.login()
 862                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
 863                             # TODO for the moment only first vdu_id contains a charm deployed
 864                             if vdur["vdu-id-ref"] != vdu["id"]:
 865                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for member_vnf_index={}"
 866                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
 867                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
 868                             charm_params = {
 869                                 "user_values": vnfr_params,
 870                                 "rw_mgmt_ip": vdur["ip-address"],
 871                                 "initial-config-primitive": {}  # vdu_config.get('initial-config-primitive') or {}
 872                             }
 873                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
 874                                          charm_params, n2vc_info)
 875                             number_to_configure += 1
 876
 877             # Check if this NS has a charm configuration
 878
 879             ns_config = nsd.get("ns-configuration")
 880             if ns_config and ns_config.get("juju"):
 881                 proxy_charm = ns_config["juju"]["charm"]
 882                 if ns_config["juju"].get("proxy") is False:
 883                     # native_charm, will be deployed after VM. Skip
 884                     proxy_charm = None
 885
 886                 if proxy_charm:
 887                     step = "deploying proxy charm to configure ns"
 888                     # TODO is NS magmt IP address needed?
 889
 890                     # Get additional parameters
 891                     additional_params = {}
 892                     if db_nsr.get("additionalParamsForNs"):
 893                         additional_params = db_nsr["additionalParamsForNs"].copy()
 894                     for k, v in additional_params.items():
 895                         if isinstance(v, str) and v.startswith("!!yaml "):
 896                             additional_params[k] = yaml.safe_load(v[7:])
 897
 898                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
 899                     charm_params = {
 900                         "user_values": additional_params,
 901                         # "rw_mgmt_ip": db_nsr["ip-address"],
 902                         "initial-config-primitive": {}   # ns_config.get('initial-config-primitive') or {}
 903                     }
 904
 905                     # Login to the VCA. If there are multiple calls to login(),
 906                     # subsequent calls will be a nop and return immediately.
 907                     await self.n2vc.login()
 908                     deploy_charm(None, None, None, None, charm_params, n2vc_info)
 909                     number_to_configure += 1
 910
 911             db_nsr_update["operational-status"] = "running"
 912
 913             # Wait until all charms has reached blocked or active status
 914             step = "waiting proxy charms to be ready"
 915             if number_to_configure:
 916                 # wait until all charms are configured.
 917                 # steps are:
 918                 #       initial-deploy
 919                 #       get-ssh-public-key
 920                 #       generate-ssh-key
 921                 #       retry-get-ssh-public-key
 922                 #       ssh-public-key-obtained
 923                 while time() <= start_deploy + self.total_deploy_timeout:
 924                     if db_nsr_update:
 925                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
 926                     if db_nslcmop_update:
 927                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
 928
 929                     all_active = True
 930                     for vca_index, vca_deployed in enumerate(vca_deployed_list):
 931                         database_entry = "_admin.deployed.VCA.{}.".format(vca_index)
 932                         if vca_deployed["step"] == "initial-deploy":
 933                             if vca_deployed["operational-status"] in ("active", "blocked"):
 934                                 step = "execute charm primitive get-ssh-public-key for member_vnf_index={} vdu_id={}" \
 935                                        .format(vca_deployed["member-vnf-index"],
 936                                                vca_deployed["vdu_id"])
 937                                 self.logger.debug(logging_text + step)
 938                                 try:
 939                                     primitive_id = await self.n2vc.ExecutePrimitive(
 940                                         vca_deployed["model"],
 941                                         vca_deployed["application"],
 942                                         "get-ssh-public-key",
 943                                         None,
 944                                     )
 945                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = "get-ssh-public-key"
 946                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 947                                         primitive_id
 948                                     db_nsr_update[database_entry + "operational-status"] =\
 949                                         vca_deployed["operational-status"]
 950                                 except PrimitiveDoesNotExist:
 951                                     ssh_public_key = None
 952                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 953                                         "ssh-public-key-obtained"
 954                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 955                                         ssh_public_key
 956                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} not needed".format(
 957                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"])
 958                                     self.logger.debug(logging_text + step)
 959
 960                         elif vca_deployed["step"] in ("get-ssh-public-key", "retry-get-ssh-public-key"):
 961                             primitive_id = vca_deployed["primitive_id"]
 962                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
 963                                                                                   primitive_id)
 964                             if primitive_status in ("completed", "failed"):
 965                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
 966                                                                                       primitive_id)
 967                                 vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] = None
 968                                 if primitive_status == "completed" and isinstance(primitive_result, dict) and \
 969                                         primitive_result.get("pubkey"):
 970                                     ssh_public_key = primitive_result.get("pubkey")
 971                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 972                                         "ssh-public-key-obtained"
 973                                     vca_deployed["ssh-public-key"] = db_nsr_update[database_entry + "ssh-public-key"] =\
 974                                         ssh_public_key
 975                                     n2vc_key_list.append(ssh_public_key)
 976                                     step = "charm ssh-public-key for  member_vnf_index={} vdu_id={} is '{}'".format(
 977                                         vca_deployed["member-vnf-index"], vca_deployed["vdu_id"], ssh_public_key)
 978                                     self.logger.debug(logging_text + step)
 979                                 else:  # primitive_status == "failed":
 980                                     if vca_deployed["step"] == "get-ssh-public-key":
 981                                         step = "execute charm primitive generate-ssh-public-key for member_vnf_index="\
 982                                                "{} vdu_id={}".format(vca_deployed["member-vnf-index"],
 983                                                                      vca_deployed["vdu_id"])
 984                                         self.logger.debug(logging_text + step)
 985                                         vca_deployed["step"] = db_nsr_update[database_entry + "step"] =\
 986                                             "generate-ssh-key"
 987                                         primitive_id = await self.n2vc.ExecutePrimitive(
 988                                             vca_deployed["model"],
 989                                             vca_deployed["application"],
 990                                             "generate-ssh-key",
 991                                             None,
 992                                         )
 993                                         vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
 994                                             primitive_id
 995                                     else:  # failed for second time
 996                                         raise LcmException(
 997                                             "error executing primitive get-ssh-public-key: {}".format(primitive_result))
 998
 999                         elif vca_deployed["step"] == "generate-ssh-key":
1000                             primitive_id = vca_deployed["primitive_id"]
1001                             primitive_status = await self.n2vc.GetPrimitiveStatus(vca_deployed["model"],
1002                                                                                   primitive_id)
1003                             if primitive_status in ("completed", "failed"):
1004                                 primitive_result = await self.n2vc.GetPrimitiveOutput(vca_deployed["model"],
1005                                                                                       primitive_id)
1006                                 vca_deployed["primitive_id"] = db_nsr_update[
1007                                     database_entry + "primitive_id"] = None
1008                                 if primitive_status == "completed":
1009                                     step = "execute primitive get-ssh-public-key again for member_vnf_index={} "\
1010                                            "vdu_id={}".format(vca_deployed["member-vnf-index"],
1011                                                               vca_deployed["vdu_id"])
1012                                     self.logger.debug(logging_text + step)
1013                                     vca_deployed["step"] = db_nsr_update[database_entry + "step"] = \
1014                                         "retry-get-ssh-public-key"
1015                                     primitive_id = await self.n2vc.ExecutePrimitive(
1016                                         vca_deployed["model"],
1017                                         vca_deployed["application"],
1018                                         "get-ssh-public-key",
1019                                         None,
1020                                     )
1021                                     vca_deployed["primitive_id"] = db_nsr_update[database_entry + "primitive_id"] =\
1022                                         primitive_id
1023
1024                                 else:  # primitive_status == "failed":
1025                                     raise LcmException("error executing primitive  generate-ssh-key: {}"
1026                                                        .format(primitive_result))
1027
1028                         if vca_deployed["step"] != "ssh-public-key-obtained":
1029                             all_active = False
1030
1031                     if all_active:
1032                         break
1033                     await asyncio.sleep(5)
1034                 else:   # total_deploy_timeout
1035                     raise LcmException("Timeout waiting charm to be initialized for member_vnf_index={} vdu_id={}"
1036                                        .format(vca_deployed["member-vnf-index"], vca_deployed["vdu_id"]))
1037
1038             # deploy RO
1039             # get vnfds, instantiate at RO
1040             for c_vnf in nsd.get("constituent-vnfd", ()):
1041                 member_vnf_index = c_vnf["member-vnf-index"]
1042                 vnfd = db_vnfds_ref[c_vnf['vnfd-id-ref']]
1043                 vnfd_ref = vnfd["id"]
1044                 step = db_nsr_update["detailed-status"] = "Creating vnfd='{}' member_vnf_index='{}' at RO".format(
1045                     vnfd_ref, member_vnf_index)
1046                 # self.logger.debug(logging_text + step)
1047                 vnfd_id_RO = "{}.{}.{}".format(nsr_id, RO_descriptor_number, member_vnf_index[:23])
1048                 vnf_index_2_RO_id[member_vnf_index] = vnfd_id_RO
1049                 RO_descriptor_number += 1
1050
1051                 # look position at deployed.RO.vnfd if not present it will be appended at the end
1052                 for index, vnf_deployed in enumerate(db_nsr["_admin"]["deployed"]["RO"]["vnfd"]):
1053                     if vnf_deployed["member-vnf-index"] == member_vnf_index:
1054                         break
1055                 else:
1056                     index = len(db_nsr["_admin"]["deployed"]["RO"]["vnfd"])
1057                     db_nsr["_admin"]["deployed"]["RO"]["vnfd"].append(None)
1058
1059                 # look if present
1060                 RO_update = {"member-vnf-index": member_vnf_index}
1061                 vnfd_list = await self.RO.get_list("vnfd", filter_by={"osm_id": vnfd_id_RO})
1062                 if vnfd_list:
1063                     RO_update["id"] = vnfd_list[0]["uuid"]
1064                     self.logger.debug(logging_text + "vnfd='{}'  member_vnf_index='{}' exists at RO. Using RO_id={}".
1065                                       format(vnfd_ref, member_vnf_index, vnfd_list[0]["uuid"]))
1066                 else:
1067                     vnfd_RO = self.vnfd2RO(vnfd, vnfd_id_RO, db_vnfrs[c_vnf["member-vnf-index"]].
1068                                            get("additionalParamsForVnf"), nsr_id)
1069                     desc = await self.RO.create("vnfd", descriptor=vnfd_RO)
1070                     RO_update["id"] = desc["uuid"]
1071                     self.logger.debug(logging_text + "vnfd='{}' member_vnf_index='{}' created at RO. RO_id={}".format(
1072                         vnfd_ref, member_vnf_index, desc["uuid"]))
1073                 db_nsr_update["_admin.deployed.RO.vnfd.{}".format(index)] = RO_update
1074                 db_nsr["_admin"]["deployed"]["RO"]["vnfd"][index] = RO_update
1075                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1076
1077             # create nsd at RO
1078             nsd_ref = nsd["id"]
1079             step = db_nsr_update["detailed-status"] = "Creating nsd={} at RO".format(nsd_ref)
1080             # self.logger.debug(logging_text + step)
1081
1082             RO_osm_nsd_id = "{}.{}.{}".format(nsr_id, RO_descriptor_number, nsd_ref[:23])
1083             RO_descriptor_number += 1
1084             nsd_list = await self.RO.get_list("nsd", filter_by={"osm_id": RO_osm_nsd_id})
1085             if nsd_list:
1086                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = nsd_list[0]["uuid"]
1087                 self.logger.debug(logging_text + "nsd={} exists at RO. Using RO_id={}".format(
1088                     nsd_ref, RO_nsd_uuid))
1089             else:
1090                 nsd_RO = deepcopy(nsd)
1091                 nsd_RO["id"] = RO_osm_nsd_id
1092                 nsd_RO.pop("_id", None)
1093                 nsd_RO.pop("_admin", None)
1094                 for c_vnf in nsd_RO.get("constituent-vnfd", ()):
1095                     member_vnf_index = c_vnf["member-vnf-index"]
1096                     c_vnf["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1097                 for c_vld in nsd_RO.get("vld", ()):
1098                     for cp in c_vld.get("vnfd-connection-point-ref", ()):
1099                         member_vnf_index = cp["member-vnf-index-ref"]
1100                         cp["vnfd-id-ref"] = vnf_index_2_RO_id[member_vnf_index]
1101
1102                 desc = await self.RO.create("nsd", descriptor=nsd_RO)
1103                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1104                 db_nsr_update["_admin.deployed.RO.nsd_id"] = RO_nsd_uuid = desc["uuid"]
1105                 self.logger.debug(logging_text + "nsd={} created at RO. RO_id={}".format(nsd_ref, RO_nsd_uuid))
1106             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1107
1108             # Crate ns at RO
1109             # if present use it unless in error status
1110             RO_nsr_id = db_nsr["_admin"].get("deployed", {}).get("RO", {}).get("nsr_id")
1111             if RO_nsr_id:
1112                 try:
1113                     step = db_nsr_update["detailed-status"] = "Looking for existing ns at RO"
1114                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsr_id))
1115                     desc = await self.RO.show("ns", RO_nsr_id)
1116                 except ROclient.ROClientException as e:
1117                     if e.http_code != HTTPStatus.NOT_FOUND:
1118                         raise
1119                     RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1120                 if RO_nsr_id:
1121                     ns_status, ns_status_info = self.RO.check_ns_status(desc)
1122                     db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1123                     if ns_status == "ERROR":
1124                         step = db_nsr_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsr_id)
1125                         self.logger.debug(logging_text + step)
1126                         await self.RO.delete("ns", RO_nsr_id)
1127                         RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1128             if not RO_nsr_id:
1129                 step = db_nsr_update["detailed-status"] = "Checking dependencies"
1130                 # self.logger.debug(logging_text + step)
1131
1132                 # check if VIM is creating and wait  look if previous tasks in process
1133                 task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", ns_params["vimAccountId"])
1134                 if task_dependency:
1135                     step = "Waiting for related tasks to be completed: {}".format(task_name)
1136                     self.logger.debug(logging_text + step)
1137                     await asyncio.wait(task_dependency, timeout=3600)
1138                 if ns_params.get("vnf"):
1139                     for vnf in ns_params["vnf"]:
1140                         if "vimAccountId" in vnf:
1141                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
1142                                                                                         vnf["vimAccountId"])
1143                         if task_dependency:
1144                             step = "Waiting for related tasks to be completed: {}".format(task_name)
1145                             self.logger.debug(logging_text + step)
1146                             await asyncio.wait(task_dependency, timeout=3600)
1147
1148                 step = db_nsr_update["detailed-status"] = "Checking instantiation parameters"
1149
1150                 # feature 1429. Add n2vc public key to needed VMs
1151                 n2vc_key = await self.n2vc.GetPublicKey()
1152                 n2vc_key_list.append(n2vc_key)
1153                 RO_ns_params = self.ns_params_2_RO(ns_params, nsd, db_vnfds_ref, n2vc_key_list)
1154
1155                 step = db_nsr_update["detailed-status"] = "Deploying ns at VIM"
1156                 desc = await self.RO.create("ns", descriptor=RO_ns_params,
1157                                             name=db_nsr["name"],
1158                                             scenario=RO_nsd_uuid)
1159                 RO_nsr_id = db_nsr_update["_admin.deployed.RO.nsr_id"] = desc["uuid"]
1160                 db_nsr_update["_admin.nsState"] = "INSTANTIATED"
1161                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "BUILD"
1162                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
1163             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1164
1165             # wait until NS is ready
1166             step = ns_status_detailed = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
1167             detailed_status_old = None
1168             self.logger.debug(logging_text + step)
1169
1170             while time() <= start_deploy + self.total_deploy_timeout:
1171                 desc = await self.RO.show("ns", RO_nsr_id)
1172                 ns_status, ns_status_info = self.RO.check_ns_status(desc)
1173                 db_nsr_update["_admin.deployed.RO.nsr_status"] = ns_status
1174                 if ns_status == "ERROR":
1175                     raise ROclient.ROClientException(ns_status_info)
1176                 elif ns_status == "BUILD":
1177                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
1178                 elif ns_status == "ACTIVE":
1179                     step = detailed_status = "Waiting for management IP address reported by the VIM. Updating VNFRs"
1180                     try:
1181                         self.ns_update_vnfr(db_vnfrs, desc)
1182                         break
1183                     except LcmExceptionNoMgmtIP:
1184                         pass
1185                 else:
1186                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
1187                 if detailed_status != detailed_status_old:
1188                     detailed_status_old = db_nsr_update["detailed-status"] = detailed_status
1189                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1190                 await asyncio.sleep(5, loop=self.loop)
1191             else:   # total_deploy_timeout
1192                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
1193
1194             step = "Updating NSR"
1195             self.ns_update_nsr(db_nsr_update, db_nsr, desc)
1196
1197             db_nsr_update["operational-status"] = "running"
1198             db_nsr["detailed-status"] = "Configuring vnfr"
1199             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1200
1201             # Configure proxy charms once VMs are up
1202             for vca_index, vca_deployed in enumerate(vca_deployed_list):
1203                 vnf_index = vca_deployed.get("member-vnf-index")
1204                 vdu_id = vca_deployed.get("vdu_id")
1205                 vdu_name = None
1206                 vdu_count_index = None
1207
1208                 step = "executing proxy charm initial primitives for member_vnf_index={} vdu_id={}".format(vnf_index,
1209                                                                                                            vdu_id)
1210                 add_params = {}
1211                 initial_config_primitive_list = []
1212                 if vnf_index:
1213                     if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1214                         add_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1215                     vnfd = db_vnfds_index[vnf_index]
1216
1217                     if vdu_id:
1218                         for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1219                             if vdu["id"] == vdu_id:
1220                                 initial_config_primitive_list = vdu['vdu-configuration'].get(
1221                                     'initial-config-primitive', [])
1222                                 break
1223                         else:
1224                             raise LcmException("Not found vdu_id={} at vnfd:vdu".format(vdu_id))
1225                         vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1226                         # TODO for the moment only first vdu_id contains a charm deployed
1227                         if vdur["vdu-id-ref"] != vdu["id"]:
1228                             raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1229                                                .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1230                         add_params["rw_mgmt_ip"] = vdur["ip-address"]
1231                     else:
1232                         add_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1233                         initial_config_primitive_list = vnfd["vnf-configuration"].get('initial-config-primitive', [])
1234                 else:
1235                     if db_nsr.get("additionalParamsForNs"):
1236                         add_params = db_nsr["additionalParamsForNs"].copy()
1237                     for k, v in add_params.items():
1238                         if isinstance(v, str) and v.startswith("!!yaml "):
1239                             add_params[k] = yaml.safe_load(v[7:])
1240                     add_params["rw_mgmt_ip"] = None
1241                     initial_config_primitive_list = nsd["ns-configuration"].get('initial-config-primitive', [])
1242
1243                 # add primitive verify-ssh-credentials to the list after config only when is a vnf or vdu charm
1244                 initial_config_primitive_list = initial_config_primitive_list.copy()
1245                 if initial_config_primitive_list and vnf_index and vca_deployed.get("ssh-public-key"):
1246                     initial_config_primitive_list.insert(1, {"name": "verify-ssh-credentials", "parameter": []})
1247
1248                 for initial_config_primitive in initial_config_primitive_list:
1249                     primitive_params_ = self._map_primitive_params(initial_config_primitive, {}, add_params)
1250                     self.logger.debug(logging_text + step + " primitive '{}' params '{}'"
1251                                       .format(initial_config_primitive["name"], primitive_params_))
1252                     primitive_result, primitive_detail = await self._ns_execute_primitive(
1253                         db_nsr["_admin"]["deployed"], vnf_index, vdu_id, vdu_name, vdu_count_index,
1254                         initial_config_primitive["name"],
1255                         primitive_params_,
1256                         retries=10 if initial_config_primitive["name"] == "verify-ssh-credentials" else 0,
1257                         retries_interval=30)
1258                     if primitive_result != "COMPLETED":
1259                         raise LcmException("charm error executing primitive {} for member_vnf_index={} vdu_id={}: '{}'"
1260                                            .format(initial_config_primitive["name"], vca_deployed["member-vnf-index"],
1261                                                    vca_deployed["vdu_id"], primitive_detail))
1262
1263             # Deploy native charms
1264             step = "Looking for needed vnfd to configure with native charm"
1265             self.logger.debug(logging_text + step)
1266
1267             for c_vnf in get_iterable(nsd, "constituent-vnfd"):
1268                 vnfd_id = c_vnf["vnfd-id-ref"]
1269                 vnf_index = str(c_vnf["member-vnf-index"])
1270                 vnfd = db_vnfds_ref[vnfd_id]
1271
1272                 # Get additional parameters
1273                 vnfr_params = {}
1274                 if db_vnfrs[vnf_index].get("additionalParamsForVnf"):
1275                     vnfr_params = db_vnfrs[vnf_index]["additionalParamsForVnf"].copy()
1276                 for k, v in vnfr_params.items():
1277                     if isinstance(v, str) and v.startswith("!!yaml "):
1278                         vnfr_params[k] = yaml.safe_load(v[7:])
1279
1280                 # Check if this VNF has a charm configuration
1281                 vnf_config = vnfd.get("vnf-configuration")
1282                 if vnf_config and vnf_config.get("juju"):
1283                     native_charm = vnf_config["juju"].get("proxy") is False
1284
1285                     if native_charm:
1286                         if not vca_model_name:
1287                             step = "creating VCA model name '{}'".format(nsr_id)
1288                             self.logger.debug(logging_text + step)
1289                             await self.n2vc.CreateNetworkService(nsr_id)
1290                             vca_model_name = nsr_id
1291                             db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1292                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1293                         step = "deploying native charm for vnf_member_index={}".format(vnf_index)
1294                         vnfr_params["rw_mgmt_ip"] = db_vnfrs[vnf_index]["ip-address"]
1295                         charm_params = {
1296                             "user_values": vnfr_params,
1297                             "rw_mgmt_ip": db_vnfrs[vnf_index]["ip-address"],
1298                             "initial-config-primitive": vnf_config.get('initial-config-primitive') or {},
1299                         }
1300
1301                         # get username
1302                         # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1303                         #  merged. Meanwhile let's get username from initial-config-primitive
1304                         if vnf_config.get("initial-config-primitive"):
1305                             for param in vnf_config["initial-config-primitive"][0].get("parameter", ()):
1306                                 if param["name"] == "ssh-username":
1307                                     charm_params["username"] = param["value"]
1308                         if vnf_config.get("config-access") and vnf_config["config-access"].get("ssh-access"):
1309                             if vnf_config["config-access"]["ssh-access"].get("required"):
1310                                 charm_params["username"] = vnf_config["config-access"]["ssh-access"].get("default-user")
1311
1312                         # Login to the VCA. If there are multiple calls to login(),
1313                         # subsequent calls will be a nop and return immediately.
1314                         await self.n2vc.login()
1315
1316                         deploy_charm(vnf_index, None, None, None, charm_params, n2vc_info, native_charm)
1317                         number_to_configure += 1
1318
1319                 # Deploy charms for each VDU that supports one.
1320                 for vdu_index, vdu in enumerate(get_iterable(vnfd, 'vdu')):
1321                     vdu_config = vdu.get('vdu-configuration')
1322                     native_charm = False
1323
1324                     if vdu_config and vdu_config.get("juju"):
1325                         native_charm = vdu_config["juju"].get("proxy") is False
1326
1327                         if native_charm:
1328                             if not vca_model_name:
1329                                 step = "creating VCA model name"
1330                                 await self.n2vc.CreateNetworkService(nsr_id)
1331                                 vca_model_name = nsr_id
1332                                 db_nsr_update["_admin.deployed.VCA-model-name"] = nsr_id
1333                                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1334                             step = "deploying native charm for vnf_member_index={} vdu_id={}".format(vnf_index,
1335                                                                                                      vdu["id"])
1336                             await self.n2vc.login()
1337                             vdur = db_vnfrs[vnf_index]["vdur"][vdu_index]
1338                             # TODO for the moment only first vdu_id contains a charm deployed
1339                             if vdur["vdu-id-ref"] != vdu["id"]:
1340                                 raise LcmException("Mismatch vdur {}, vdu {} at index {} for vnf {}"
1341                                                    .format(vdur["vdu-id-ref"], vdu["id"], vdu_index, vnf_index))
1342                             vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
1343                             charm_params = {
1344                                 "user_values": vnfr_params,
1345                                 "rw_mgmt_ip": vdur["ip-address"],
1346                                 "initial-config-primitive": vdu_config.get('initial-config-primitive') or {}
1347                             }
1348
1349                             # get username
1350                             # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1351                             #  merged. Meanwhile let's get username from initial-config-primitive
1352                             if vdu_config.get("initial-config-primitive"):
1353                                 for param in vdu_config["initial-config-primitive"][0].get("parameter", ()):
1354                                     if param["name"] == "ssh-username":
1355                                         charm_params["username"] = param["value"]
1356                             if vdu_config.get("config-access") and vdu_config["config-access"].get("ssh-access"):
1357                                 if vdu_config["config-access"]["ssh-access"].get("required"):
1358                                     charm_params["username"] = vdu_config["config-access"]["ssh-access"].get(
1359                                         "default-user")
1360
1361                             deploy_charm(vnf_index, vdu["id"], vdur.get("name"), vdur["count-index"],
1362                                          charm_params, n2vc_info, native_charm)
1363                             number_to_configure += 1
1364
1365             # Check if this NS has a charm configuration
1366
1367             ns_config = nsd.get("ns-configuration")
1368             if ns_config and ns_config.get("juju"):
1369                 native_charm = ns_config["juju"].get("proxy") is False
1370
1371                 if native_charm:
1372                     step = "deploying native charm to configure ns"
1373                     # TODO is NS magmt IP address needed?
1374
1375                     # Get additional parameters
1376                     additional_params = {}
1377                     if db_nsr.get("additionalParamsForNs"):
1378                         additional_params = db_nsr["additionalParamsForNs"].copy()
1379                     for k, v in additional_params.items():
1380                         if isinstance(v, str) and v.startswith("!!yaml "):
1381                             additional_params[k] = yaml.safe_load(v[7:])
1382
1383                     # additional_params["rw_mgmt_ip"] = db_nsr["ip-address"]
1384                     charm_params = {
1385                         "user_values": additional_params,
1386                         "rw_mgmt_ip": db_nsr.get("ip-address"),
1387                         "initial-config-primitive": ns_config.get('initial-config-primitive') or {}
1388                     }
1389
1390                     # get username
1391                     # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1392                     #  merged. Meanwhile let's get username from initial-config-primitive
1393                     if ns_config.get("initial-config-primitive"):
1394                         for param in ns_config["initial-config-primitive"][0].get("parameter", ()):
1395                             if param["name"] == "ssh-username":
1396                                 charm_params["username"] = param["value"]
1397                     if ns_config.get("config-access") and ns_config["config-access"].get("ssh-access"):
1398                         if ns_config["config-access"]["ssh-access"].get("required"):
1399                             charm_params["username"] = ns_config["config-access"]["ssh-access"].get("default-user")
1400
1401                     # Login to the VCA. If there are multiple calls to login(),
1402                     # subsequent calls will be a nop and return immediately.
1403                     await self.n2vc.login()
1404                     deploy_charm(None, None, None, None, charm_params, n2vc_info, native_charm)
1405                     number_to_configure += 1
1406
1407             # waiting all charms are ok
1408             configuration_failed = False
1409             if number_to_configure:
1410                 step = "Waiting all charms are active"
1411                 old_status = "configuring: init: {}".format(number_to_configure)
1412                 db_nsr_update["config-status"] = old_status
1413                 db_nsr_update["detailed-status"] = old_status
1414                 db_nslcmop_update["detailed-status"] = old_status
1415
1416                 # wait until all are configured.
1417                 while time() <= start_deploy + self.total_deploy_timeout:
1418                     if db_nsr_update:
1419                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1420                     if db_nslcmop_update:
1421                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1422                     # TODO add a fake task that set n2vc_event after some time
1423                     await n2vc_info["n2vc_event"].wait()
1424                     n2vc_info["n2vc_event"].clear()
1425                     all_active = True
1426                     status_map = {}
1427                     n2vc_error_text = []  # contain text error list. If empty no one is in error status
1428                     now = time()
1429                     for vca_deployed in vca_deployed_list:
1430                         vca_status = vca_deployed["operational-status"]
1431                         if vca_status not in status_map:
1432                             # Initialize it
1433                             status_map[vca_status] = 0
1434                         status_map[vca_status] += 1
1435
1436                         if vca_status == "active":
1437                             vca_deployed.pop("time_first_error", None)
1438                             vca_deployed.pop("status_first_error", None)
1439                             continue
1440
1441                         all_active = False
1442                         if vca_status in ("error", "blocked"):
1443                             vca_deployed["detailed-status-error"] = vca_deployed["detailed-status"]
1444                             # if not first time in this status error
1445                             if not vca_deployed.get("time_first_error"):
1446                                 vca_deployed["time_first_error"] = now
1447                                 continue
1448                         if vca_deployed.get("time_first_error") and \
1449                                 now <= vca_deployed["time_first_error"] + self.timeout_vca_on_error:
1450                             n2vc_error_text.append("member_vnf_index={} vdu_id={} {}: {}"
1451                                                    .format(vca_deployed["member-vnf-index"],
1452                                                            vca_deployed["vdu_id"], vca_status,
1453                                                            vca_deployed["detailed-status-error"]))
1454
1455                     if all_active:
1456                         break
1457                     elif n2vc_error_text:
1458                         db_nsr_update["config-status"] = "failed"
1459                         error_text = "fail configuring " + ";".join(n2vc_error_text)
1460                         db_nsr_update["detailed-status"] = error_text
1461                         db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED_TEMP"
1462                         db_nslcmop_update["detailed-status"] = error_text
1463                         db_nslcmop_update["statusEnteredTime"] = time()
1464                         configuration_failed = True
1465                         break
1466                     else:
1467                         cs = "configuring: "
1468                         separator = ""
1469                         for status, num in status_map.items():
1470                             cs += separator + "{}: {}".format(status, num)
1471                             separator = ", "
1472                         if old_status != cs:
1473                             db_nsr_update["config-status"] = cs
1474                             db_nsr_update["detailed-status"] = cs
1475                             db_nslcmop_update["detailed-status"] = cs
1476                             old_status = cs
1477                 else:   # total_deploy_timeout
1478                     raise LcmException("Timeout waiting ns to be configured")
1479
1480             if not configuration_failed:
1481                 # all is done
1482                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1483                 db_nslcmop_update["statusEnteredTime"] = time()
1484                 db_nslcmop_update["detailed-status"] = "done"
1485                 db_nsr_update["config-status"] = "configured"
1486                 db_nsr_update["detailed-status"] = "done"
1487
1488             return
1489
1490         except (ROclient.ROClientException, DbException, LcmException) as e:
1491             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
1492             exc = e
1493         except asyncio.CancelledError:
1494             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1495             exc = "Operation was cancelled"
1496         except Exception as e:
1497             exc = traceback.format_exc()
1498             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
1499                                  exc_info=True)
1500         finally:
1501             if exc:
1502                 if db_nsr:
1503                     db_nsr_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
1504                     db_nsr_update["operational-status"] = "failed"
1505                 if db_nslcmop:
1506                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1507                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1508                     db_nslcmop_update["statusEnteredTime"] = time()
1509             try:
1510                 if db_nsr:
1511                     db_nsr_update["_admin.nslcmop"] = None
1512                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1513                 if db_nslcmop_update:
1514                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1515             except DbException as e:
1516                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1517             if nslcmop_operation_state:
1518                 try:
1519                     await self.msg.aiowrite("ns", "instantiated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1520                                                                    "operationState": nslcmop_operation_state},
1521                                             loop=self.loop)
1522                 except Exception as e:
1523                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1524
1525             self.logger.debug(logging_text + "Exit")
1526             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
1527
1528     async def _destroy_charm(self, model, application):
1529         """
1530         Order N2VC destroy a charm
1531         :param model:
1532         :param application:
1533         :return: True if charm does not exist. False if it exist
1534         """
1535         if not await self.n2vc.HasApplication(model, application):
1536             return True  # Already removed
1537         await self.n2vc.RemoveCharms(model, application)
1538         return False
1539
1540     async def _wait_charm_destroyed(self, model, application, timeout):
1541         """
1542         Wait until charm does not exist
1543         :param model:
1544         :param application:
1545         :param timeout:
1546         :return: True if not exist, False if timeout
1547         """
1548         while True:
1549             if not await self.n2vc.HasApplication(model, application):
1550                 return True
1551             if timeout < 0:
1552                 return False
1553             await asyncio.sleep(10)
1554             timeout -= 10
1555
1556     # Check if this VNFD has a configured terminate action
1557     def _has_terminate_config_primitive(self, vnfd):
1558         vnf_config = vnfd.get("vnf-configuration")
1559         if vnf_config and vnf_config.get("terminate-config-primitive"):
1560             return True
1561         else:
1562             return False
1563
1564     @staticmethod
1565     def _get_terminate_config_primitive_seq_list(vnfd):
1566         """ Get a numerically sorted list of the sequences for this VNFD's terminate action """
1567         # No need to check for existing primitive twice, already done before
1568         vnf_config = vnfd.get("vnf-configuration")
1569         seq_list = vnf_config.get("terminate-config-primitive")
1570         # Get all 'seq' tags in seq_list, order sequences numerically, ascending.
1571         seq_list_sorted = sorted(seq_list, key=lambda x: int(x['seq']))
1572         return seq_list_sorted
1573
1574     @staticmethod
1575     def _create_nslcmop(nsr_id, operation, params):
1576         """
1577         Creates a ns-lcm-opp content to be stored at database.
1578         :param nsr_id: internal id of the instance
1579         :param operation: instantiate, terminate, scale, action, ...
1580         :param params: user parameters for the operation
1581         :return: dictionary following SOL005 format
1582         """
1583         # Raise exception if invalid arguments
1584         if not (nsr_id and operation and params):
1585             raise LcmException(
1586                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided")
1587         now = time()
1588         _id = str(uuid4())
1589         nslcmop = {
1590             "id": _id,
1591             "_id": _id,
1592             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
1593             "operationState": "PROCESSING",
1594             "statusEnteredTime": now,
1595             "nsInstanceId": nsr_id,
1596             "lcmOperationType": operation,
1597             "startTime": now,
1598             "isAutomaticInvocation": False,
1599             "operationParams": params,
1600             "isCancelPending": False,
1601             "links": {
1602                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
1603                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
1604             }
1605         }
1606         return nslcmop
1607
1608     def _get_terminate_primitive_params(self, seq, vnf_index):
1609         primitive = seq.get('name')
1610         primitive_params = {}
1611         params = {
1612             "member_vnf_index": vnf_index,
1613             "primitive": primitive,
1614             "primitive_params": primitive_params,
1615         }
1616         desc_params = {}
1617         return self._map_primitive_params(seq, params, desc_params)
1618
1619     def _add_suboperation(self, db_nslcmop, nslcmop_id, vnf_index, vdu_id, vdu_count_index,
1620                           vdu_name, primitive, mapped_primitive_params):
1621         # Create the "_admin.operations" array, or append operation if array already exists
1622         key_admin = '_admin'
1623         key_operations = 'operations'
1624         db_nslcmop_update = {}
1625         db_nslcmop_admin = db_nslcmop.get(key_admin, {})
1626         op_list = db_nslcmop_admin.get(key_operations)
1627         new_op = {'member_vnf_index': vnf_index,
1628                   'vdu_id': vdu_id,
1629                   'vdu_count_index': vdu_count_index,
1630                   'primitive': primitive,
1631                   'primitive_params': mapped_primitive_params}
1632         if db_nslcmop_admin:
1633             if not op_list:
1634                 # First operation, create key 'operations' with current operation as first list element
1635                 db_nslcmop_admin.update({key_operations: [new_op]})
1636                 op_list = db_nslcmop_admin.get(key_operations)
1637             else:
1638                 # Not first operation, append operation to existing list
1639                 op_list.append(new_op)
1640
1641         db_nslcmop_update['_admin.operations'] = op_list
1642         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1643
1644     async def _terminate_action(self, db_nslcmop, nslcmop_id, nsr_id):
1645         """ Create a primitive with params from VNFD
1646             Called from terminate() before deleting instance
1647             Calls action() to execute the primitive """
1648         logging_text = "Task ns={} _terminate_action={} ".format(nsr_id, nslcmop_id)
1649         db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
1650         db_vnfds = {}
1651         # Loop over VNFRs
1652         for vnfr in db_vnfrs_list:
1653             vnfd_id = vnfr["vnfd-id"]
1654             vnf_index = vnfr["member-vnf-index-ref"]
1655             if vnfd_id not in db_vnfds:
1656                 step = "Getting vnfd={} id='{}' from db".format(vnfd_id, vnfd_id)
1657                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
1658                 db_vnfds[vnfd_id] = vnfd
1659             vnfd = db_vnfds[vnfd_id]
1660             if not self._has_terminate_config_primitive(vnfd):
1661                 continue
1662             # Get the primitive's sorted sequence list
1663             seq_list = self._get_terminate_config_primitive_seq_list(vnfd)
1664             for seq in seq_list:
1665                 # For each sequence in list, get primitive and call _ns_execute_primitive()
1666                 step = "Calling terminate action for vnf_member_index={} primitive={}".format(
1667                     vnf_index, seq.get("name"))
1668                 self.logger.debug(logging_text + step)
1669                 # Create the primitive for each sequence, i.e. "primitive": "touch"
1670                 primitive = seq.get('name')
1671                 mapped_primitive_params = self._get_terminate_primitive_params(seq, vnf_index)
1672                 # The following 3 parameters are currently set to None for 'terminate':
1673                 # vdu_id, vdu_count_index, vdu_name
1674                 vdu_id = db_nslcmop["operationParams"].get("vdu_id")
1675                 vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
1676                 vdu_name = db_nslcmop["operationParams"].get("vdu_name")
1677                 # Add suboperation
1678                 self._add_suboperation(db_nslcmop,
1679                                        nslcmop_id,
1680                                        vnf_index,
1681                                        vdu_id,
1682                                        vdu_count_index,
1683                                        vdu_name,
1684                                        primitive,
1685                                        mapped_primitive_params)
1686                 # Suboperations: Call _ns_execute_primitive() instead of action()
1687                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1688                 nsr_deployed = db_nsr["_admin"]["deployed"]
1689                 result, result_detail = await self._ns_execute_primitive(
1690                     nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
1691                     mapped_primitive_params)
1692
1693                 # nslcmop_operation_state, nslcmop_operation_state_detail = await self.action(
1694                 #    nsr_id, nslcmop_terminate_action_id)
1695                 # Launch Exception if action() returns other than ['COMPLETED', 'PARTIALLY_COMPLETED']
1696                 result_ok = ['COMPLETED', 'PARTIALLY_COMPLETED']
1697                 if result not in result_ok:
1698                     raise LcmException(
1699                         "terminate_primitive_action for vnf_member_index={}",
1700                         " primitive={} fails with error {}".format(
1701                             vnf_index, seq.get("name"), result_detail))
1702
1703     async def terminate(self, nsr_id, nslcmop_id):
1704
1705         # Try to lock HA task here
1706         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
1707         if not task_is_locked_by_me:
1708             return
1709
1710         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
1711         self.logger.debug(logging_text + "Enter")
1712         db_nsr = None
1713         db_nslcmop = None
1714         exc = None
1715         failed_detail = []   # annotates all failed error messages
1716         vca_time_destroy = None   # time of where destroy charm order
1717         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
1718         db_nslcmop_update = {}
1719         nslcmop_operation_state = None
1720         autoremove = False  # autoremove after terminated
1721         try:
1722             # wait for any previous tasks in process
1723             await self.lcm_tasks.waitfor_related_HA("ns", 'nslcmops', nslcmop_id)
1724
1725             step = "Getting nslcmop={} from db".format(nslcmop_id)
1726             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
1727             step = "Getting nsr={} from db".format(nsr_id)
1728             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1729             # nsd = db_nsr["nsd"]
1730             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed"))
1731             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
1732                 return
1733             # #TODO check if VIM is creating and wait
1734             # RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
1735             # Call internal terminate action
1736             await self._terminate_action(db_nslcmop, nslcmop_id, nsr_id)
1737
1738             db_nsr_update["operational-status"] = "terminating"
1739             db_nsr_update["config-status"] = "terminating"
1740
1741             if nsr_deployed and nsr_deployed.get("VCA-model-name"):
1742                 vca_model_name = nsr_deployed["VCA-model-name"]
1743                 step = "deleting VCA model name '{}' and all charms".format(vca_model_name)
1744                 self.logger.debug(logging_text + step)
1745                 try:
1746                     await self.n2vc.DestroyNetworkService(vca_model_name)
1747                 except NetworkServiceDoesNotExist:
1748                     pass
1749                 db_nsr_update["_admin.deployed.VCA-model-name"] = None
1750                 if nsr_deployed.get("VCA"):
1751                     for vca_index in range(0, len(nsr_deployed["VCA"])):
1752                         db_nsr_update["_admin.deployed.VCA.{}".format(vca_index)] = None
1753                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1754             # for backward compatibility if charm have been created with "default"  model name delete one by one
1755             elif nsr_deployed and nsr_deployed.get("VCA"):
1756                 try:
1757                     step = "Scheduling configuration charms removing"
1758                     db_nsr_update["detailed-status"] = "Deleting charms"
1759                     self.logger.debug(logging_text + step)
1760                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1761                     # for backward compatibility
1762                     if isinstance(nsr_deployed["VCA"], dict):
1763                         nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
1764                         db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
1765                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1766
1767                     for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1768                         if vca_deployed:
1769                             if await self._destroy_charm(vca_deployed['model'], vca_deployed["application"]):
1770                                 vca_deployed.clear()
1771                                 db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1772                             else:
1773                                 vca_time_destroy = time()
1774                 except Exception as e:
1775                     self.logger.debug(logging_text + "Failed while deleting charms: {}".format(e))
1776
1777             # remove from RO
1778             RO_fail = False
1779
1780             # Delete ns
1781             RO_nsr_id = RO_delete_action = None
1782             if nsr_deployed and nsr_deployed.get("RO"):
1783                 RO_nsr_id = nsr_deployed["RO"].get("nsr_id")
1784                 RO_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
1785             try:
1786                 if RO_nsr_id:
1787                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = \
1788                         "Deleting ns from VIM"
1789                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1790                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1791                     self.logger.debug(logging_text + step)
1792                     desc = await self.RO.delete("ns", RO_nsr_id)
1793                     RO_delete_action = desc["action_id"]
1794                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
1795                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1796                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1797                 if RO_delete_action:
1798                     # wait until NS is deleted from VIM
1799                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={} RO_delete_action={}".\
1800                         format(RO_nsr_id, RO_delete_action)
1801                     detailed_status_old = None
1802                     self.logger.debug(logging_text + step)
1803
1804                     delete_timeout = 20 * 60   # 20 minutes
1805                     while delete_timeout > 0:
1806                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
1807                                                   extra_item_id=RO_delete_action)
1808                         ns_status, ns_status_info = self.RO.check_action_status(desc)
1809                         if ns_status == "ERROR":
1810                             raise ROclient.ROClientException(ns_status_info)
1811                         elif ns_status == "BUILD":
1812                             detailed_status = step + "; {}".format(ns_status_info)
1813                         elif ns_status == "ACTIVE":
1814                             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1815                             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1816                             break
1817                         else:
1818                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
1819                         if detailed_status != detailed_status_old:
1820                             detailed_status_old = db_nslcmop_update["detailed-status"] = \
1821                                 db_nsr_update["detailed-status"] = detailed_status
1822                             self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1823                             self.update_db_2("nsrs", nsr_id, db_nsr_update)
1824                         await asyncio.sleep(5, loop=self.loop)
1825                         delete_timeout -= 5
1826                     else:  # delete_timeout <= 0:
1827                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
1828
1829             except ROclient.ROClientException as e:
1830                 if e.http_code == 404:  # not found
1831                     db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1832                     db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1833                     db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1834                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsr_id))
1835                 elif e.http_code == 409:   # conflict
1836                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsr_id, e))
1837                     self.logger.debug(logging_text + failed_detail[-1])
1838                     RO_fail = True
1839                 else:
1840                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsr_id, e))
1841                     self.logger.error(logging_text + failed_detail[-1])
1842                     RO_fail = True
1843
1844             # Delete nsd
1845             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("nsd_id"):
1846                 RO_nsd_id = nsr_deployed["RO"]["nsd_id"]
1847                 try:
1848                     step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1849                         "Deleting nsd from RO"
1850                     await self.RO.delete("nsd", RO_nsd_id)
1851                     self.logger.debug(logging_text + "RO_nsd_id={} deleted".format(RO_nsd_id))
1852                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1853                 except ROclient.ROClientException as e:
1854                     if e.http_code == 404:  # not found
1855                         db_nsr_update["_admin.deployed.RO.nsd_id"] = None
1856                         self.logger.debug(logging_text + "RO_nsd_id={} already deleted".format(RO_nsd_id))
1857                     elif e.http_code == 409:   # conflict
1858                         failed_detail.append("RO_nsd_id={} delete conflict: {}".format(RO_nsd_id, e))
1859                         self.logger.debug(logging_text + failed_detail[-1])
1860                         RO_fail = True
1861                     else:
1862                         failed_detail.append("RO_nsd_id={} delete error: {}".format(RO_nsd_id, e))
1863                         self.logger.error(logging_text + failed_detail[-1])
1864                         RO_fail = True
1865
1866             if not RO_fail and nsr_deployed and nsr_deployed.get("RO") and nsr_deployed["RO"].get("vnfd"):
1867                 for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
1868                     if not vnf_deployed or not vnf_deployed["id"]:
1869                         continue
1870                     try:
1871                         RO_vnfd_id = vnf_deployed["id"]
1872                         step = db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] =\
1873                             "Deleting member_vnf_index={} RO_vnfd_id={} from RO".format(
1874                                 vnf_deployed["member-vnf-index"], RO_vnfd_id)
1875                         await self.RO.delete("vnfd", RO_vnfd_id)
1876                         self.logger.debug(logging_text + "RO_vnfd_id={} deleted".format(RO_vnfd_id))
1877                         db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1878                     except ROclient.ROClientException as e:
1879                         if e.http_code == 404:  # not found
1880                             db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
1881                             self.logger.debug(logging_text + "RO_vnfd_id={} already deleted ".format(RO_vnfd_id))
1882                         elif e.http_code == 409:   # conflict
1883                             failed_detail.append("RO_vnfd_id={} delete conflict: {}".format(RO_vnfd_id, e))
1884                             self.logger.debug(logging_text + failed_detail[-1])
1885                         else:
1886                             failed_detail.append("RO_vnfd_id={} delete error: {}".format(RO_vnfd_id, e))
1887                             self.logger.error(logging_text + failed_detail[-1])
1888
1889             # wait until charm deleted
1890             if vca_time_destroy:
1891                 db_nsr_update["detailed-status"] = db_nslcmop_update["detailed-status"] = step = \
1892                     "Waiting for deletion of configuration charms"
1893                 self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1894                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1895                 for vca_index, vca_deployed in enumerate(nsr_deployed["VCA"]):
1896                     if not vca_deployed:
1897                         continue
1898                     step = "Waiting for deletion of charm application_name={}".format(vca_deployed["application"])
1899                     timeout = self.timeout_charm_delete - int(time() - vca_time_destroy)
1900                     if not await self._wait_charm_destroyed(vca_deployed['model'], vca_deployed["application"],
1901                                                             timeout):
1902                         failed_detail.append("VCA[application_name={}] Deletion timeout".format(
1903                             vca_deployed["application"]))
1904                     else:
1905                         db_nsr["_admin.deployed.VCA.{}".format(vca_index)] = None
1906
1907             if failed_detail:
1908                 self.logger.error(logging_text + " ;".join(failed_detail))
1909                 db_nsr_update["operational-status"] = "failed"
1910                 db_nsr_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
1911                 db_nslcmop_update["detailed-status"] = "; ".join(failed_detail)
1912                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1913                 db_nslcmop_update["statusEnteredTime"] = time()
1914             else:
1915                 db_nsr_update["operational-status"] = "terminated"
1916                 db_nsr_update["detailed-status"] = "Done"
1917                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
1918                 db_nslcmop_update["detailed-status"] = "Done"
1919                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
1920                 db_nslcmop_update["statusEnteredTime"] = time()
1921                 if db_nslcmop["operationParams"].get("autoremove"):
1922                     autoremove = True
1923
1924         except (ROclient.ROClientException, DbException, LcmException) as e:
1925             self.logger.error(logging_text + "Exit Exception {}".format(e))
1926             exc = e
1927         except asyncio.CancelledError:
1928             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
1929             exc = "Operation was cancelled"
1930         except Exception as e:
1931             exc = traceback.format_exc()
1932             self.logger.critical(logging_text + "Exit Exception {}".format(e), exc_info=True)
1933         finally:
1934             if exc and db_nslcmop:
1935                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
1936                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
1937                 db_nslcmop_update["statusEnteredTime"] = time()
1938             try:
1939                 if db_nslcmop and db_nslcmop_update:
1940                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
1941                 if db_nsr:
1942                     db_nsr_update["_admin.nslcmop"] = None
1943                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
1944             except DbException as e:
1945                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
1946             if nslcmop_operation_state:
1947                 try:
1948                     await self.msg.aiowrite("ns", "terminated", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
1949                                                                  "operationState": nslcmop_operation_state,
1950                                                                  "autoremove": autoremove},
1951                                             loop=self.loop)
1952                 except Exception as e:
1953                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
1954             self.logger.debug(logging_text + "Exit")
1955             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
1956
1957     @staticmethod
1958     def _map_primitive_params(primitive_desc, params, instantiation_params):
1959         """
1960         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
1961         The default-value is used. If it is between < > it look for a value at instantiation_params
1962         :param primitive_desc: portion of VNFD/NSD that describes primitive
1963         :param params: Params provided by user
1964         :param instantiation_params: Instantiation params provided by user
1965         :return: a dictionary with the calculated params
1966         """
1967         calculated_params = {}
1968         for parameter in primitive_desc.get("parameter", ()):
1969             param_name = parameter["name"]
1970             if param_name in params:
1971                 calculated_params[param_name] = params[param_name]
1972             elif "default-value" in parameter or "value" in parameter:
1973                 if "value" in parameter:
1974                     calculated_params[param_name] = parameter["value"]
1975                 else:
1976                     calculated_params[param_name] = parameter["default-value"]
1977                 if isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("<") \
1978                         and calculated_params[param_name].endswith(">"):
1979                     if calculated_params[param_name][1:-1] in instantiation_params:
1980                         calculated_params[param_name] = instantiation_params[calculated_params[param_name][1:-1]]
1981                     else:
1982                         raise LcmException("Parameter {} needed to execute primitive {} not provided".
1983                                            format(parameter["default-value"], primitive_desc["name"]))
1984             else:
1985                 raise LcmException("Parameter {} needed to execute primitive {} not provided".
1986                                    format(param_name, primitive_desc["name"]))
1987
1988             if isinstance(calculated_params[param_name], (dict, list, tuple)):
1989                 calculated_params[param_name] = yaml.safe_dump(calculated_params[param_name], default_flow_style=True,
1990                                                                width=256)
1991             elif isinstance(calculated_params[param_name], str) and calculated_params[param_name].startswith("!!yaml "):
1992                 calculated_params[param_name] = calculated_params[param_name][7:]
1993         return calculated_params
1994
1995     async def _ns_execute_primitive(self, db_deployed, member_vnf_index, vdu_id, vdu_name, vdu_count_index,
1996                                     primitive, primitive_params, retries=0, retries_interval=30):
1997         start_primitive_time = time()
1998         try:
1999             for vca_deployed in db_deployed["VCA"]:
2000                 if not vca_deployed:
2001                     continue
2002                 if member_vnf_index != vca_deployed["member-vnf-index"] or vdu_id != vca_deployed["vdu_id"]:
2003                     continue
2004                 if vdu_name and vdu_name != vca_deployed["vdu_name"]:
2005                     continue
2006                 if vdu_count_index and vdu_count_index != vca_deployed["vdu_count_index"]:
2007                     continue
2008                 break
2009             else:
2010                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} is not "
2011                                    "deployed".format(member_vnf_index, vdu_id, vdu_name, vdu_count_index))
2012             model_name = vca_deployed.get("model")
2013             application_name = vca_deployed.get("application")
2014             if not model_name or not application_name:
2015                 raise LcmException("charm for member_vnf_index={} vdu_id={} vdu_name={} vdu_count_index={} has not "
2016                                    "model or application name" .format(member_vnf_index, vdu_id, vdu_name,
2017                                                                        vdu_count_index))
2018             # if vca_deployed["operational-status"] != "active":
2019             #   raise LcmException("charm for member_vnf_index={} vdu_id={} operational_status={} not 'active'".format(
2020             #   member_vnf_index, vdu_id, vca_deployed["operational-status"]))
2021             callback = None  # self.n2vc_callback
2022             callback_args = ()  # [db_nsr, db_nslcmop, member_vnf_index, None]
2023             await self.n2vc.login()
2024             if primitive == "config":
2025                 primitive_params = {"params": primitive_params}
2026             while retries >= 0:
2027                 primitive_id = await self.n2vc.ExecutePrimitive(
2028                     model_name,
2029                     application_name,
2030                     primitive,
2031                     callback,
2032                     *callback_args,
2033                     **primitive_params
2034                 )
2035                 while time() - start_primitive_time < self.timeout_primitive:
2036                     primitive_result_ = await self.n2vc.GetPrimitiveStatus(model_name, primitive_id)
2037                     if primitive_result_ in ("completed", "failed"):
2038                         primitive_result = "COMPLETED" if primitive_result_ == "completed" else "FAILED"
2039                         detailed_result = await self.n2vc.GetPrimitiveOutput(model_name, primitive_id)
2040                         break
2041                     elif primitive_result_ is None and primitive == "config":
2042                         primitive_result = "COMPLETED"
2043                         detailed_result = None
2044                         break
2045                     else:  # ("running", "pending", None):
2046                         pass
2047                     await asyncio.sleep(5)
2048                 else:
2049                     raise LcmException("timeout after {} seconds".format(self.timeout_primitive))
2050                 if primitive_result == "COMPLETED":
2051                     break
2052                 retries -= 1
2053                 if retries >= 0:
2054                     await asyncio.sleep(retries_interval)
2055
2056             return primitive_result, detailed_result
2057         except (N2VCPrimitiveExecutionFailed, LcmException) as e:
2058             return "FAILED", str(e)
2059
2060     async def action(self, nsr_id, nslcmop_id):
2061
2062         # Try to lock HA task here
2063         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2064         if not task_is_locked_by_me:
2065             return
2066
2067         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
2068         self.logger.debug(logging_text + "Enter")
2069         # get all needed from database
2070         db_nsr = None
2071         db_nslcmop = None
2072         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2073         db_nslcmop_update = {}
2074         nslcmop_operation_state = None
2075         nslcmop_operation_state_detail = None
2076         exc = None
2077         try:
2078             # wait for any previous tasks in process
2079             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2080
2081             step = "Getting information from database"
2082             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2083             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2084
2085             nsr_deployed = db_nsr["_admin"].get("deployed")
2086             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
2087             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
2088             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
2089             vdu_name = db_nslcmop["operationParams"].get("vdu_name")
2090
2091             if vnf_index:
2092                 step = "Getting vnfr from database"
2093                 db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2094                 step = "Getting vnfd from database"
2095                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2096             else:
2097                 if db_nsr.get("nsd"):
2098                     db_nsd = db_nsr.get("nsd")    # TODO this will be removed
2099                 else:
2100                     step = "Getting nsd from database"
2101                     db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2102
2103             # for backward compatibility
2104             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2105                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2106                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2107                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2108
2109             primitive = db_nslcmop["operationParams"]["primitive"]
2110             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
2111
2112             # look for primitive
2113             config_primitive_desc = None
2114             if vdu_id:
2115                 for vdu in get_iterable(db_vnfd, "vdu"):
2116                     if vdu_id == vdu["id"]:
2117                         for config_primitive in vdu.get("vdu-configuration", {}).get("config-primitive", ()):
2118                             if config_primitive["name"] == primitive:
2119                                 config_primitive_desc = config_primitive
2120                                 break
2121             elif vnf_index:
2122                 for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2123                     if config_primitive["name"] == primitive:
2124                         config_primitive_desc = config_primitive
2125                         break
2126             else:
2127                 for config_primitive in db_nsd.get("ns-configuration", {}).get("config-primitive", ()):
2128                     if config_primitive["name"] == primitive:
2129                         config_primitive_desc = config_primitive
2130                         break
2131
2132             if not config_primitive_desc:
2133                 raise LcmException("Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".
2134                                    format(primitive))
2135
2136             desc_params = {}
2137             if vnf_index:
2138                 if db_vnfr.get("additionalParamsForVnf"):
2139                     desc_params.update(db_vnfr["additionalParamsForVnf"])
2140             else:
2141                 if db_nsr.get("additionalParamsForVnf"):
2142                     desc_params.update(db_nsr["additionalParamsForNs"])
2143
2144             # TODO check if ns is in a proper status
2145             result, result_detail = await self._ns_execute_primitive(
2146                 nsr_deployed, vnf_index, vdu_id, vdu_name, vdu_count_index, primitive,
2147                 self._map_primitive_params(config_primitive_desc, primitive_params, desc_params))
2148             db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = result_detail
2149             db_nslcmop_update["operationState"] = nslcmop_operation_state = result
2150             db_nslcmop_update["statusEnteredTime"] = time()
2151             self.logger.debug(logging_text + " task Done with result {} {}".format(result, result_detail))
2152             return  # database update is called inside finally
2153
2154         except (DbException, LcmException) as e:
2155             self.logger.error(logging_text + "Exit Exception {}".format(e))
2156             exc = e
2157         except asyncio.CancelledError:
2158             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2159             exc = "Operation was cancelled"
2160         except Exception as e:
2161             exc = traceback.format_exc()
2162             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2163         finally:
2164             if exc and db_nslcmop:
2165                 db_nslcmop_update["detailed-status"] = nslcmop_operation_state_detail = \
2166                     "FAILED {}: {}".format(step, exc)
2167                 db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2168                 db_nslcmop_update["statusEnteredTime"] = time()
2169             try:
2170                 if db_nslcmop_update:
2171                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2172                 if db_nsr:
2173                     db_nsr_update["_admin.nslcmop"] = None
2174                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2175             except DbException as e:
2176                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2177             self.logger.debug(logging_text + "Exit")
2178             if nslcmop_operation_state:
2179                 try:
2180                     await self.msg.aiowrite("ns", "actioned", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2181                                                                "operationState": nslcmop_operation_state},
2182                                             loop=self.loop)
2183                 except Exception as e:
2184                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2185             self.logger.debug(logging_text + "Exit")
2186             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
2187             return nslcmop_operation_state, nslcmop_operation_state_detail
2188
2189     async def scale(self, nsr_id, nslcmop_id):
2190
2191         # Try to lock HA task here
2192         task_is_locked_by_me = self.lcm_tasks.lock_HA('ns', 'nslcmops', nslcmop_id)
2193         if not task_is_locked_by_me:
2194             return
2195
2196         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
2197         self.logger.debug(logging_text + "Enter")
2198         # get all needed from database
2199         db_nsr = None
2200         db_nslcmop = None
2201         db_nslcmop_update = {}
2202         nslcmop_operation_state = None
2203         db_nsr_update = {"_admin.nslcmop": nslcmop_id}
2204         exc = None
2205         # in case of error, indicates what part of scale was failed to put nsr at error status
2206         scale_process = None
2207         old_operational_status = ""
2208         old_config_status = ""
2209         vnfr_scaled = False
2210         try:
2211             # wait for any previous tasks in process
2212             await self.lcm_tasks.waitfor_related_HA('ns', 'nslcmops', nslcmop_id)
2213
2214             step = "Getting nslcmop from database"
2215             self.logger.debug(step + " after having waited for previous tasks to be completed")
2216             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2217             step = "Getting nsr from database"
2218             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2219
2220             old_operational_status = db_nsr["operational-status"]
2221             old_config_status = db_nsr["config-status"]
2222             step = "Parsing scaling parameters"
2223             # self.logger.debug(step)
2224             db_nsr_update["operational-status"] = "scaling"
2225             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2226             nsr_deployed = db_nsr["_admin"].get("deployed")
2227             RO_nsr_id = nsr_deployed["RO"]["nsr_id"]
2228             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["member-vnf-index"]
2229             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"]["scaling-group-descriptor"]
2230             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
2231             # scaling_policy = db_nslcmop["operationParams"]["scaleVnfData"]["scaleByStepData"].get("scaling-policy")
2232
2233             # for backward compatibility
2234             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
2235                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
2236                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
2237                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2238
2239             step = "Getting vnfr from database"
2240             db_vnfr = self.db.get_one("vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id})
2241             step = "Getting vnfd from database"
2242             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
2243
2244             step = "Getting scaling-group-descriptor"
2245             for scaling_descriptor in db_vnfd["scaling-group-descriptor"]:
2246                 if scaling_descriptor["name"] == scaling_group:
2247                     break
2248             else:
2249                 raise LcmException("input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
2250                                    "at vnfd:scaling-group-descriptor".format(scaling_group))
2251
2252             # cooldown_time = 0
2253             # for scaling_policy_descriptor in scaling_descriptor.get("scaling-policy", ()):
2254             #     cooldown_time = scaling_policy_descriptor.get("cooldown-time", 0)
2255             #     if scaling_policy and scaling_policy == scaling_policy_descriptor.get("name"):
2256             #         break
2257
2258             # TODO check if ns is in a proper status
2259             step = "Sending scale order to VIM"
2260             nb_scale_op = 0
2261             if not db_nsr["_admin"].get("scaling-group"):
2262                 self.update_db_2("nsrs", nsr_id, {"_admin.scaling-group": [{"name": scaling_group, "nb-scale-op": 0}]})
2263                 admin_scale_index = 0
2264             else:
2265                 for admin_scale_index, admin_scale_info in enumerate(db_nsr["_admin"]["scaling-group"]):
2266                     if admin_scale_info["name"] == scaling_group:
2267                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
2268                         break
2269                 else:  # not found, set index one plus last element and add new entry with the name
2270                     admin_scale_index += 1
2271                     db_nsr_update["_admin.scaling-group.{}.name".format(admin_scale_index)] = scaling_group
2272             RO_scaling_info = []
2273             vdu_scaling_info = {"scaling_group_name": scaling_group, "vdu": []}
2274             if scaling_type == "SCALE_OUT":
2275                 # count if max-instance-count is reached
2276                 max_instance_count = scaling_descriptor.get("max-instance-count", 10)
2277                 # self.logger.debug("MAX_INSTANCE_COUNT is {}".format(max_instance_count))
2278                 if nb_scale_op >= max_instance_count:
2279                     raise LcmException("reached the limit of {} (max-instance-count) "
2280                                        "scaling-out operations for the "
2281                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2282
2283                 nb_scale_op += 1
2284                 vdu_scaling_info["scaling_direction"] = "OUT"
2285                 vdu_scaling_info["vdu-create"] = {}
2286                 for vdu_scale_info in scaling_descriptor["vdu"]:
2287                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2288                                             "type": "create", "count": vdu_scale_info.get("count", 1)})
2289                     vdu_scaling_info["vdu-create"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2290
2291             elif scaling_type == "SCALE_IN":
2292                 # count if min-instance-count is reached
2293                 min_instance_count = 0
2294                 if "min-instance-count" in scaling_descriptor and scaling_descriptor["min-instance-count"] is not None:
2295                     min_instance_count = int(scaling_descriptor["min-instance-count"])
2296                 if nb_scale_op <= min_instance_count:
2297                     raise LcmException("reached the limit of {} (min-instance-count) scaling-in operations for the "
2298                                        "scaling-group-descriptor '{}'".format(nb_scale_op, scaling_group))
2299                 nb_scale_op -= 1
2300                 vdu_scaling_info["scaling_direction"] = "IN"
2301                 vdu_scaling_info["vdu-delete"] = {}
2302                 for vdu_scale_info in scaling_descriptor["vdu"]:
2303                     RO_scaling_info.append({"osm_vdu_id": vdu_scale_info["vdu-id-ref"], "member-vnf-index": vnf_index,
2304                                             "type": "delete", "count": vdu_scale_info.get("count", 1)})
2305                     vdu_scaling_info["vdu-delete"][vdu_scale_info["vdu-id-ref"]] = vdu_scale_info.get("count", 1)
2306
2307             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
2308             vdu_create = vdu_scaling_info.get("vdu-create")
2309             vdu_delete = copy(vdu_scaling_info.get("vdu-delete"))
2310             if vdu_scaling_info["scaling_direction"] == "IN":
2311                 for vdur in reversed(db_vnfr["vdur"]):
2312                     if vdu_delete.get(vdur["vdu-id-ref"]):
2313                         vdu_delete[vdur["vdu-id-ref"]] -= 1
2314                         vdu_scaling_info["vdu"].append({
2315                             "name": vdur["name"],
2316                             "vdu_id": vdur["vdu-id-ref"],
2317                             "interface": []
2318                         })
2319                         for interface in vdur["interfaces"]:
2320                             vdu_scaling_info["vdu"][-1]["interface"].append({
2321                                 "name": interface["name"],
2322                                 "ip_address": interface["ip-address"],
2323                                 "mac_address": interface.get("mac-address"),
2324                             })
2325                 vdu_delete = vdu_scaling_info.pop("vdu-delete")
2326
2327             # execute primitive service PRE-SCALING
2328             step = "Executing pre-scale vnf-config-primitive"
2329             if scaling_descriptor.get("scaling-config-action"):
2330                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2331                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "pre-scale-in" \
2332                             and scaling_type == "SCALE_IN":
2333                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2334                         step = db_nslcmop_update["detailed-status"] = \
2335                             "executing pre-scale scaling-config-action '{}'".format(vnf_config_primitive)
2336
2337                         # look for primitive
2338                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2339                             if config_primitive["name"] == vnf_config_primitive:
2340                                 break
2341                         else:
2342                             raise LcmException(
2343                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
2344                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
2345                                 "primitive".format(scaling_group, config_primitive))
2346
2347                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2348                         if db_vnfr.get("additionalParamsForVnf"):
2349                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2350
2351                         scale_process = "VCA"
2352                         db_nsr_update["config-status"] = "configuring pre-scaling"
2353                         result, result_detail = await self._ns_execute_primitive(
2354                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2355                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2356                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2357                             vnf_config_primitive, result, result_detail))
2358                         if result == "FAILED":
2359                             raise LcmException(result_detail)
2360                         db_nsr_update["config-status"] = old_config_status
2361                         scale_process = None
2362
2363             if RO_scaling_info:
2364                 scale_process = "RO"
2365                 RO_desc = await self.RO.create_action("ns", RO_nsr_id, {"vdu-scaling": RO_scaling_info})
2366                 db_nsr_update["_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)] = nb_scale_op
2367                 db_nsr_update["_admin.scaling-group.{}.time".format(admin_scale_index)] = time()
2368                 # wait until ready
2369                 RO_nslcmop_id = RO_desc["instance_action_id"]
2370                 db_nslcmop_update["_admin.deploy.RO"] = RO_nslcmop_id
2371
2372                 RO_task_done = False
2373                 step = detailed_status = "Waiting RO_task_id={} to complete the scale action.".format(RO_nslcmop_id)
2374                 detailed_status_old = None
2375                 self.logger.debug(logging_text + step)
2376
2377                 deployment_timeout = 1 * 3600   # One hour
2378                 while deployment_timeout > 0:
2379                     if not RO_task_done:
2380                         desc = await self.RO.show("ns", item_id_name=RO_nsr_id, extra_item="action",
2381                                                   extra_item_id=RO_nslcmop_id)
2382                         ns_status, ns_status_info = self.RO.check_action_status(desc)
2383                         if ns_status == "ERROR":
2384                             raise ROclient.ROClientException(ns_status_info)
2385                         elif ns_status == "BUILD":
2386                             detailed_status = step + "; {}".format(ns_status_info)
2387                         elif ns_status == "ACTIVE":
2388                             RO_task_done = True
2389                             step = detailed_status = "Waiting VIM to deploy ns. RO_id={}".format(RO_nsr_id)
2390                             self.logger.debug(logging_text + step)
2391                         else:
2392                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
2393                     else:
2394                         desc = await self.RO.show("ns", RO_nsr_id)
2395                         ns_status, ns_status_info = self.RO.check_ns_status(desc)
2396                         if ns_status == "ERROR":
2397                             raise ROclient.ROClientException(ns_status_info)
2398                         elif ns_status == "BUILD":
2399                             detailed_status = step + "; {}".format(ns_status_info)
2400                         elif ns_status == "ACTIVE":
2401                             step = detailed_status = \
2402                                 "Waiting for management IP address reported by the VIM. Updating VNFRs"
2403                             if not vnfr_scaled:
2404                                 self.scale_vnfr(db_vnfr, vdu_create=vdu_create, vdu_delete=vdu_delete)
2405                                 vnfr_scaled = True
2406                             try:
2407                                 desc = await self.RO.show("ns", RO_nsr_id)
2408                                 # nsr_deployed["nsr_ip"] = RO.get_ns_vnf_info(desc)
2409                                 self.ns_update_vnfr({db_vnfr["member-vnf-index-ref"]: db_vnfr}, desc)
2410                                 break
2411                             except LcmExceptionNoMgmtIP:
2412                                 pass
2413                         else:
2414                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
2415                     if detailed_status != detailed_status_old:
2416                         detailed_status_old = db_nslcmop_update["detailed-status"] = detailed_status
2417                         self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2418
2419                     await asyncio.sleep(5, loop=self.loop)
2420                     deployment_timeout -= 5
2421                 if deployment_timeout <= 0:
2422                     raise ROclient.ROClientException("Timeout waiting ns to be ready")
2423
2424                 # update VDU_SCALING_INFO with the obtained ip_addresses
2425                 if vdu_scaling_info["scaling_direction"] == "OUT":
2426                     for vdur in reversed(db_vnfr["vdur"]):
2427                         if vdu_scaling_info["vdu-create"].get(vdur["vdu-id-ref"]):
2428                             vdu_scaling_info["vdu-create"][vdur["vdu-id-ref"]] -= 1
2429                             vdu_scaling_info["vdu"].append({
2430                                 "name": vdur["name"],
2431                                 "vdu_id": vdur["vdu-id-ref"],
2432                                 "interface": []
2433                             })
2434                             for interface in vdur["interfaces"]:
2435                                 vdu_scaling_info["vdu"][-1]["interface"].append({
2436                                     "name": interface["name"],
2437                                     "ip_address": interface["ip-address"],
2438                                     "mac_address": interface.get("mac-address"),
2439                                 })
2440                     del vdu_scaling_info["vdu-create"]
2441
2442             scale_process = None
2443             if db_nsr_update:
2444                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2445
2446             # execute primitive service POST-SCALING
2447             step = "Executing post-scale vnf-config-primitive"
2448             if scaling_descriptor.get("scaling-config-action"):
2449                 for scaling_config_action in scaling_descriptor["scaling-config-action"]:
2450                     if scaling_config_action.get("trigger") and scaling_config_action["trigger"] == "post-scale-out" \
2451                             and scaling_type == "SCALE_OUT":
2452                         vnf_config_primitive = scaling_config_action["vnf-config-primitive-name-ref"]
2453                         step = db_nslcmop_update["detailed-status"] = \
2454                             "executing post-scale scaling-config-action '{}'".format(vnf_config_primitive)
2455
2456                         vnfr_params = {"VDU_SCALE_INFO": vdu_scaling_info}
2457                         if db_vnfr.get("additionalParamsForVnf"):
2458                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
2459
2460                         # look for primitive
2461                         for config_primitive in db_vnfd.get("vnf-configuration", {}).get("config-primitive", ()):
2462                             if config_primitive["name"] == vnf_config_primitive:
2463                                 break
2464                         else:
2465                             raise LcmException("Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:"
2466                                                "scaling-config-action[vnf-config-primitive-name-ref='{}'] does not "
2467                                                "match any vnf-configuration:config-primitive".format(scaling_group,
2468                                                                                                      config_primitive))
2469                         scale_process = "VCA"
2470                         db_nsr_update["config-status"] = "configuring post-scaling"
2471
2472                         result, result_detail = await self._ns_execute_primitive(
2473                             nsr_deployed, vnf_index, None, None, None, vnf_config_primitive,
2474                             self._map_primitive_params(config_primitive, {}, vnfr_params))
2475                         self.logger.debug(logging_text + "vnf_config_primitive={} Done with result {} {}".format(
2476                             vnf_config_primitive, result, result_detail))
2477                         if result == "FAILED":
2478                             raise LcmException(result_detail)
2479                         db_nsr_update["config-status"] = old_config_status
2480                         scale_process = None
2481
2482             db_nslcmop_update["operationState"] = nslcmop_operation_state = "COMPLETED"
2483             db_nslcmop_update["statusEnteredTime"] = time()
2484             db_nslcmop_update["detailed-status"] = "done"
2485             db_nsr_update["detailed-status"] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
2486             db_nsr_update["operational-status"] = "running" if old_operational_status == "failed" \
2487                 else old_operational_status
2488             db_nsr_update["config-status"] = old_config_status
2489             return
2490         except (ROclient.ROClientException, DbException, LcmException) as e:
2491             self.logger.error(logging_text + "Exit Exception {}".format(e))
2492             exc = e
2493         except asyncio.CancelledError:
2494             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
2495             exc = "Operation was cancelled"
2496         except Exception as e:
2497             exc = traceback.format_exc()
2498             self.logger.critical(logging_text + "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True)
2499         finally:
2500             if exc:
2501                 if db_nslcmop:
2502                     db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
2503                     db_nslcmop_update["operationState"] = nslcmop_operation_state = "FAILED"
2504                     db_nslcmop_update["statusEnteredTime"] = time()
2505                 if db_nsr:
2506                     db_nsr_update["operational-status"] = old_operational_status
2507                     db_nsr_update["config-status"] = old_config_status
2508                     db_nsr_update["detailed-status"] = ""
2509                     db_nsr_update["_admin.nslcmop"] = None
2510                     if scale_process:
2511                         if "VCA" in scale_process:
2512                             db_nsr_update["config-status"] = "failed"
2513                         if "RO" in scale_process:
2514                             db_nsr_update["operational-status"] = "failed"
2515                         db_nsr_update["detailed-status"] = "FAILED scaling nslcmop={} {}: {}".format(nslcmop_id, step,
2516                                                                                                      exc)
2517             try:
2518                 if db_nslcmop and db_nslcmop_update:
2519                     self.update_db_2("nslcmops", nslcmop_id, db_nslcmop_update)
2520                 if db_nsr:
2521                     db_nsr_update["_admin.nslcmop"] = None
2522                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
2523             except DbException as e:
2524                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
2525             if nslcmop_operation_state:
2526                 try:
2527                     await self.msg.aiowrite("ns", "scaled", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id,
2528                                                              "operationState": nslcmop_operation_state},
2529                                             loop=self.loop)
2530                     # if cooldown_time:
2531                     #     await asyncio.sleep(cooldown_time)
2532                     # await self.msg.aiowrite("ns","scaled-cooldown-time", {"nsr_id": nsr_id, "nslcmop_id": nslcmop_id})
2533                 except Exception as e:
2534                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
2535             self.logger.debug(logging_text + "Exit")
2536             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")