osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import shutil
  21 from typing import Any, Dict, List
  22 import yaml
  23 import logging
  24 import logging.handlers
  25 import traceback
  26 import ipaddress
  27 import json
  28 from jinja2 import (
  29     Environment,
  30     TemplateError,
  31     TemplateNotFound,
  32     StrictUndefined,
  33     UndefinedError,
  34     select_autoescape,
  35 )
  36
  37 from osm_lcm import ROclient
  38 from osm_lcm.data_utils.lcm_config import LcmCfg
  39 from osm_lcm.data_utils.nsr import (
  40     get_deployed_kdu,
  41     get_deployed_vca,
  42     get_deployed_vca_list,
  43     get_nsd,
  44 )
  45 from osm_lcm.data_utils.vca import (
  46     DeployedComponent,
  47     DeployedK8sResource,
  48     DeployedVCA,
  49     EELevel,
  50     Relation,
  51     EERelation,
  52     safe_get_ee_relation,
  53 )
  54 from osm_lcm.ng_ro import NgRoClient, NgRoException
  55 from osm_lcm.lcm_utils import (
  56     LcmException,
  57     LcmBase,
  58     deep_get,
  59     get_iterable,
  60     populate_dict,
  61     check_juju_bundle_existence,
  62     get_charm_artifact_path,
  63     get_ee_id_parts,
  64     vld_to_ro_ip_profile,
  65 )
  66 from osm_lcm.data_utils.nsd import (
  67     get_ns_configuration_relation_list,
  68     get_vnf_profile,
  69     get_vnf_profiles,
  70 )
  71 from osm_lcm.data_utils.vnfd import (
  72     get_kdu,
  73     get_kdu_services,
  74     get_relation_list,
  75     get_vdu_list,
  76     get_vdu_profile,
  77     get_ee_sorted_initial_config_primitive_list,
  78     get_ee_sorted_terminate_config_primitive_list,
  79     get_kdu_list,
  80     get_virtual_link_profiles,
  81     get_vdu,
  82     get_configuration,
  83     get_vdu_index,
  84     get_scaling_aspect,
  85     get_number_of_instances,
  86     get_juju_ee_ref,
  87     get_kdu_resource_profile,
  88     find_software_version,
  89     check_helm_ee_in_ns,
  90 )
  91 from osm_lcm.data_utils.list_utils import find_in_list
  92 from osm_lcm.data_utils.vnfr import (
  93     get_osm_params,
  94     get_vdur_index,
  95     get_kdur,
  96     get_volumes_from_instantiation_params,
  97 )
  98 from osm_lcm.data_utils.dict_utils import parse_yaml_strings
  99 from osm_lcm.data_utils.database.vim_account import VimAccountDB
 100 from n2vc.definitions import RelationEndpoint
 101 from n2vc.k8s_helm3_conn import K8sHelm3Connector
 102 from n2vc.k8s_juju_conn import K8sJujuConnector
 103
 104 from osm_common.dbbase import DbException
 105 from osm_common.fsbase import FsException
 106
 107 from osm_lcm.data_utils.database.database import Database
 108 from osm_lcm.data_utils.filesystem.filesystem import Filesystem
 109 from osm_lcm.data_utils.wim import (
 110     get_sdn_ports,
 111     get_target_wim_attrs,
 112     select_feasible_wim_account,
 113 )
 114
 115 from n2vc.n2vc_juju_conn import N2VCJujuConnector
 116 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
 117
 118 from osm_lcm.lcm_helm_conn import LCMHelmConn
 119 from osm_lcm.osm_config import OsmConfigBuilder
 120 from osm_lcm.prometheus import parse_job
 121
 122 from copy import copy, deepcopy
 123 from time import time
 124 from uuid import uuid4
 125
 126 from random import SystemRandom
 127
 128 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 129
 130
 131 class NsLcm(LcmBase):
 132     SUBOPERATION_STATUS_NOT_FOUND = -1
 133     SUBOPERATION_STATUS_NEW = -2
 134     SUBOPERATION_STATUS_SKIP = -3
 135     EE_TLS_NAME = "ee-tls"
 136     task_name_deploy_vca = "Deploying VCA"
 137     rel_operation_types = {
 138         "GE": ">=",
 139         "LE": "<=",
 140         "GT": ">",
 141         "LT": "<",
 142         "EQ": "==",
 143         "NE": "!=",
 144     }
 145
 146     def __init__(self, msg, lcm_tasks, config: LcmCfg):
 147         """
 148         Init, Connect to database, filesystem storage, and messaging
 149         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 150         :return: None
 151         """
 152         super().__init__(msg=msg, logger=logging.getLogger("lcm.ns"))
 153
 154         self.db = Database().instance.db
 155         self.fs = Filesystem().instance.fs
 156         self.lcm_tasks = lcm_tasks
 157         self.timeout = config.timeout
 158         self.ro_config = config.RO
 159         self.vca_config = config.VCA
 160
 161         # create N2VC connector
 162         self.n2vc = N2VCJujuConnector(
 163             log=self.logger,
 164             on_update_db=self._on_update_n2vc_db,
 165             fs=self.fs,
 166             db=self.db,
 167         )
 168
 169         self.conn_helm_ee = LCMHelmConn(
 170             log=self.logger,
 171             vca_config=self.vca_config,
 172             on_update_db=self._on_update_n2vc_db,
 173         )
 174
 175         self.k8sclusterhelm3 = K8sHelm3Connector(
 176             kubectl_command=self.vca_config.kubectlpath,
 177             helm_command=self.vca_config.helm3path,
 178             fs=self.fs,
 179             log=self.logger,
 180             db=self.db,
 181             on_update_db=None,
 182         )
 183
 184         self.k8sclusterjuju = K8sJujuConnector(
 185             kubectl_command=self.vca_config.kubectlpath,
 186             juju_command=self.vca_config.jujupath,
 187             log=self.logger,
 188             on_update_db=self._on_update_k8s_db,
 189             fs=self.fs,
 190             db=self.db,
 191         )
 192
 193         self.k8scluster_map = {
 194             "helm-chart-v3": self.k8sclusterhelm3,
 195             "chart": self.k8sclusterhelm3,
 196             "juju-bundle": self.k8sclusterjuju,
 197             "juju": self.k8sclusterjuju,
 198         }
 199
 200         self.vca_map = {
 201             "lxc_proxy_charm": self.n2vc,
 202             "native_charm": self.n2vc,
 203             "k8s_proxy_charm": self.n2vc,
 204             "helm": self.conn_helm_ee,
 205             "helm-v3": self.conn_helm_ee,
 206         }
 207
 208         # create RO client
 209         self.RO = NgRoClient(**self.ro_config.to_dict())
 210
 211         self.op_status_map = {
 212             "instantiation": self.RO.status,
 213             "termination": self.RO.status,
 214             "migrate": self.RO.status,
 215             "healing": self.RO.recreate_status,
 216             "verticalscale": self.RO.status,
 217             "start_stop_rebuild": self.RO.status,
 218         }
 219
 220     @staticmethod
 221     def increment_ip_mac(ip_mac, vm_index=1):
 222         if not isinstance(ip_mac, str):
 223             return ip_mac
 224         try:
 225             next_ipv6 = None
 226             next_ipv4 = None
 227             dual_ip = ip_mac.split(";")
 228             if len(dual_ip) == 2:
 229                 for ip in dual_ip:
 230                     if ipaddress.ip_address(ip).version == 6:
 231                         ipv6 = ipaddress.IPv6Address(ip)
 232                         next_ipv6 = str(ipaddress.IPv6Address(int(ipv6) + 1))
 233                     elif ipaddress.ip_address(ip).version == 4:
 234                         ipv4 = ipaddress.IPv4Address(ip)
 235                         next_ipv4 = str(ipaddress.IPv4Address(int(ipv4) + 1))
 236                 return [next_ipv4, next_ipv6]
 237             # try with ipv4 look for last dot
 238             i = ip_mac.rfind(".")
 239             if i > 0:
 240                 i += 1
 241                 return "{}{}".format(ip_mac[:i], int(ip_mac[i:]) + vm_index)
 242             # try with ipv6 or mac look for last colon. Operate in hex
 243             i = ip_mac.rfind(":")
 244             if i > 0:
 245                 i += 1
 246                 # format in hex, len can be 2 for mac or 4 for ipv6
 247                 return ("{}{:0" + str(len(ip_mac) - i) + "x}").format(
 248                     ip_mac[:i], int(ip_mac[i:], 16) + vm_index
 249                 )
 250         except Exception:
 251             pass
 252         return None
 253
 254     async def _on_update_n2vc_db(self, table, filter, path, updated_data, vca_id=None):
 255         # remove last dot from path (if exists)
 256         if path.endswith("."):
 257             path = path[:-1]
 258
 259         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 260         #                   .format(table, filter, path, updated_data))
 261         try:
 262             nsr_id = filter.get("_id")
 263
 264             # read ns record from database
 265             nsr = self.db.get_one(table="nsrs", q_filter=filter)
 266             current_ns_status = nsr.get("nsState")
 267
 268             # get vca status for NS
 269             status_dict = await self.n2vc.get_status(
 270                 namespace="." + nsr_id, yaml_format=False, vca_id=vca_id
 271             )
 272
 273             # vcaStatus
 274             db_dict = dict()
 275             db_dict["vcaStatus"] = status_dict
 276
 277             # update configurationStatus for this VCA
 278             try:
 279                 vca_index = int(path[path.rfind(".") + 1 :])
 280
 281                 vca_list = deep_get(
 282                     target_dict=nsr, key_list=("_admin", "deployed", "VCA")
 283                 )
 284                 vca_status = vca_list[vca_index].get("status")
 285
 286                 configuration_status_list = nsr.get("configurationStatus")
 287                 config_status = configuration_status_list[vca_index].get("status")
 288
 289                 if config_status == "BROKEN" and vca_status != "failed":
 290                     db_dict["configurationStatus"][vca_index] = "READY"
 291                 elif config_status != "BROKEN" and vca_status == "failed":
 292                     db_dict["configurationStatus"][vca_index] = "BROKEN"
 293             except Exception as e:
 294                 # not update configurationStatus
 295                 self.logger.debug("Error updating vca_index (ignore): {}".format(e))
 296
 297             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 298             # if nsState = 'DEGRADED' check if all is OK
 299             is_degraded = False
 300             if current_ns_status in ("READY", "DEGRADED"):
 301                 error_description = ""
 302                 # check machines
 303                 if status_dict.get("machines"):
 304                     for machine_id in status_dict.get("machines"):
 305                         machine = status_dict.get("machines").get(machine_id)
 306                         # check machine agent-status
 307                         if machine.get("agent-status"):
 308                             s = machine.get("agent-status").get("status")
 309                             if s != "started":
 310                                 is_degraded = True
 311                                 error_description += (
 312                                     "machine {} agent-status={} ; ".format(
 313                                         machine_id, s
 314                                     )
 315                                 )
 316                         # check machine instance status
 317                         if machine.get("instance-status"):
 318                             s = machine.get("instance-status").get("status")
 319                             if s != "running":
 320                                 is_degraded = True
 321                                 error_description += (
 322                                     "machine {} instance-status={} ; ".format(
 323                                         machine_id, s
 324                                     )
 325                                 )
 326                 # check applications
 327                 if status_dict.get("applications"):
 328                     for app_id in status_dict.get("applications"):
 329                         app = status_dict.get("applications").get(app_id)
 330                         # check application status
 331                         if app.get("status"):
 332                             s = app.get("status").get("status")
 333                             if s != "active":
 334                                 is_degraded = True
 335                                 error_description += (
 336                                     "application {} status={} ; ".format(app_id, s)
 337                                 )
 338
 339                 if error_description:
 340                     db_dict["errorDescription"] = error_description
 341                 if current_ns_status == "READY" and is_degraded:
 342                     db_dict["nsState"] = "DEGRADED"
 343                 if current_ns_status == "DEGRADED" and not is_degraded:
 344                     db_dict["nsState"] = "READY"
 345
 346             # write to database
 347             self.update_db_2("nsrs", nsr_id, db_dict)
 348
 349         except (asyncio.CancelledError, asyncio.TimeoutError):
 350             raise
 351         except Exception as e:
 352             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 353
 354     async def _on_update_k8s_db(
 355         self, cluster_uuid, kdu_instance, filter=None, vca_id=None, cluster_type="juju"
 356     ):
 357         """
 358         Updating vca status in NSR record
 359         :param cluster_uuid: UUID of a k8s cluster
 360         :param kdu_instance: The unique name of the KDU instance
 361         :param filter: To get nsr_id
 362         :cluster_type: The cluster type (juju, k8s)
 363         :return: none
 364         """
 365
 366         # self.logger.debug("_on_update_k8s_db(cluster_uuid={}, kdu_instance={}, filter={}"
 367         #                   .format(cluster_uuid, kdu_instance, filter))
 368
 369         nsr_id = filter.get("_id")
 370         try:
 371             vca_status = await self.k8scluster_map[cluster_type].status_kdu(
 372                 cluster_uuid=cluster_uuid,
 373                 kdu_instance=kdu_instance,
 374                 yaml_format=False,
 375                 complete_status=True,
 376                 vca_id=vca_id,
 377             )
 378
 379             # vcaStatus
 380             db_dict = dict()
 381             db_dict["vcaStatus"] = {nsr_id: vca_status}
 382
 383             self.logger.debug(
 384                 f"Obtained VCA status for cluster type '{cluster_type}': {vca_status}"
 385             )
 386
 387             # write to database
 388             self.update_db_2("nsrs", nsr_id, db_dict)
 389         except (asyncio.CancelledError, asyncio.TimeoutError):
 390             raise
 391         except Exception as e:
 392             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 393
 394     @staticmethod
 395     def _parse_cloud_init(cloud_init_text, additional_params, vnfd_id, vdu_id):
 396         try:
 397             env = Environment(
 398                 undefined=StrictUndefined,
 399                 autoescape=select_autoescape(default_for_string=True, default=True),
 400             )
 401             template = env.from_string(cloud_init_text)
 402             return template.render(additional_params or {})
 403         except UndefinedError as e:
 404             raise LcmException(
 405                 "Variable {} at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 406                 "file, must be provided in the instantiation parameters inside the "
 407                 "'additionalParamsForVnf/Vdu' block".format(e, vnfd_id, vdu_id)
 408             )
 409         except (TemplateError, TemplateNotFound) as e:
 410             raise LcmException(
 411                 "Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".format(
 412                     vnfd_id, vdu_id, e
 413                 )
 414             )
 415
 416     def _get_vdu_cloud_init_content(self, vdu, vnfd):
 417         cloud_init_content = cloud_init_file = None
 418         try:
 419             if vdu.get("cloud-init-file"):
 420                 base_folder = vnfd["_admin"]["storage"]
 421                 if base_folder["pkg-dir"]:
 422                     cloud_init_file = "{}/{}/cloud_init/{}".format(
 423                         base_folder["folder"],
 424                         base_folder["pkg-dir"],
 425                         vdu["cloud-init-file"],
 426                     )
 427                 else:
 428                     cloud_init_file = "{}/Scripts/cloud_init/{}".format(
 429                         base_folder["folder"],
 430                         vdu["cloud-init-file"],
 431                     )
 432                 with self.fs.file_open(cloud_init_file, "r") as ci_file:
 433                     cloud_init_content = ci_file.read()
 434             elif vdu.get("cloud-init"):
 435                 cloud_init_content = vdu["cloud-init"]
 436
 437             return cloud_init_content
 438         except FsException as e:
 439             raise LcmException(
 440                 "Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".format(
 441                     vnfd["id"], vdu["id"], cloud_init_file, e
 442                 )
 443             )
 444
 445     def _get_vdu_additional_params(self, db_vnfr, vdu_id):
 446         vdur = next(
 447             (vdur for vdur in db_vnfr.get("vdur") if vdu_id == vdur["vdu-id-ref"]), {}
 448         )
 449         additional_params = vdur.get("additionalParams")
 450         return parse_yaml_strings(additional_params)
 451
 452     @staticmethod
 453     def ip_profile_2_RO(ip_profile):
 454         RO_ip_profile = deepcopy(ip_profile)
 455         if "dns-server" in RO_ip_profile:
 456             if isinstance(RO_ip_profile["dns-server"], list):
 457                 RO_ip_profile["dns-address"] = []
 458                 for ds in RO_ip_profile.pop("dns-server"):
 459                     RO_ip_profile["dns-address"].append(ds["address"])
 460             else:
 461                 RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 462         if RO_ip_profile.get("ip-version") == "ipv4":
 463             RO_ip_profile["ip-version"] = "IPv4"
 464         if RO_ip_profile.get("ip-version") == "ipv6":
 465             RO_ip_profile["ip-version"] = "IPv6"
 466         if "dhcp-params" in RO_ip_profile:
 467             RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 468         return RO_ip_profile
 469
 470     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None, mark_delete=False):
 471         db_vdu_push_list = []
 472         template_vdur = []
 473         db_update = {"_admin.modified": time()}
 474         if vdu_create:
 475             for vdu_id, vdu_count in vdu_create.items():
 476                 vdur = next(
 477                     (
 478                         vdur
 479                         for vdur in reversed(db_vnfr["vdur"])
 480                         if vdur["vdu-id-ref"] == vdu_id
 481                     ),
 482                     None,
 483                 )
 484                 if not vdur:
 485                     # Read the template saved in the db:
 486                     self.logger.debug(
 487                         "No vdur in the database. Using the vdur-template to scale"
 488                     )
 489                     vdur_template = db_vnfr.get("vdur-template")
 490                     if not vdur_template:
 491                         raise LcmException(
 492                             "Error scaling OUT VNFR for {}. No vnfr or template exists".format(
 493                                 vdu_id
 494                             )
 495                         )
 496                     vdur = vdur_template[0]
 497                     # Delete a template from the database after using it
 498                     self.db.set_one(
 499                         "vnfrs",
 500                         {"_id": db_vnfr["_id"]},
 501                         None,
 502                         pull={"vdur-template": {"_id": vdur["_id"]}},
 503                     )
 504                 for count in range(vdu_count):
 505                     vdur_copy = deepcopy(vdur)
 506                     vdur_copy["status"] = "BUILD"
 507                     vdur_copy["status-detailed"] = None
 508                     vdur_copy["ip-address"] = None
 509                     vdur_copy["_id"] = str(uuid4())
 510                     vdur_copy["count-index"] += count + 1
 511                     vdur_copy["id"] = "{}-{}".format(
 512                         vdur_copy["vdu-id-ref"], vdur_copy["count-index"]
 513                     )
 514                     vdur_copy.pop("vim_info", None)
 515                     for iface in vdur_copy["interfaces"]:
 516                         if iface.get("fixed-ip"):
 517                             iface["ip-address"] = self.increment_ip_mac(
 518                                 iface["ip-address"], count + 1
 519                             )
 520                         else:
 521                             iface.pop("ip-address", None)
 522                         if iface.get("fixed-mac"):
 523                             iface["mac-address"] = self.increment_ip_mac(
 524                                 iface["mac-address"], count + 1
 525                             )
 526                         else:
 527                             iface.pop("mac-address", None)
 528                         if db_vnfr["vdur"]:
 529                             iface.pop(
 530                                 "mgmt_vnf", None
 531                             )  # only first vdu can be managment of vnf
 532                     db_vdu_push_list.append(vdur_copy)
 533                     # self.logger.debug("scale out, adding vdu={}".format(vdur_copy))
 534         if vdu_delete:
 535             if len(db_vnfr["vdur"]) == 1:
 536                 # The scale will move to 0 instances
 537                 self.logger.debug(
 538                     "Scaling to 0 !, creating the template with the last vdur"
 539                 )
 540                 template_vdur = [db_vnfr["vdur"][0]]
 541             for vdu_id, vdu_count in vdu_delete.items():
 542                 if mark_delete:
 543                     indexes_to_delete = [
 544                         iv[0]
 545                         for iv in enumerate(db_vnfr["vdur"])
 546                         if iv[1]["vdu-id-ref"] == vdu_id
 547                     ]
 548                     db_update.update(
 549                         {
 550                             "vdur.{}.status".format(i): "DELETING"
 551                             for i in indexes_to_delete[-vdu_count:]
 552                         }
 553                     )
 554                 else:
 555                     # it must be deleted one by one because common.db does not allow otherwise
 556                     vdus_to_delete = [
 557                         v
 558                         for v in reversed(db_vnfr["vdur"])
 559                         if v["vdu-id-ref"] == vdu_id
 560                     ]
 561                     for vdu in vdus_to_delete[:vdu_count]:
 562                         self.db.set_one(
 563                             "vnfrs",
 564                             {"_id": db_vnfr["_id"]},
 565                             None,
 566                             pull={"vdur": {"_id": vdu["_id"]}},
 567                         )
 568         db_push = {}
 569         if db_vdu_push_list:
 570             db_push["vdur"] = db_vdu_push_list
 571         if template_vdur:
 572             db_push["vdur-template"] = template_vdur
 573         if not db_push:
 574             db_push = None
 575         db_vnfr["vdur-template"] = template_vdur
 576         self.db.set_one("vnfrs", {"_id": db_vnfr["_id"]}, db_update, push_list=db_push)
 577         # modify passed dictionary db_vnfr
 578         db_vnfr_ = self.db.get_one("vnfrs", {"_id": db_vnfr["_id"]})
 579         db_vnfr["vdur"] = db_vnfr_["vdur"]
 580
 581     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 582         """
 583         Updates database nsr with the RO info for the created vld
 584         :param ns_update_nsr: dictionary to be filled with the updated info
 585         :param db_nsr: content of db_nsr. This is also modified
 586         :param nsr_desc_RO: nsr descriptor from RO
 587         :return: Nothing, LcmException is raised on errors
 588         """
 589
 590         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 591             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 592                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 593                     continue
 594                 vld["vim-id"] = net_RO.get("vim_net_id")
 595                 vld["name"] = net_RO.get("vim_name")
 596                 vld["status"] = net_RO.get("status")
 597                 vld["status-detailed"] = net_RO.get("error_msg")
 598                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 599                 break
 600             else:
 601                 raise LcmException(
 602                     "ns_update_nsr: Not found vld={} at RO info".format(vld["id"])
 603                 )
 604
 605     def set_vnfr_at_error(self, db_vnfrs, error_text):
 606         try:
 607             for db_vnfr in db_vnfrs.values():
 608                 vnfr_update = {"status": "ERROR"}
 609                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 610                     if "status" not in vdur:
 611                         vdur["status"] = "ERROR"
 612                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 613                         if error_text:
 614                             vdur["status-detailed"] = str(error_text)
 615                             vnfr_update[
 616                                 "vdur.{}.status-detailed".format(vdu_index)
 617                             ] = "ERROR"
 618                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 619         except DbException as e:
 620             self.logger.error("Cannot update vnf. {}".format(e))
 621
 622     def _get_ns_config_info(self, nsr_id):
 623         """
 624         Generates a mapping between vnf,vdu elements and the N2VC id
 625         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 626         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 627             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 628             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 629         """
 630         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 631         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 632         mapping = {}
 633         ns_config_info = {"osm-config-mapping": mapping}
 634         for vca in vca_deployed_list:
 635             if not vca["member-vnf-index"]:
 636                 continue
 637             if not vca["vdu_id"]:
 638                 mapping[vca["member-vnf-index"]] = vca["application"]
 639             else:
 640                 mapping[
 641                     "{}.{}.{}".format(
 642                         vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"]
 643                     )
 644                 ] = vca["application"]
 645         return ns_config_info
 646
 647     async def _instantiate_ng_ro(
 648         self,
 649         logging_text,
 650         nsr_id,
 651         nsd,
 652         db_nsr,
 653         db_nslcmop,
 654         db_vnfrs,
 655         db_vnfds,
 656         n2vc_key_list,
 657         stage,
 658         start_deploy,
 659         timeout_ns_deploy,
 660     ):
 661         db_vims = {}
 662
 663         def get_vim_account(vim_account_id):
 664             nonlocal db_vims
 665             if vim_account_id in db_vims:
 666                 return db_vims[vim_account_id]
 667             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
 668             db_vims[vim_account_id] = db_vim
 669             return db_vim
 670
 671         # modify target_vld info with instantiation parameters
 672         def parse_vld_instantiation_params(
 673             target_vim, target_vld, vld_params, target_sdn
 674         ):
 675             if vld_params.get("ip-profile"):
 676                 target_vld["vim_info"][target_vim]["ip_profile"] = vld_to_ro_ip_profile(
 677                     vld_params["ip-profile"]
 678                 )
 679             if vld_params.get("provider-network"):
 680                 target_vld["vim_info"][target_vim]["provider_network"] = vld_params[
 681                     "provider-network"
 682                 ]
 683                 if "sdn-ports" in vld_params["provider-network"] and target_sdn:
 684                     target_vld["vim_info"][target_sdn]["sdn-ports"] = vld_params[
 685                         "provider-network"
 686                     ]["sdn-ports"]
 687
 688             # check if WIM is needed; if needed, choose a feasible WIM able to connect VIMs
 689             # if wim_account_id is specified in vld_params, validate if it is feasible.
 690             wim_account_id, db_wim = select_feasible_wim_account(
 691                 db_nsr, db_vnfrs, target_vld, vld_params, self.logger
 692             )
 693
 694             if wim_account_id:
 695                 # WIM is needed and a feasible one was found, populate WIM target and SDN ports
 696                 self.logger.info("WIM selected: {:s}".format(str(wim_account_id)))
 697                 # update vld_params with correct WIM account Id
 698                 vld_params["wimAccountId"] = wim_account_id
 699
 700                 target_wim = "wim:{}".format(wim_account_id)
 701                 target_wim_attrs = get_target_wim_attrs(nsr_id, target_vld, vld_params)
 702                 sdn_ports = get_sdn_ports(vld_params, db_wim)
 703                 if len(sdn_ports) > 0:
 704                     target_vld["vim_info"][target_wim] = target_wim_attrs
 705                     target_vld["vim_info"][target_wim]["sdn-ports"] = sdn_ports
 706
 707                 self.logger.debug(
 708                     "Target VLD with WIM data: {:s}".format(str(target_vld))
 709                 )
 710
 711             for param in ("vim-network-name", "vim-network-id"):
 712                 if vld_params.get(param):
 713                     if isinstance(vld_params[param], dict):
 714                         for vim, vim_net in vld_params[param].items():
 715                             other_target_vim = "vim:" + vim
 716                             populate_dict(
 717                                 target_vld["vim_info"],
 718                                 (other_target_vim, param.replace("-", "_")),
 719                                 vim_net,
 720                             )
 721                     else:  # isinstance str
 722                         target_vld["vim_info"][target_vim][
 723                             param.replace("-", "_")
 724                         ] = vld_params[param]
 725             if vld_params.get("common_id"):
 726                 target_vld["common_id"] = vld_params.get("common_id")
 727
 728         # modify target["ns"]["vld"] with instantiation parameters to override vnf vim-account
 729         def update_ns_vld_target(target, ns_params):
 730             for vnf_params in ns_params.get("vnf", ()):
 731                 if vnf_params.get("vimAccountId"):
 732                     target_vnf = next(
 733                         (
 734                             vnfr
 735                             for vnfr in db_vnfrs.values()
 736                             if vnf_params["member-vnf-index"]
 737                             == vnfr["member-vnf-index-ref"]
 738                         ),
 739                         None,
 740                     )
 741                     vdur = next((vdur for vdur in target_vnf.get("vdur", ())), None)
 742                     if not vdur:
 743                         continue
 744                     for a_index, a_vld in enumerate(target["ns"]["vld"]):
 745                         target_vld = find_in_list(
 746                             get_iterable(vdur, "interfaces"),
 747                             lambda iface: iface.get("ns-vld-id") == a_vld["name"],
 748                         )
 749
 750                         vld_params = find_in_list(
 751                             get_iterable(ns_params, "vld"),
 752                             lambda v_vld: v_vld["name"] in (a_vld["name"], a_vld["id"]),
 753                         )
 754                         if target_vld:
 755                             if vnf_params.get("vimAccountId") not in a_vld.get(
 756                                 "vim_info", {}
 757                             ):
 758                                 target_vim_network_list = [
 759                                     v for _, v in a_vld.get("vim_info").items()
 760                                 ]
 761                                 target_vim_network_name = next(
 762                                     (
 763                                         item.get("vim_network_name", "")
 764                                         for item in target_vim_network_list
 765                                     ),
 766                                     "",
 767                                 )
 768
 769                                 target["ns"]["vld"][a_index].get("vim_info").update(
 770                                     {
 771                                         "vim:{}".format(vnf_params["vimAccountId"]): {
 772                                             "vim_network_name": target_vim_network_name,
 773                                         }
 774                                     }
 775                                 )
 776
 777                                 if vld_params:
 778                                     for param in ("vim-network-name", "vim-network-id"):
 779                                         if vld_params.get(param) and isinstance(
 780                                             vld_params[param], dict
 781                                         ):
 782                                             for vim, vim_net in vld_params[
 783                                                 param
 784                                             ].items():
 785                                                 other_target_vim = "vim:" + vim
 786                                                 populate_dict(
 787                                                     target["ns"]["vld"][a_index].get(
 788                                                         "vim_info"
 789                                                     ),
 790                                                     (
 791                                                         other_target_vim,
 792                                                         param.replace("-", "_"),
 793                                                     ),
 794                                                     vim_net,
 795                                                 )
 796
 797         nslcmop_id = db_nslcmop["_id"]
 798         target = {
 799             "name": db_nsr["name"],
 800             "ns": {"vld": []},
 801             "vnf": [],
 802             "image": deepcopy(db_nsr["image"]),
 803             "flavor": deepcopy(db_nsr["flavor"]),
 804             "action_id": nslcmop_id,
 805             "cloud_init_content": {},
 806         }
 807         for image in target["image"]:
 808             image["vim_info"] = {}
 809         for flavor in target["flavor"]:
 810             flavor["vim_info"] = {}
 811         if db_nsr.get("shared-volumes"):
 812             target["shared-volumes"] = deepcopy(db_nsr["shared-volumes"])
 813             for shared_volumes in target["shared-volumes"]:
 814                 shared_volumes["vim_info"] = {}
 815         if db_nsr.get("affinity-or-anti-affinity-group"):
 816             target["affinity-or-anti-affinity-group"] = deepcopy(
 817                 db_nsr["affinity-or-anti-affinity-group"]
 818             )
 819             for affinity_or_anti_affinity_group in target[
 820                 "affinity-or-anti-affinity-group"
 821             ]:
 822                 affinity_or_anti_affinity_group["vim_info"] = {}
 823
 824         if db_nslcmop.get("lcmOperationType") != "instantiate":
 825             # get parameters of instantiation:
 826             db_nslcmop_instantiate = self.db.get_list(
 827                 "nslcmops",
 828                 {
 829                     "nsInstanceId": db_nslcmop["nsInstanceId"],
 830                     "lcmOperationType": "instantiate",
 831                 },
 832             )[-1]
 833             ns_params = db_nslcmop_instantiate.get("operationParams")
 834         else:
 835             ns_params = db_nslcmop.get("operationParams")
 836         ssh_keys_instantiation = ns_params.get("ssh_keys") or []
 837         ssh_keys_all = ssh_keys_instantiation + (n2vc_key_list or [])
 838
 839         cp2target = {}
 840         for vld_index, vld in enumerate(db_nsr.get("vld")):
 841             target_vim = "vim:{}".format(ns_params["vimAccountId"])
 842             target_vld = {
 843                 "id": vld["id"],
 844                 "name": vld["name"],
 845                 "mgmt-network": vld.get("mgmt-network", False),
 846                 "type": vld.get("type"),
 847                 "vim_info": {
 848                     target_vim: {
 849                         "vim_network_name": vld.get("vim-network-name"),
 850                         "vim_account_id": ns_params["vimAccountId"],
 851                     }
 852                 },
 853             }
 854             # check if this network needs SDN assist
 855             if vld.get("pci-interfaces"):
 856                 db_vim = get_vim_account(ns_params["vimAccountId"])
 857                 if vim_config := db_vim.get("config"):
 858                     if sdnc_id := vim_config.get("sdn-controller"):
 859                         sdn_vld = "nsrs:{}:vld.{}".format(nsr_id, vld["id"])
 860                         target_sdn = "sdn:{}".format(sdnc_id)
 861                         target_vld["vim_info"][target_sdn] = {
 862                             "sdn": True,
 863                             "target_vim": target_vim,
 864                             "vlds": [sdn_vld],
 865                             "type": vld.get("type"),
 866                         }
 867
 868             nsd_vnf_profiles = get_vnf_profiles(nsd)
 869             for nsd_vnf_profile in nsd_vnf_profiles:
 870                 for cp in nsd_vnf_profile["virtual-link-connectivity"]:
 871                     if cp["virtual-link-profile-id"] == vld["id"]:
 872                         cp2target[
 873                             "member_vnf:{}.{}".format(
 874                                 cp["constituent-cpd-id"][0][
 875                                     "constituent-base-element-id"
 876                                 ],
 877                                 cp["constituent-cpd-id"][0]["constituent-cpd-id"],
 878                             )
 879                         ] = "nsrs:{}:vld.{}".format(nsr_id, vld_index)
 880
 881             # check at nsd descriptor, if there is an ip-profile
 882             vld_params = {}
 883             nsd_vlp = find_in_list(
 884                 get_virtual_link_profiles(nsd),
 885                 lambda a_link_profile: a_link_profile["virtual-link-desc-id"]
 886                 == vld["id"],
 887             )
 888             if (
 889                 nsd_vlp
 890                 and nsd_vlp.get("virtual-link-protocol-data")
 891                 and nsd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
 892             ):
 893                 vld_params["ip-profile"] = nsd_vlp["virtual-link-protocol-data"][
 894                     "l3-protocol-data"
 895                 ]
 896
 897             # update vld_params with instantiation params
 898             vld_instantiation_params = find_in_list(
 899                 get_iterable(ns_params, "vld"),
 900                 lambda a_vld: a_vld["name"] in (vld["name"], vld["id"]),
 901             )
 902             if vld_instantiation_params:
 903                 vld_params.update(vld_instantiation_params)
 904             parse_vld_instantiation_params(target_vim, target_vld, vld_params, None)
 905             target["ns"]["vld"].append(target_vld)
 906         # Update the target ns_vld if vnf vim_account is overriden by instantiation params
 907         update_ns_vld_target(target, ns_params)
 908
 909         for vnfr in db_vnfrs.values():
 910             vnfd = find_in_list(
 911                 db_vnfds, lambda db_vnf: db_vnf["id"] == vnfr["vnfd-ref"]
 912             )
 913             vnf_params = find_in_list(
 914                 get_iterable(ns_params, "vnf"),
 915                 lambda a_vnf: a_vnf["member-vnf-index"] == vnfr["member-vnf-index-ref"],
 916             )
 917             target_vnf = deepcopy(vnfr)
 918             target_vim = "vim:{}".format(vnfr["vim-account-id"])
 919             for vld in target_vnf.get("vld", ()):
 920                 # check if connected to a ns.vld, to fill target'
 921                 vnf_cp = find_in_list(
 922                     vnfd.get("int-virtual-link-desc", ()),
 923                     lambda cpd: cpd.get("id") == vld["id"],
 924                 )
 925                 if vnf_cp:
 926                     ns_cp = "member_vnf:{}.{}".format(
 927                         vnfr["member-vnf-index-ref"], vnf_cp["id"]
 928                     )
 929                     if cp2target.get(ns_cp):
 930                         vld["target"] = cp2target[ns_cp]
 931
 932                 vld["vim_info"] = {
 933                     target_vim: {"vim_network_name": vld.get("vim-network-name")}
 934                 }
 935                 # check if this network needs SDN assist
 936                 target_sdn = None
 937                 if vld.get("pci-interfaces"):
 938                     db_vim = get_vim_account(vnfr["vim-account-id"])
 939                     sdnc_id = db_vim["config"].get("sdn-controller")
 940                     if sdnc_id:
 941                         sdn_vld = "vnfrs:{}:vld.{}".format(target_vnf["_id"], vld["id"])
 942                         target_sdn = "sdn:{}".format(sdnc_id)
 943                         vld["vim_info"][target_sdn] = {
 944                             "sdn": True,
 945                             "target_vim": target_vim,
 946                             "vlds": [sdn_vld],
 947                             "type": vld.get("type"),
 948                         }
 949
 950                 # check at vnfd descriptor, if there is an ip-profile
 951                 vld_params = {}
 952                 vnfd_vlp = find_in_list(
 953                     get_virtual_link_profiles(vnfd),
 954                     lambda a_link_profile: a_link_profile["id"] == vld["id"],
 955                 )
 956                 if (
 957                     vnfd_vlp
 958                     and vnfd_vlp.get("virtual-link-protocol-data")
 959                     and vnfd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
 960                 ):
 961                     vld_params["ip-profile"] = vnfd_vlp["virtual-link-protocol-data"][
 962                         "l3-protocol-data"
 963                     ]
 964                 # update vld_params with instantiation params
 965                 if vnf_params:
 966                     vld_instantiation_params = find_in_list(
 967                         get_iterable(vnf_params, "internal-vld"),
 968                         lambda i_vld: i_vld["name"] == vld["id"],
 969                     )
 970                     if vld_instantiation_params:
 971                         vld_params.update(vld_instantiation_params)
 972                 parse_vld_instantiation_params(target_vim, vld, vld_params, target_sdn)
 973
 974             vdur_list = []
 975             for vdur in target_vnf.get("vdur", ()):
 976                 if vdur.get("status") == "DELETING" or vdur.get("pdu-type"):
 977                     continue  # This vdu must not be created
 978                 vdur["vim_info"] = {"vim_account_id": vnfr["vim-account-id"]}
 979
 980                 self.logger.debug("NS > ssh_keys > {}".format(ssh_keys_all))
 981
 982                 if ssh_keys_all:
 983                     vdu_configuration = get_configuration(vnfd, vdur["vdu-id-ref"])
 984                     vnf_configuration = get_configuration(vnfd, vnfd["id"])
 985                     if (
 986                         vdu_configuration
 987                         and vdu_configuration.get("config-access")
 988                         and vdu_configuration.get("config-access").get("ssh-access")
 989                     ):
 990                         vdur["ssh-keys"] = ssh_keys_all
 991                         vdur["ssh-access-required"] = vdu_configuration[
 992                             "config-access"
 993                         ]["ssh-access"]["required"]
 994                     elif (
 995                         vnf_configuration
 996                         and vnf_configuration.get("config-access")
 997                         and vnf_configuration.get("config-access").get("ssh-access")
 998                         and any(iface.get("mgmt-vnf") for iface in vdur["interfaces"])
 999                     ):
1000                         vdur["ssh-keys"] = ssh_keys_all
1001                         vdur["ssh-access-required"] = vnf_configuration[
1002                             "config-access"
1003                         ]["ssh-access"]["required"]
1004                     elif ssh_keys_instantiation and find_in_list(
1005                         vdur["interfaces"], lambda iface: iface.get("mgmt-vnf")
1006                     ):
1007                         vdur["ssh-keys"] = ssh_keys_instantiation
1008
1009                 self.logger.debug("NS > vdur > {}".format(vdur))
1010
1011                 vdud = get_vdu(vnfd, vdur["vdu-id-ref"])
1012                 # cloud-init
1013                 if vdud.get("cloud-init-file"):
1014                     vdur["cloud-init"] = "{}:file:{}".format(
1015                         vnfd["_id"], vdud.get("cloud-init-file")
1016                     )
1017                     # read file and put content at target.cloul_init_content. Avoid ng_ro to use shared package system
1018                     if vdur["cloud-init"] not in target["cloud_init_content"]:
1019                         base_folder = vnfd["_admin"]["storage"]
1020                         if base_folder["pkg-dir"]:
1021                             cloud_init_file = "{}/{}/cloud_init/{}".format(
1022                                 base_folder["folder"],
1023                                 base_folder["pkg-dir"],
1024                                 vdud.get("cloud-init-file"),
1025                             )
1026                         else:
1027                             cloud_init_file = "{}/Scripts/cloud_init/{}".format(
1028                                 base_folder["folder"],
1029                                 vdud.get("cloud-init-file"),
1030                             )
1031                         with self.fs.file_open(cloud_init_file, "r") as ci_file:
1032                             target["cloud_init_content"][
1033                                 vdur["cloud-init"]
1034                             ] = ci_file.read()
1035                 elif vdud.get("cloud-init"):
1036                     vdur["cloud-init"] = "{}:vdu:{}".format(
1037                         vnfd["_id"], get_vdu_index(vnfd, vdur["vdu-id-ref"])
1038                     )
1039                     # put content at target.cloul_init_content. Avoid ng_ro read vnfd descriptor
1040                     target["cloud_init_content"][vdur["cloud-init"]] = vdud[
1041                         "cloud-init"
1042                     ]
1043                 vdur["additionalParams"] = vdur.get("additionalParams") or {}
1044                 deploy_params_vdu = self._format_additional_params(
1045                     vdur.get("additionalParams") or {}
1046                 )
1047                 deploy_params_vdu["OSM"] = get_osm_params(
1048                     vnfr, vdur["vdu-id-ref"], vdur["count-index"]
1049                 )
1050                 vdur["additionalParams"] = deploy_params_vdu
1051
1052                 # flavor
1053                 ns_flavor = target["flavor"][int(vdur["ns-flavor-id"])]
1054                 if target_vim not in ns_flavor["vim_info"]:
1055                     ns_flavor["vim_info"][target_vim] = {}
1056
1057                 # deal with images
1058                 # in case alternative images are provided we must check if they should be applied
1059                 # for the vim_type, modify the vim_type taking into account
1060                 ns_image_id = int(vdur["ns-image-id"])
1061                 if vdur.get("alt-image-ids"):
1062                     db_vim = get_vim_account(vnfr["vim-account-id"])
1063                     vim_type = db_vim["vim_type"]
1064                     for alt_image_id in vdur.get("alt-image-ids"):
1065                         ns_alt_image = target["image"][int(alt_image_id)]
1066                         if vim_type == ns_alt_image.get("vim-type"):
1067                             # must use alternative image
1068                             self.logger.debug(
1069                                 "use alternative image id: {}".format(alt_image_id)
1070                             )
1071                             ns_image_id = alt_image_id
1072                             vdur["ns-image-id"] = ns_image_id
1073                             break
1074                 ns_image = target["image"][int(ns_image_id)]
1075                 if target_vim not in ns_image["vim_info"]:
1076                     ns_image["vim_info"][target_vim] = {}
1077
1078                 # Affinity groups
1079                 if vdur.get("affinity-or-anti-affinity-group-id"):
1080                     for ags_id in vdur["affinity-or-anti-affinity-group-id"]:
1081                         ns_ags = target["affinity-or-anti-affinity-group"][int(ags_id)]
1082                         if target_vim not in ns_ags["vim_info"]:
1083                             ns_ags["vim_info"][target_vim] = {}
1084
1085                 # shared-volumes
1086                 if vdur.get("shared-volumes-id"):
1087                     for sv_id in vdur["shared-volumes-id"]:
1088                         ns_sv = find_in_list(
1089                             target["shared-volumes"], lambda sv: sv_id in sv["id"]
1090                         )
1091                         if ns_sv:
1092                             ns_sv["vim_info"][target_vim] = {}
1093
1094                 vdur["vim_info"] = {target_vim: {}}
1095                 # instantiation parameters
1096                 if vnf_params:
1097                     vdu_instantiation_params = find_in_list(
1098                         get_iterable(vnf_params, "vdu"),
1099                         lambda i_vdu: i_vdu["id"] == vdud["id"],
1100                     )
1101                     if vdu_instantiation_params:
1102                         # Parse the vdu_volumes from the instantiation params
1103                         vdu_volumes = get_volumes_from_instantiation_params(
1104                             vdu_instantiation_params, vdud
1105                         )
1106                         vdur["additionalParams"]["OSM"]["vdu_volumes"] = vdu_volumes
1107                         vdur["additionalParams"]["OSM"][
1108                             "vim_flavor_id"
1109                         ] = vdu_instantiation_params.get("vim-flavor-id")
1110                 vdur_list.append(vdur)
1111             target_vnf["vdur"] = vdur_list
1112             target["vnf"].append(target_vnf)
1113
1114         self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
1115         desc = await self.RO.deploy(nsr_id, target)
1116         self.logger.debug("RO return > {}".format(desc))
1117         action_id = desc["action_id"]
1118         await self._wait_ng_ro(
1119             nsr_id,
1120             action_id,
1121             nslcmop_id,
1122             start_deploy,
1123             timeout_ns_deploy,
1124             stage,
1125             operation="instantiation",
1126         )
1127
1128         # Updating NSR
1129         db_nsr_update = {
1130             "_admin.deployed.RO.operational-status": "running",
1131             "detailed-status": " ".join(stage),
1132         }
1133         # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
1134         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1135         self._write_op_status(nslcmop_id, stage)
1136         self.logger.debug(
1137             logging_text + "ns deployed at RO. RO_id={}".format(action_id)
1138         )
1139         return
1140
1141     async def _wait_ng_ro(
1142         self,
1143         nsr_id,
1144         action_id,
1145         nslcmop_id=None,
1146         start_time=None,
1147         timeout=600,
1148         stage=None,
1149         operation=None,
1150     ):
1151         detailed_status_old = None
1152         db_nsr_update = {}
1153         start_time = start_time or time()
1154         while time() <= start_time + timeout:
1155             desc_status = await self.op_status_map[operation](nsr_id, action_id)
1156             self.logger.debug("Wait NG RO > {}".format(desc_status))
1157             if desc_status["status"] == "FAILED":
1158                 raise NgRoException(desc_status["details"])
1159             elif desc_status["status"] == "BUILD":
1160                 if stage:
1161                     stage[2] = "VIM: ({})".format(desc_status["details"])
1162             elif desc_status["status"] == "DONE":
1163                 if stage:
1164                     stage[2] = "Deployed at VIM"
1165                 break
1166             else:
1167                 assert False, "ROclient.check_ns_status returns unknown {}".format(
1168                     desc_status["status"]
1169                 )
1170             if stage and nslcmop_id and stage[2] != detailed_status_old:
1171                 detailed_status_old = stage[2]
1172                 db_nsr_update["detailed-status"] = " ".join(stage)
1173                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1174                 self._write_op_status(nslcmop_id, stage)
1175             await asyncio.sleep(15)
1176         else:  # timeout_ns_deploy
1177             raise NgRoException("Timeout waiting ns to deploy")
1178
1179     async def _terminate_ng_ro(
1180         self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
1181     ):
1182         db_nsr_update = {}
1183         failed_detail = []
1184         action_id = None
1185         start_deploy = time()
1186         try:
1187             target = {
1188                 "ns": {"vld": []},
1189                 "vnf": [],
1190                 "image": [],
1191                 "flavor": [],
1192                 "action_id": nslcmop_id,
1193             }
1194             desc = await self.RO.deploy(nsr_id, target)
1195             action_id = desc["action_id"]
1196             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETING"
1197             self.logger.debug(
1198                 logging_text
1199                 + "ns terminate action at RO. action_id={}".format(action_id)
1200             )
1201
1202             # wait until done
1203             delete_timeout = 20 * 60  # 20 minutes
1204             await self._wait_ng_ro(
1205                 nsr_id,
1206                 action_id,
1207                 nslcmop_id,
1208                 start_deploy,
1209                 delete_timeout,
1210                 stage,
1211                 operation="termination",
1212             )
1213             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1214             # delete all nsr
1215             await self.RO.delete(nsr_id)
1216         except NgRoException as e:
1217             if e.http_code == 404:  # not found
1218                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1219                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1220                 self.logger.debug(
1221                     logging_text + "RO_action_id={} already deleted".format(action_id)
1222                 )
1223             elif e.http_code == 409:  # conflict
1224                 failed_detail.append("delete conflict: {}".format(e))
1225                 self.logger.debug(
1226                     logging_text
1227                     + "RO_action_id={} delete conflict: {}".format(action_id, e)
1228                 )
1229             else:
1230                 failed_detail.append("delete error: {}".format(e))
1231                 self.logger.error(
1232                     logging_text
1233                     + "RO_action_id={} delete error: {}".format(action_id, e)
1234                 )
1235         except Exception as e:
1236             failed_detail.append("delete error: {}".format(e))
1237             self.logger.error(
1238                 logging_text + "RO_action_id={} delete error: {}".format(action_id, e)
1239             )
1240
1241         if failed_detail:
1242             stage[2] = "Error deleting from VIM"
1243         else:
1244             stage[2] = "Deleted from VIM"
1245         db_nsr_update["detailed-status"] = " ".join(stage)
1246         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1247         self._write_op_status(nslcmop_id, stage)
1248
1249         if failed_detail:
1250             raise LcmException("; ".join(failed_detail))
1251         return
1252
1253     async def instantiate_RO(
1254         self,
1255         logging_text,
1256         nsr_id,
1257         nsd,
1258         db_nsr,
1259         db_nslcmop,
1260         db_vnfrs,
1261         db_vnfds,
1262         n2vc_key_list,
1263         stage,
1264     ):
1265         """
1266         Instantiate at RO
1267         :param logging_text: preffix text to use at logging
1268         :param nsr_id: nsr identity
1269         :param nsd: database content of ns descriptor
1270         :param db_nsr: database content of ns record
1271         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
1272         :param db_vnfrs:
1273         :param db_vnfds: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
1274         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
1275         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
1276         :return: None or exception
1277         """
1278         try:
1279             start_deploy = time()
1280             ns_params = db_nslcmop.get("operationParams")
1281             if ns_params and ns_params.get("timeout_ns_deploy"):
1282                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1283             else:
1284                 timeout_ns_deploy = self.timeout.ns_deploy
1285
1286             # Check for and optionally request placement optimization. Database will be updated if placement activated
1287             stage[2] = "Waiting for Placement."
1288             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
1289                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
1290                 for vnfr in db_vnfrs.values():
1291                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
1292                         break
1293                 else:
1294                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
1295
1296             return await self._instantiate_ng_ro(
1297                 logging_text,
1298                 nsr_id,
1299                 nsd,
1300                 db_nsr,
1301                 db_nslcmop,
1302                 db_vnfrs,
1303                 db_vnfds,
1304                 n2vc_key_list,
1305                 stage,
1306                 start_deploy,
1307                 timeout_ns_deploy,
1308             )
1309         except Exception as e:
1310             stage[2] = "ERROR deploying at VIM"
1311             self.set_vnfr_at_error(db_vnfrs, str(e))
1312             self.logger.error(
1313                 "Error deploying at VIM {}".format(e),
1314                 exc_info=not isinstance(
1315                     e,
1316                     (
1317                         ROclient.ROClientException,
1318                         LcmException,
1319                         DbException,
1320                         NgRoException,
1321                     ),
1322                 ),
1323             )
1324             raise
1325
1326     async def wait_kdu_up(self, logging_text, nsr_id, vnfr_id, kdu_name):
1327         """
1328         Wait for kdu to be up, get ip address
1329         :param logging_text: prefix use for logging
1330         :param nsr_id:
1331         :param vnfr_id:
1332         :param kdu_name:
1333         :return: IP address, K8s services
1334         """
1335
1336         # self.logger.debug(logging_text + "Starting wait_kdu_up")
1337         nb_tries = 0
1338
1339         while nb_tries < 360:
1340             db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1341             kdur = next(
1342                 (
1343                     x
1344                     for x in get_iterable(db_vnfr, "kdur")
1345                     if x.get("kdu-name") == kdu_name
1346                 ),
1347                 None,
1348             )
1349             if not kdur:
1350                 raise LcmException(
1351                     "Not found vnfr_id={}, kdu_name={}".format(vnfr_id, kdu_name)
1352                 )
1353             if kdur.get("status"):
1354                 if kdur["status"] in ("READY", "ENABLED"):
1355                     return kdur.get("ip-address"), kdur.get("services")
1356                 else:
1357                     raise LcmException(
1358                         "target KDU={} is in error state".format(kdu_name)
1359                     )
1360
1361             await asyncio.sleep(10)
1362             nb_tries += 1
1363         raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
1364
1365     async def wait_vm_up_insert_key_ro(
1366         self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None
1367     ):
1368         """
1369         Wait for ip addres at RO, and optionally, insert public key in virtual machine
1370         :param logging_text: prefix use for logging
1371         :param nsr_id:
1372         :param vnfr_id:
1373         :param vdu_id:
1374         :param vdu_index:
1375         :param pub_key: public ssh key to inject, None to skip
1376         :param user: user to apply the public ssh key
1377         :return: IP address
1378         """
1379
1380         self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1381         ip_address = None
1382         target_vdu_id = None
1383         ro_retries = 0
1384
1385         while True:
1386             ro_retries += 1
1387             if ro_retries >= 360:  # 1 hour
1388                 raise LcmException(
1389                     "Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id)
1390                 )
1391
1392             await asyncio.sleep(10)
1393
1394             # get ip address
1395             if not target_vdu_id:
1396                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1397
1398                 if not vdu_id:  # for the VNF case
1399                     if db_vnfr.get("status") == "ERROR":
1400                         raise LcmException(
1401                             "Cannot inject ssh-key because target VNF is in error state"
1402                         )
1403                     ip_address = db_vnfr.get("ip-address")
1404                     if not ip_address:
1405                         continue
1406                     vdur = next(
1407                         (
1408                             x
1409                             for x in get_iterable(db_vnfr, "vdur")
1410                             if x.get("ip-address") == ip_address
1411                         ),
1412                         None,
1413                     )
1414                 else:  # VDU case
1415                     vdur = next(
1416                         (
1417                             x
1418                             for x in get_iterable(db_vnfr, "vdur")
1419                             if x.get("vdu-id-ref") == vdu_id
1420                             and x.get("count-index") == vdu_index
1421                         ),
1422                         None,
1423                     )
1424
1425                 if (
1426                     not vdur and len(db_vnfr.get("vdur", ())) == 1
1427                 ):  # If only one, this should be the target vdu
1428                     vdur = db_vnfr["vdur"][0]
1429                 if not vdur:
1430                     raise LcmException(
1431                         "Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(
1432                             vnfr_id, vdu_id, vdu_index
1433                         )
1434                     )
1435                 # New generation RO stores information at "vim_info"
1436                 ng_ro_status = None
1437                 target_vim = None
1438                 if vdur.get("vim_info"):
1439                     target_vim = next(
1440                         t for t in vdur["vim_info"]
1441                     )  # there should be only one key
1442                     ng_ro_status = vdur["vim_info"][target_vim].get("vim_status")
1443                 if (
1444                     vdur.get("pdu-type")
1445                     or vdur.get("status") == "ACTIVE"
1446                     or ng_ro_status == "ACTIVE"
1447                 ):
1448                     ip_address = vdur.get("ip-address")
1449                     if not ip_address:
1450                         continue
1451                     target_vdu_id = vdur["vdu-id-ref"]
1452                 elif vdur.get("status") == "ERROR" or ng_ro_status == "ERROR":
1453                     raise LcmException(
1454                         "Cannot inject ssh-key because target VM is in error state"
1455                     )
1456
1457             if not target_vdu_id:
1458                 continue
1459
1460             # inject public key into machine
1461             if pub_key and user:
1462                 self.logger.debug(logging_text + "Inserting RO key")
1463                 self.logger.debug("SSH > PubKey > {}".format(pub_key))
1464                 if vdur.get("pdu-type"):
1465                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1466                     return ip_address
1467                 try:
1468                     target = {
1469                         "action": {
1470                             "action": "inject_ssh_key",
1471                             "key": pub_key,
1472                             "user": user,
1473                         },
1474                         "vnf": [{"_id": vnfr_id, "vdur": [{"id": vdur["id"]}]}],
1475                     }
1476                     desc = await self.RO.deploy(nsr_id, target)
1477                     action_id = desc["action_id"]
1478                     await self._wait_ng_ro(
1479                         nsr_id, action_id, timeout=600, operation="instantiation"
1480                     )
1481                     break
1482                 except NgRoException as e:
1483                     raise LcmException(
1484                         "Reaching max tries injecting key. Error: {}".format(e)
1485                     )
1486             else:
1487                 break
1488
1489         return ip_address
1490
1491     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1492         """
1493         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1494         """
1495         my_vca = vca_deployed_list[vca_index]
1496         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1497             # vdu or kdu: no dependencies
1498             return
1499         timeout = 300
1500         while timeout >= 0:
1501             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1502             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1503             configuration_status_list = db_nsr["configurationStatus"]
1504             for index, vca_deployed in enumerate(configuration_status_list):
1505                 if index == vca_index:
1506                     # myself
1507                     continue
1508                 if not my_vca.get("member-vnf-index") or (
1509                     vca_deployed.get("member-vnf-index")
1510                     == my_vca.get("member-vnf-index")
1511                 ):
1512                     internal_status = configuration_status_list[index].get("status")
1513                     if internal_status == "READY":
1514                         continue
1515                     elif internal_status == "BROKEN":
1516                         raise LcmException(
1517                             "Configuration aborted because dependent charm/s has failed"
1518                         )
1519                     else:
1520                         break
1521             else:
1522                 # no dependencies, return
1523                 return
1524             await asyncio.sleep(10)
1525             timeout -= 1
1526
1527         raise LcmException("Configuration aborted because dependent charm/s timeout")
1528
1529     def get_vca_id(self, db_vnfr: dict, db_nsr: dict):
1530         vca_id = None
1531         if db_vnfr:
1532             vca_id = deep_get(db_vnfr, ("vca-id",))
1533         elif db_nsr:
1534             vim_account_id = deep_get(db_nsr, ("instantiate_params", "vimAccountId"))
1535             vca_id = VimAccountDB.get_vim_account_with_id(vim_account_id).get("vca")
1536         return vca_id
1537
1538     async def instantiate_N2VC(
1539         self,
1540         logging_text,
1541         vca_index,
1542         nsi_id,
1543         db_nsr,
1544         db_vnfr,
1545         vdu_id,
1546         kdu_name,
1547         vdu_index,
1548         kdu_index,
1549         config_descriptor,
1550         deploy_params,
1551         base_folder,
1552         nslcmop_id,
1553         stage,
1554         vca_type,
1555         vca_name,
1556         ee_config_descriptor,
1557     ):
1558         nsr_id = db_nsr["_id"]
1559         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1560         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1561         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1562         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
1563         db_dict = {
1564             "collection": "nsrs",
1565             "filter": {"_id": nsr_id},
1566             "path": db_update_entry,
1567         }
1568         step = ""
1569         try:
1570             element_type = "NS"
1571             element_under_configuration = nsr_id
1572
1573             vnfr_id = None
1574             if db_vnfr:
1575                 vnfr_id = db_vnfr["_id"]
1576                 osm_config["osm"]["vnf_id"] = vnfr_id
1577
1578             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
1579
1580             if vca_type == "native_charm":
1581                 index_number = 0
1582             else:
1583                 index_number = vdu_index or 0
1584
1585             if vnfr_id:
1586                 element_type = "VNF"
1587                 element_under_configuration = vnfr_id
1588                 namespace += ".{}-{}".format(vnfr_id, index_number)
1589                 if vdu_id:
1590                     namespace += ".{}-{}".format(vdu_id, index_number)
1591                     element_type = "VDU"
1592                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
1593                     osm_config["osm"]["vdu_id"] = vdu_id
1594                 elif kdu_name:
1595                     namespace += ".{}".format(kdu_name)
1596                     element_type = "KDU"
1597                     element_under_configuration = kdu_name
1598                     osm_config["osm"]["kdu_name"] = kdu_name
1599
1600             # Get artifact path
1601             if base_folder["pkg-dir"]:
1602                 artifact_path = "{}/{}/{}/{}".format(
1603                     base_folder["folder"],
1604                     base_folder["pkg-dir"],
1605                     "charms"
1606                     if vca_type
1607                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1608                     else "helm-charts",
1609                     vca_name,
1610                 )
1611             else:
1612                 artifact_path = "{}/Scripts/{}/{}/".format(
1613                     base_folder["folder"],
1614                     "charms"
1615                     if vca_type
1616                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1617                     else "helm-charts",
1618                     vca_name,
1619                 )
1620
1621             self.logger.debug("Artifact path > {}".format(artifact_path))
1622
1623             # get initial_config_primitive_list that applies to this element
1624             initial_config_primitive_list = config_descriptor.get(
1625                 "initial-config-primitive"
1626             )
1627
1628             self.logger.debug(
1629                 "Initial config primitive list > {}".format(
1630                     initial_config_primitive_list
1631                 )
1632             )
1633
1634             # add config if not present for NS charm
1635             ee_descriptor_id = ee_config_descriptor.get("id")
1636             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
1637             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
1638                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
1639             )
1640
1641             self.logger.debug(
1642                 "Initial config primitive list #2 > {}".format(
1643                     initial_config_primitive_list
1644                 )
1645             )
1646             # n2vc_redesign STEP 3.1
1647             # find old ee_id if exists
1648             ee_id = vca_deployed.get("ee_id")
1649
1650             vca_id = self.get_vca_id(db_vnfr, db_nsr)
1651             # create or register execution environment in VCA
1652             if vca_type in ("lxc_proxy_charm", "k8s_proxy_charm", "helm-v3"):
1653                 self._write_configuration_status(
1654                     nsr_id=nsr_id,
1655                     vca_index=vca_index,
1656                     status="CREATING",
1657                     element_under_configuration=element_under_configuration,
1658                     element_type=element_type,
1659                 )
1660
1661                 step = "create execution environment"
1662                 self.logger.debug(logging_text + step)
1663
1664                 ee_id = None
1665                 credentials = None
1666                 if vca_type == "k8s_proxy_charm":
1667                     ee_id = await self.vca_map[vca_type].install_k8s_proxy_charm(
1668                         charm_name=artifact_path[artifact_path.rfind("/") + 1 :],
1669                         namespace=namespace,
1670                         artifact_path=artifact_path,
1671                         db_dict=db_dict,
1672                         vca_id=vca_id,
1673                     )
1674                 elif vca_type == "helm-v3":
1675                     ee_id, credentials = await self.vca_map[
1676                         vca_type
1677                     ].create_execution_environment(
1678                         namespace=nsr_id,
1679                         reuse_ee_id=ee_id,
1680                         db_dict=db_dict,
1681                         config=osm_config,
1682                         artifact_path=artifact_path,
1683                         chart_model=vca_name,
1684                         vca_type=vca_type,
1685                     )
1686                 else:
1687                     ee_id, credentials = await self.vca_map[
1688                         vca_type
1689                     ].create_execution_environment(
1690                         namespace=namespace,
1691                         reuse_ee_id=ee_id,
1692                         db_dict=db_dict,
1693                         vca_id=vca_id,
1694                     )
1695
1696             elif vca_type == "native_charm":
1697                 step = "Waiting to VM being up and getting IP address"
1698                 self.logger.debug(logging_text + step)
1699                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
1700                     logging_text,
1701                     nsr_id,
1702                     vnfr_id,
1703                     vdu_id,
1704                     vdu_index,
1705                     user=None,
1706                     pub_key=None,
1707                 )
1708                 credentials = {"hostname": rw_mgmt_ip}
1709                 # get username
1710                 username = deep_get(
1711                     config_descriptor, ("config-access", "ssh-access", "default-user")
1712                 )
1713                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1714                 #  merged. Meanwhile let's get username from initial-config-primitive
1715                 if not username and initial_config_primitive_list:
1716                     for config_primitive in initial_config_primitive_list:
1717                         for param in config_primitive.get("parameter", ()):
1718                             if param["name"] == "ssh-username":
1719                                 username = param["value"]
1720                                 break
1721                 if not username:
1722                     raise LcmException(
1723                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
1724                         "'config-access.ssh-access.default-user'"
1725                     )
1726                 credentials["username"] = username
1727                 # n2vc_redesign STEP 3.2
1728
1729                 self._write_configuration_status(
1730                     nsr_id=nsr_id,
1731                     vca_index=vca_index,
1732                     status="REGISTERING",
1733                     element_under_configuration=element_under_configuration,
1734                     element_type=element_type,
1735                 )
1736
1737                 step = "register execution environment {}".format(credentials)
1738                 self.logger.debug(logging_text + step)
1739                 ee_id = await self.vca_map[vca_type].register_execution_environment(
1740                     credentials=credentials,
1741                     namespace=namespace,
1742                     db_dict=db_dict,
1743                     vca_id=vca_id,
1744                 )
1745
1746             # for compatibility with MON/POL modules, the need model and application name at database
1747             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
1748             ee_id_parts = ee_id.split(".")
1749             db_nsr_update = {db_update_entry + "ee_id": ee_id}
1750             if len(ee_id_parts) >= 2:
1751                 model_name = ee_id_parts[0]
1752                 application_name = ee_id_parts[1]
1753                 db_nsr_update[db_update_entry + "model"] = model_name
1754                 db_nsr_update[db_update_entry + "application"] = application_name
1755
1756             # n2vc_redesign STEP 3.3
1757             step = "Install configuration Software"
1758
1759             self._write_configuration_status(
1760                 nsr_id=nsr_id,
1761                 vca_index=vca_index,
1762                 status="INSTALLING SW",
1763                 element_under_configuration=element_under_configuration,
1764                 element_type=element_type,
1765                 other_update=db_nsr_update,
1766             )
1767
1768             # TODO check if already done
1769             self.logger.debug(logging_text + step)
1770             config = None
1771             if vca_type == "native_charm":
1772                 config_primitive = next(
1773                     (p for p in initial_config_primitive_list if p["name"] == "config"),
1774                     None,
1775                 )
1776                 if config_primitive:
1777                     config = self._map_primitive_params(
1778                         config_primitive, {}, deploy_params
1779                     )
1780             num_units = 1
1781             if vca_type == "lxc_proxy_charm":
1782                 if element_type == "NS":
1783                     num_units = db_nsr.get("config-units") or 1
1784                 elif element_type == "VNF":
1785                     num_units = db_vnfr.get("config-units") or 1
1786                 elif element_type == "VDU":
1787                     for v in db_vnfr["vdur"]:
1788                         if vdu_id == v["vdu-id-ref"]:
1789                             num_units = v.get("config-units") or 1
1790                             break
1791             if vca_type != "k8s_proxy_charm":
1792                 await self.vca_map[vca_type].install_configuration_sw(
1793                     ee_id=ee_id,
1794                     artifact_path=artifact_path,
1795                     db_dict=db_dict,
1796                     config=config,
1797                     num_units=num_units,
1798                     vca_id=vca_id,
1799                     vca_type=vca_type,
1800                 )
1801
1802             # write in db flag of configuration_sw already installed
1803             self.update_db_2(
1804                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
1805             )
1806
1807             # add relations for this VCA (wait for other peers related with this VCA)
1808             is_relation_added = await self._add_vca_relations(
1809                 logging_text=logging_text,
1810                 nsr_id=nsr_id,
1811                 vca_type=vca_type,
1812                 vca_index=vca_index,
1813             )
1814
1815             if not is_relation_added:
1816                 raise LcmException("Relations could not be added to VCA.")
1817
1818             # if SSH access is required, then get execution environment SSH public
1819             # if native charm we have waited already to VM be UP
1820             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm-v3"):
1821                 pub_key = None
1822                 user = None
1823                 # self.logger.debug("get ssh key block")
1824                 if deep_get(
1825                     config_descriptor, ("config-access", "ssh-access", "required")
1826                 ):
1827                     # self.logger.debug("ssh key needed")
1828                     # Needed to inject a ssh key
1829                     user = deep_get(
1830                         config_descriptor,
1831                         ("config-access", "ssh-access", "default-user"),
1832                     )
1833                     step = "Install configuration Software, getting public ssh key"
1834                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
1835                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
1836                     )
1837
1838                     step = "Insert public key into VM user={} ssh_key={}".format(
1839                         user, pub_key
1840                     )
1841                 else:
1842                     # self.logger.debug("no need to get ssh key")
1843                     step = "Waiting to VM being up and getting IP address"
1844                 self.logger.debug(logging_text + step)
1845
1846                 # default rw_mgmt_ip to None, avoiding the non definition of the variable
1847                 rw_mgmt_ip = None
1848
1849                 # n2vc_redesign STEP 5.1
1850                 # wait for RO (ip-address) Insert pub_key into VM
1851                 if vnfr_id:
1852                     if kdu_name:
1853                         rw_mgmt_ip, services = await self.wait_kdu_up(
1854                             logging_text, nsr_id, vnfr_id, kdu_name
1855                         )
1856                         vnfd = self.db.get_one(
1857                             "vnfds_revisions",
1858                             {"_id": f'{db_vnfr["vnfd-id"]}:{db_vnfr["revision"]}'},
1859                         )
1860                         kdu = get_kdu(vnfd, kdu_name)
1861                         kdu_services = [
1862                             service["name"] for service in get_kdu_services(kdu)
1863                         ]
1864                         exposed_services = []
1865                         for service in services:
1866                             if any(s in service["name"] for s in kdu_services):
1867                                 exposed_services.append(service)
1868                         await self.vca_map[vca_type].exec_primitive(
1869                             ee_id=ee_id,
1870                             primitive_name="config",
1871                             params_dict={
1872                                 "osm-config": json.dumps(
1873                                     OsmConfigBuilder(
1874                                         k8s={"services": exposed_services}
1875                                     ).build()
1876                                 )
1877                             },
1878                             vca_id=vca_id,
1879                         )
1880
1881                     # This verification is needed in order to avoid trying to add a public key
1882                     # to a VM, when the VNF is a KNF (in the edge case where the user creates a VCA
1883                     # for a KNF and not for its KDUs, the previous verification gives False, and the code
1884                     # jumps to this block, meaning that there is the need to verify if the VNF is actually a VNF
1885                     # or it is a KNF)
1886                     elif db_vnfr.get("vdur"):
1887                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
1888                             logging_text,
1889                             nsr_id,
1890                             vnfr_id,
1891                             vdu_id,
1892                             vdu_index,
1893                             user=user,
1894                             pub_key=pub_key,
1895                         )
1896
1897                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
1898
1899             # store rw_mgmt_ip in deploy params for later replacement
1900             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
1901
1902             # n2vc_redesign STEP 6  Execute initial config primitive
1903             step = "execute initial config primitive"
1904
1905             # wait for dependent primitives execution (NS -> VNF -> VDU)
1906             if initial_config_primitive_list:
1907                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
1908
1909             # stage, in function of element type: vdu, kdu, vnf or ns
1910             my_vca = vca_deployed_list[vca_index]
1911             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1912                 # VDU or KDU
1913                 stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
1914             elif my_vca.get("member-vnf-index"):
1915                 # VNF
1916                 stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
1917             else:
1918                 # NS
1919                 stage[0] = "Stage 5/5: running Day-1 primitives for NS."
1920
1921             self._write_configuration_status(
1922                 nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
1923             )
1924
1925             self._write_op_status(op_id=nslcmop_id, stage=stage)
1926
1927             check_if_terminated_needed = True
1928             for initial_config_primitive in initial_config_primitive_list:
1929                 # adding information on the vca_deployed if it is a NS execution environment
1930                 if not vca_deployed["member-vnf-index"]:
1931                     deploy_params["ns_config_info"] = json.dumps(
1932                         self._get_ns_config_info(nsr_id)
1933                     )
1934                 # TODO check if already done
1935                 primitive_params_ = self._map_primitive_params(
1936                     initial_config_primitive, {}, deploy_params
1937                 )
1938
1939                 step = "execute primitive '{}' params '{}'".format(
1940                     initial_config_primitive["name"], primitive_params_
1941                 )
1942                 self.logger.debug(logging_text + step)
1943                 await self.vca_map[vca_type].exec_primitive(
1944                     ee_id=ee_id,
1945                     primitive_name=initial_config_primitive["name"],
1946                     params_dict=primitive_params_,
1947                     db_dict=db_dict,
1948                     vca_id=vca_id,
1949                     vca_type=vca_type,
1950                 )
1951                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
1952                 if check_if_terminated_needed:
1953                     if config_descriptor.get("terminate-config-primitive"):
1954                         self.update_db_2(
1955                             "nsrs", nsr_id, {db_update_entry + "needed_terminate": True}
1956                         )
1957                     check_if_terminated_needed = False
1958
1959                 # TODO register in database that primitive is done
1960
1961             # STEP 7 Configure metrics
1962             if vca_type == "helm-v3":
1963                 # TODO: review for those cases where the helm chart is a reference and
1964                 # is not part of the NF package
1965                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
1966                     ee_id=ee_id,
1967                     artifact_path=artifact_path,
1968                     ee_config_descriptor=ee_config_descriptor,
1969                     vnfr_id=vnfr_id,
1970                     nsr_id=nsr_id,
1971                     target_ip=rw_mgmt_ip,
1972                     element_type=element_type,
1973                     vnf_member_index=db_vnfr.get("member-vnf-index-ref", ""),
1974                     vdu_id=vdu_id,
1975                     vdu_index=vdu_index,
1976                     kdu_name=kdu_name,
1977                     kdu_index=kdu_index,
1978                 )
1979                 if prometheus_jobs:
1980                     self.update_db_2(
1981                         "nsrs",
1982                         nsr_id,
1983                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
1984                     )
1985
1986                     for job in prometheus_jobs:
1987                         self.db.set_one(
1988                             "prometheus_jobs",
1989                             {"job_name": job["job_name"]},
1990                             job,
1991                             upsert=True,
1992                             fail_on_empty=False,
1993                         )
1994
1995             step = "instantiated at VCA"
1996             self.logger.debug(logging_text + step)
1997
1998             self._write_configuration_status(
1999                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
2000             )
2001
2002         except Exception as e:  # TODO not use Exception but N2VC exception
2003             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
2004             if not isinstance(
2005                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
2006             ):
2007                 self.logger.error(
2008                     "Exception while {} : {}".format(step, e), exc_info=True
2009                 )
2010             self._write_configuration_status(
2011                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
2012             )
2013             raise LcmException("{}. {}".format(step, e)) from e
2014
2015     def _write_ns_status(
2016         self,
2017         nsr_id: str,
2018         ns_state: str,
2019         current_operation: str,
2020         current_operation_id: str,
2021         error_description: str = None,
2022         error_detail: str = None,
2023         other_update: dict = None,
2024     ):
2025         """
2026         Update db_nsr fields.
2027         :param nsr_id:
2028         :param ns_state:
2029         :param current_operation:
2030         :param current_operation_id:
2031         :param error_description:
2032         :param error_detail:
2033         :param other_update: Other required changes at database if provided, will be cleared
2034         :return:
2035         """
2036         try:
2037             db_dict = other_update or {}
2038             db_dict[
2039                 "_admin.nslcmop"
2040             ] = current_operation_id  # for backward compatibility
2041             db_dict["_admin.current-operation"] = current_operation_id
2042             db_dict["_admin.operation-type"] = (
2043                 current_operation if current_operation != "IDLE" else None
2044             )
2045             db_dict["currentOperation"] = current_operation
2046             db_dict["currentOperationID"] = current_operation_id
2047             db_dict["errorDescription"] = error_description
2048             db_dict["errorDetail"] = error_detail
2049
2050             if ns_state:
2051                 db_dict["nsState"] = ns_state
2052             self.update_db_2("nsrs", nsr_id, db_dict)
2053         except DbException as e:
2054             self.logger.warn("Error writing NS status, ns={}: {}".format(nsr_id, e))
2055
2056     def _write_op_status(
2057         self,
2058         op_id: str,
2059         stage: list = None,
2060         error_message: str = None,
2061         queuePosition: int = 0,
2062         operation_state: str = None,
2063         other_update: dict = None,
2064     ):
2065         try:
2066             db_dict = other_update or {}
2067             db_dict["queuePosition"] = queuePosition
2068             if isinstance(stage, list):
2069                 db_dict["stage"] = stage[0]
2070                 db_dict["detailed-status"] = " ".join(stage)
2071             elif stage is not None:
2072                 db_dict["stage"] = str(stage)
2073
2074             if error_message is not None:
2075                 db_dict["errorMessage"] = error_message
2076             if operation_state is not None:
2077                 db_dict["operationState"] = operation_state
2078                 db_dict["statusEnteredTime"] = time()
2079             self.update_db_2("nslcmops", op_id, db_dict)
2080         except DbException as e:
2081             self.logger.warn(
2082                 "Error writing OPERATION status for op_id: {} -> {}".format(op_id, e)
2083             )
2084
2085     def _write_all_config_status(self, db_nsr: dict, status: str):
2086         try:
2087             nsr_id = db_nsr["_id"]
2088             # configurationStatus
2089             config_status = db_nsr.get("configurationStatus")
2090             if config_status:
2091                 db_nsr_update = {
2092                     "configurationStatus.{}.status".format(index): status
2093                     for index, v in enumerate(config_status)
2094                     if v
2095                 }
2096                 # update status
2097                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2098
2099         except DbException as e:
2100             self.logger.warn(
2101                 "Error writing all configuration status, ns={}: {}".format(nsr_id, e)
2102             )
2103
2104     def _write_configuration_status(
2105         self,
2106         nsr_id: str,
2107         vca_index: int,
2108         status: str = None,
2109         element_under_configuration: str = None,
2110         element_type: str = None,
2111         other_update: dict = None,
2112     ):
2113         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
2114         #                   .format(vca_index, status))
2115
2116         try:
2117             db_path = "configurationStatus.{}.".format(vca_index)
2118             db_dict = other_update or {}
2119             if status:
2120                 db_dict[db_path + "status"] = status
2121             if element_under_configuration:
2122                 db_dict[
2123                     db_path + "elementUnderConfiguration"
2124                 ] = element_under_configuration
2125             if element_type:
2126                 db_dict[db_path + "elementType"] = element_type
2127             self.update_db_2("nsrs", nsr_id, db_dict)
2128         except DbException as e:
2129             self.logger.warn(
2130                 "Error writing configuration status={}, ns={}, vca_index={}: {}".format(
2131                     status, nsr_id, vca_index, e
2132                 )
2133             )
2134
2135     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
2136         """
2137         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
2138         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
2139         Database is used because the result can be obtained from a different LCM worker in case of HA.
2140         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
2141         :param db_nslcmop: database content of nslcmop
2142         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
2143         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
2144             computed 'vim-account-id'
2145         """
2146         modified = False
2147         nslcmop_id = db_nslcmop["_id"]
2148         placement_engine = deep_get(db_nslcmop, ("operationParams", "placement-engine"))
2149         if placement_engine == "PLA":
2150             self.logger.debug(
2151                 logging_text + "Invoke and wait for placement optimization"
2152             )
2153             await self.msg.aiowrite("pla", "get_placement", {"nslcmopId": nslcmop_id})
2154             db_poll_interval = 5
2155             wait = db_poll_interval * 10
2156             pla_result = None
2157             while not pla_result and wait >= 0:
2158                 await asyncio.sleep(db_poll_interval)
2159                 wait -= db_poll_interval
2160                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2161                 pla_result = deep_get(db_nslcmop, ("_admin", "pla"))
2162
2163             if not pla_result:
2164                 raise LcmException(
2165                     "Placement timeout for nslcmopId={}".format(nslcmop_id)
2166                 )
2167
2168             for pla_vnf in pla_result["vnf"]:
2169                 vnfr = db_vnfrs.get(pla_vnf["member-vnf-index"])
2170                 if not pla_vnf.get("vimAccountId") or not vnfr:
2171                     continue
2172                 modified = True
2173                 self.db.set_one(
2174                     "vnfrs",
2175                     {"_id": vnfr["_id"]},
2176                     {"vim-account-id": pla_vnf["vimAccountId"]},
2177                 )
2178                 # Modifies db_vnfrs
2179                 vnfr["vim-account-id"] = pla_vnf["vimAccountId"]
2180         return modified
2181
2182     def _gather_vnfr_healing_alerts(self, vnfr, vnfd):
2183         alerts = []
2184         nsr_id = vnfr["nsr-id-ref"]
2185         df = vnfd.get("df", [{}])[0]
2186         # Checking for auto-healing configuration
2187         if "healing-aspect" in df:
2188             healing_aspects = df["healing-aspect"]
2189             for healing in healing_aspects:
2190                 for healing_policy in healing.get("healing-policy", ()):
2191                     vdu_id = healing_policy["vdu-id"]
2192                     vdur = next(
2193                         (vdur for vdur in vnfr["vdur"] if vdu_id == vdur["vdu-id-ref"]),
2194                         {},
2195                     )
2196                     if not vdur:
2197                         continue
2198                     metric_name = "vm_status"
2199                     vdu_name = vdur.get("name")
2200                     vnf_member_index = vnfr["member-vnf-index-ref"]
2201                     uuid = str(uuid4())
2202                     name = f"healing_{uuid}"
2203                     action = healing_policy
2204                     # action_on_recovery = healing.get("action-on-recovery")
2205                     # cooldown_time = healing.get("cooldown-time")
2206                     # day1 = healing.get("day1")
2207                     alert = {
2208                         "uuid": uuid,
2209                         "name": name,
2210                         "metric": metric_name,
2211                         "tags": {
2212                             "ns_id": nsr_id,
2213                             "vnf_member_index": vnf_member_index,
2214                             "vdu_name": vdu_name,
2215                         },
2216                         "alarm_status": "ok",
2217                         "action_type": "healing",
2218                         "action": action,
2219                     }
2220                     alerts.append(alert)
2221         return alerts
2222
2223     def _gather_vnfr_scaling_alerts(self, vnfr, vnfd):
2224         alerts = []
2225         nsr_id = vnfr["nsr-id-ref"]
2226         df = vnfd.get("df", [{}])[0]
2227         # Checking for auto-scaling configuration
2228         if "scaling-aspect" in df:
2229             scaling_aspects = df["scaling-aspect"]
2230             all_vnfd_monitoring_params = {}
2231             for ivld in vnfd.get("int-virtual-link-desc", ()):
2232                 for mp in ivld.get("monitoring-parameters", ()):
2233                     all_vnfd_monitoring_params[mp.get("id")] = mp
2234             for vdu in vnfd.get("vdu", ()):
2235                 for mp in vdu.get("monitoring-parameter", ()):
2236                     all_vnfd_monitoring_params[mp.get("id")] = mp
2237             for df in vnfd.get("df", ()):
2238                 for mp in df.get("monitoring-parameter", ()):
2239                     all_vnfd_monitoring_params[mp.get("id")] = mp
2240             for scaling_aspect in scaling_aspects:
2241                 scaling_group_name = scaling_aspect.get("name", "")
2242                 # Get monitored VDUs
2243                 all_monitored_vdus = set()
2244                 for delta in scaling_aspect.get("aspect-delta-details", {}).get(
2245                     "deltas", ()
2246                 ):
2247                     for vdu_delta in delta.get("vdu-delta", ()):
2248                         all_monitored_vdus.add(vdu_delta.get("id"))
2249                 monitored_vdurs = list(
2250                     filter(
2251                         lambda vdur: vdur["vdu-id-ref"] in all_monitored_vdus,
2252                         vnfr["vdur"],
2253                     )
2254                 )
2255                 if not monitored_vdurs:
2256                     self.logger.error(
2257                         "Scaling criteria is referring to a vnf-monitoring-param that does not contain a reference to a vdu or vnf metric"
2258                     )
2259                     continue
2260                 for scaling_policy in scaling_aspect.get("scaling-policy", ()):
2261                     if scaling_policy["scaling-type"] != "automatic":
2262                         continue
2263                     threshold_time = scaling_policy.get("threshold-time", "1")
2264                     cooldown_time = scaling_policy.get("cooldown-time", "0")
2265                     for scaling_criteria in scaling_policy["scaling-criteria"]:
2266                         monitoring_param_ref = scaling_criteria.get(
2267                             "vnf-monitoring-param-ref"
2268                         )
2269                         vnf_monitoring_param = all_vnfd_monitoring_params[
2270                             monitoring_param_ref
2271                         ]
2272                         for vdur in monitored_vdurs:
2273                             vdu_id = vdur["vdu-id-ref"]
2274                             metric_name = vnf_monitoring_param.get("performance-metric")
2275                             metric_name = f"osm_{metric_name}"
2276                             vnf_member_index = vnfr["member-vnf-index-ref"]
2277                             scalein_threshold = scaling_criteria.get(
2278                                 "scale-in-threshold"
2279                             )
2280                             scaleout_threshold = scaling_criteria.get(
2281                                 "scale-out-threshold"
2282                             )
2283                             # Looking for min/max-number-of-instances
2284                             instances_min_number = 1
2285                             instances_max_number = 1
2286                             vdu_profile = df["vdu-profile"]
2287                             if vdu_profile:
2288                                 profile = next(
2289                                     item for item in vdu_profile if item["id"] == vdu_id
2290                                 )
2291                                 instances_min_number = profile.get(
2292                                     "min-number-of-instances", 1
2293                                 )
2294                                 instances_max_number = profile.get(
2295                                     "max-number-of-instances", 1
2296                                 )
2297
2298                             if scalein_threshold:
2299                                 uuid = str(uuid4())
2300                                 name = f"scalein_{uuid}"
2301                                 operation = scaling_criteria[
2302                                     "scale-in-relational-operation"
2303                                 ]
2304                                 rel_operator = self.rel_operation_types.get(
2305                                     operation, "<="
2306                                 )
2307                                 metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2308                                 expression = f"(count ({metric_selector}) > {instances_min_number}) and (avg({metric_selector}) {rel_operator} {scalein_threshold})"
2309                                 labels = {
2310                                     "ns_id": nsr_id,
2311                                     "vnf_member_index": vnf_member_index,
2312                                     "vdu_id": vdu_id,
2313                                 }
2314                                 prom_cfg = {
2315                                     "alert": name,
2316                                     "expr": expression,
2317                                     "for": str(threshold_time) + "m",
2318                                     "labels": labels,
2319                                 }
2320                                 action = scaling_policy
2321                                 action = {
2322                                     "scaling-group": scaling_group_name,
2323                                     "cooldown-time": cooldown_time,
2324                                 }
2325                                 alert = {
2326                                     "uuid": uuid,
2327                                     "name": name,
2328                                     "metric": metric_name,
2329                                     "tags": {
2330                                         "ns_id": nsr_id,
2331                                         "vnf_member_index": vnf_member_index,
2332                                         "vdu_id": vdu_id,
2333                                     },
2334                                     "alarm_status": "ok",
2335                                     "action_type": "scale_in",
2336                                     "action": action,
2337                                     "prometheus_config": prom_cfg,
2338                                 }
2339                                 alerts.append(alert)
2340
2341                             if scaleout_threshold:
2342                                 uuid = str(uuid4())
2343                                 name = f"scaleout_{uuid}"
2344                                 operation = scaling_criteria[
2345                                     "scale-out-relational-operation"
2346                                 ]
2347                                 rel_operator = self.rel_operation_types.get(
2348                                     operation, "<="
2349                                 )
2350                                 metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2351                                 expression = f"(count ({metric_selector}) < {instances_max_number}) and (avg({metric_selector}) {rel_operator} {scaleout_threshold})"
2352                                 labels = {
2353                                     "ns_id": nsr_id,
2354                                     "vnf_member_index": vnf_member_index,
2355                                     "vdu_id": vdu_id,
2356                                 }
2357                                 prom_cfg = {
2358                                     "alert": name,
2359                                     "expr": expression,
2360                                     "for": str(threshold_time) + "m",
2361                                     "labels": labels,
2362                                 }
2363                                 action = scaling_policy
2364                                 action = {
2365                                     "scaling-group": scaling_group_name,
2366                                     "cooldown-time": cooldown_time,
2367                                 }
2368                                 alert = {
2369                                     "uuid": uuid,
2370                                     "name": name,
2371                                     "metric": metric_name,
2372                                     "tags": {
2373                                         "ns_id": nsr_id,
2374                                         "vnf_member_index": vnf_member_index,
2375                                         "vdu_id": vdu_id,
2376                                     },
2377                                     "alarm_status": "ok",
2378                                     "action_type": "scale_out",
2379                                     "action": action,
2380                                     "prometheus_config": prom_cfg,
2381                                 }
2382                                 alerts.append(alert)
2383         return alerts
2384
2385     def _gather_vnfr_alarm_alerts(self, vnfr, vnfd):
2386         alerts = []
2387         nsr_id = vnfr["nsr-id-ref"]
2388         vnf_member_index = vnfr["member-vnf-index-ref"]
2389
2390         # Checking for VNF alarm configuration
2391         for vdur in vnfr["vdur"]:
2392             vdu_id = vdur["vdu-id-ref"]
2393             vdu = next(filter(lambda vdu: vdu["id"] == vdu_id, vnfd["vdu"]))
2394             if "alarm" in vdu:
2395                 # Get VDU monitoring params, since alerts are based on them
2396                 vdu_monitoring_params = {}
2397                 for mp in vdu.get("monitoring-parameter", []):
2398                     vdu_monitoring_params[mp.get("id")] = mp
2399                 if not vdu_monitoring_params:
2400                     self.logger.error(
2401                         "VDU alarm refers to a VDU monitoring param, but there are no VDU monitoring params in the VDU"
2402                     )
2403                     continue
2404                 # Get alarms in the VDU
2405                 alarm_descriptors = vdu["alarm"]
2406                 # Create VDU alarms for each alarm in the VDU
2407                 for alarm_descriptor in alarm_descriptors:
2408                     # Check that the VDU alarm refers to a proper monitoring param
2409                     alarm_monitoring_param = alarm_descriptor.get(
2410                         "vnf-monitoring-param-ref", ""
2411                     )
2412                     vdu_specific_monitoring_param = vdu_monitoring_params.get(
2413                         alarm_monitoring_param, {}
2414                     )
2415                     if not vdu_specific_monitoring_param:
2416                         self.logger.error(
2417                             "VDU alarm refers to a VDU monitoring param not present in the VDU"
2418                         )
2419                         continue
2420                     metric_name = vdu_specific_monitoring_param.get(
2421                         "performance-metric"
2422                     )
2423                     if not metric_name:
2424                         self.logger.error(
2425                             "VDU alarm refers to a VDU monitoring param that has no associated performance-metric"
2426                         )
2427                         continue
2428                     # Set params of the alarm to be created in Prometheus
2429                     metric_name = f"osm_{metric_name}"
2430                     metric_threshold = alarm_descriptor.get("value")
2431                     uuid = str(uuid4())
2432                     alert_name = f"vdu_alarm_{uuid}"
2433                     operation = alarm_descriptor["operation"]
2434                     rel_operator = self.rel_operation_types.get(operation, "<=")
2435                     metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2436                     expression = f"{metric_selector} {rel_operator} {metric_threshold}"
2437                     labels = {
2438                         "ns_id": nsr_id,
2439                         "vnf_member_index": vnf_member_index,
2440                         "vdu_id": vdu_id,
2441                         "vdu_name": "{{ $labels.vdu_name }}",
2442                     }
2443                     prom_cfg = {
2444                         "alert": alert_name,
2445                         "expr": expression,
2446                         "for": "1m",  # default value. Ideally, this should be related to an IM param, but there is not such param
2447                         "labels": labels,
2448                     }
2449                     alarm_action = dict()
2450                     for action_type in ["ok", "insufficient-data", "alarm"]:
2451                         if (
2452                             "actions" in alarm_descriptor
2453                             and action_type in alarm_descriptor["actions"]
2454                         ):
2455                             alarm_action[action_type] = alarm_descriptor["actions"][
2456                                 action_type
2457                             ]
2458                     alert = {
2459                         "uuid": uuid,
2460                         "name": alert_name,
2461                         "metric": metric_name,
2462                         "tags": {
2463                             "ns_id": nsr_id,
2464                             "vnf_member_index": vnf_member_index,
2465                             "vdu_id": vdu_id,
2466                         },
2467                         "alarm_status": "ok",
2468                         "action_type": "vdu_alarm",
2469                         "action": alarm_action,
2470                         "prometheus_config": prom_cfg,
2471                     }
2472                     alerts.append(alert)
2473         return alerts
2474
2475     def update_nsrs_with_pla_result(self, params):
2476         try:
2477             nslcmop_id = deep_get(params, ("placement", "nslcmopId"))
2478             self.update_db_2(
2479                 "nslcmops", nslcmop_id, {"_admin.pla": params.get("placement")}
2480             )
2481         except Exception as e:
2482             self.logger.warn("Update failed for nslcmop_id={}:{}".format(nslcmop_id, e))
2483
2484     async def instantiate(self, nsr_id, nslcmop_id):
2485         """
2486
2487         :param nsr_id: ns instance to deploy
2488         :param nslcmop_id: operation to run
2489         :return:
2490         """
2491
2492         # Try to lock HA task here
2493         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
2494         if not task_is_locked_by_me:
2495             self.logger.debug(
2496                 "instantiate() task is not locked by me, ns={}".format(nsr_id)
2497             )
2498             return
2499
2500         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
2501         self.logger.debug(logging_text + "Enter")
2502
2503         # get all needed from database
2504
2505         # database nsrs record
2506         db_nsr = None
2507
2508         # database nslcmops record
2509         db_nslcmop = None
2510
2511         # update operation on nsrs
2512         db_nsr_update = {}
2513         # update operation on nslcmops
2514         db_nslcmop_update = {}
2515
2516         timeout_ns_deploy = self.timeout.ns_deploy
2517
2518         nslcmop_operation_state = None
2519         db_vnfrs = {}  # vnf's info indexed by member-index
2520         # n2vc_info = {}
2521         tasks_dict_info = {}  # from task to info text
2522         exc = None
2523         error_list = []
2524         stage = [
2525             "Stage 1/5: preparation of the environment.",
2526             "Waiting for previous operations to terminate.",
2527             "",
2528         ]
2529         # ^ stage, step, VIM progress
2530         try:
2531             # wait for any previous tasks in process
2532             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
2533
2534             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
2535             stage[1] = "Reading from database."
2536             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
2537             db_nsr_update["detailed-status"] = "creating"
2538             db_nsr_update["operational-status"] = "init"
2539             self._write_ns_status(
2540                 nsr_id=nsr_id,
2541                 ns_state="BUILDING",
2542                 current_operation="INSTANTIATING",
2543                 current_operation_id=nslcmop_id,
2544                 other_update=db_nsr_update,
2545             )
2546             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
2547
2548             # read from db: operation
2549             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
2550             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2551             if db_nslcmop["operationParams"].get("additionalParamsForVnf"):
2552                 db_nslcmop["operationParams"]["additionalParamsForVnf"] = json.loads(
2553                     db_nslcmop["operationParams"]["additionalParamsForVnf"]
2554                 )
2555             ns_params = db_nslcmop.get("operationParams")
2556             if ns_params and ns_params.get("timeout_ns_deploy"):
2557                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
2558
2559             # read from db: ns
2560             stage[1] = "Getting nsr={} from db.".format(nsr_id)
2561             self.logger.debug(logging_text + stage[1])
2562             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2563             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
2564             self.logger.debug(logging_text + stage[1])
2565             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2566             self.fs.sync(db_nsr["nsd-id"])
2567             db_nsr["nsd"] = nsd
2568             # nsr_name = db_nsr["name"]   # TODO short-name??
2569
2570             # read from db: vnf's of this ns
2571             stage[1] = "Getting vnfrs from db."
2572             self.logger.debug(logging_text + stage[1])
2573             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
2574
2575             # read from db: vnfd's for every vnf
2576             db_vnfds = []  # every vnfd data
2577
2578             # for each vnf in ns, read vnfd
2579             for vnfr in db_vnfrs_list:
2580                 if vnfr.get("kdur"):
2581                     kdur_list = []
2582                     for kdur in vnfr["kdur"]:
2583                         if kdur.get("additionalParams"):
2584                             kdur["additionalParams"] = json.loads(
2585                                 kdur["additionalParams"]
2586                             )
2587                         kdur_list.append(kdur)
2588                     vnfr["kdur"] = kdur_list
2589
2590                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
2591                 vnfd_id = vnfr["vnfd-id"]
2592                 vnfd_ref = vnfr["vnfd-ref"]
2593                 self.fs.sync(vnfd_id)
2594
2595                 # if we haven't this vnfd, read it from db
2596                 if vnfd_id not in db_vnfds:
2597                     # read from db
2598                     stage[1] = "Getting vnfd={} id='{}' from db.".format(
2599                         vnfd_id, vnfd_ref
2600                     )
2601                     self.logger.debug(logging_text + stage[1])
2602                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
2603
2604                     # store vnfd
2605                     db_vnfds.append(vnfd)
2606
2607             # Get or generates the _admin.deployed.VCA list
2608             vca_deployed_list = None
2609             if db_nsr["_admin"].get("deployed"):
2610                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
2611             if vca_deployed_list is None:
2612                 vca_deployed_list = []
2613                 configuration_status_list = []
2614                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2615                 db_nsr_update["configurationStatus"] = configuration_status_list
2616                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
2617                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2618             elif isinstance(vca_deployed_list, dict):
2619                 # maintain backward compatibility. Change a dict to list at database
2620                 vca_deployed_list = list(vca_deployed_list.values())
2621                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2622                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2623
2624             if not isinstance(
2625                 deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list
2626             ):
2627                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
2628                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
2629
2630             # set state to INSTANTIATED. When instantiated NBI will not delete directly
2631             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
2632             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2633             self.db.set_list(
2634                 "vnfrs", {"nsr-id-ref": nsr_id}, {"_admin.nsState": "INSTANTIATED"}
2635             )
2636
2637             # n2vc_redesign STEP 2 Deploy Network Scenario
2638             stage[0] = "Stage 2/5: deployment of KDUs, VMs and execution environments."
2639             self._write_op_status(op_id=nslcmop_id, stage=stage)
2640
2641             stage[1] = "Deploying KDUs."
2642             # self.logger.debug(logging_text + "Before deploy_kdus")
2643             # Call to deploy_kdus in case exists the "vdu:kdu" param
2644             await self.deploy_kdus(
2645                 logging_text=logging_text,
2646                 nsr_id=nsr_id,
2647                 nslcmop_id=nslcmop_id,
2648                 db_vnfrs=db_vnfrs,
2649                 db_vnfds=db_vnfds,
2650                 task_instantiation_info=tasks_dict_info,
2651             )
2652
2653             stage[1] = "Getting VCA public key."
2654             # n2vc_redesign STEP 1 Get VCA public ssh-key
2655             # feature 1429. Add n2vc public key to needed VMs
2656             n2vc_key = self.n2vc.get_public_key()
2657             n2vc_key_list = [n2vc_key]
2658             if self.vca_config.public_key:
2659                 n2vc_key_list.append(self.vca_config.public_key)
2660
2661             stage[1] = "Deploying NS at VIM."
2662             task_ro = asyncio.ensure_future(
2663                 self.instantiate_RO(
2664                     logging_text=logging_text,
2665                     nsr_id=nsr_id,
2666                     nsd=nsd,
2667                     db_nsr=db_nsr,
2668                     db_nslcmop=db_nslcmop,
2669                     db_vnfrs=db_vnfrs,
2670                     db_vnfds=db_vnfds,
2671                     n2vc_key_list=n2vc_key_list,
2672                     stage=stage,
2673                 )
2674             )
2675             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
2676             tasks_dict_info[task_ro] = "Deploying at VIM"
2677
2678             # n2vc_redesign STEP 3 to 6 Deploy N2VC
2679             stage[1] = "Deploying Execution Environments."
2680             self.logger.debug(logging_text + stage[1])
2681
2682             # create namespace and certificate if any helm based EE is present in the NS
2683             if check_helm_ee_in_ns(db_vnfds):
2684                 await self.vca_map["helm-v3"].setup_ns_namespace(
2685                     name=nsr_id,
2686                 )
2687                 # create TLS certificates
2688                 await self.vca_map["helm-v3"].create_tls_certificate(
2689                     secret_name=self.EE_TLS_NAME,
2690                     dns_prefix="*",
2691                     nsr_id=nsr_id,
2692                     usage="server auth",
2693                     namespace=nsr_id,
2694                 )
2695
2696             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
2697             for vnf_profile in get_vnf_profiles(nsd):
2698                 vnfd_id = vnf_profile["vnfd-id"]
2699                 vnfd = find_in_list(db_vnfds, lambda a_vnf: a_vnf["id"] == vnfd_id)
2700                 member_vnf_index = str(vnf_profile["id"])
2701                 db_vnfr = db_vnfrs[member_vnf_index]
2702                 base_folder = vnfd["_admin"]["storage"]
2703                 vdu_id = None
2704                 vdu_index = 0
2705                 vdu_name = None
2706                 kdu_name = None
2707                 kdu_index = None
2708
2709                 # Get additional parameters
2710                 deploy_params = {"OSM": get_osm_params(db_vnfr)}
2711                 if db_vnfr.get("additionalParamsForVnf"):
2712                     deploy_params.update(
2713                         parse_yaml_strings(db_vnfr["additionalParamsForVnf"].copy())
2714                     )
2715
2716                 descriptor_config = get_configuration(vnfd, vnfd["id"])
2717                 if descriptor_config:
2718                     self._deploy_n2vc(
2719                         logging_text=logging_text
2720                         + "member_vnf_index={} ".format(member_vnf_index),
2721                         db_nsr=db_nsr,
2722                         db_vnfr=db_vnfr,
2723                         nslcmop_id=nslcmop_id,
2724                         nsr_id=nsr_id,
2725                         nsi_id=nsi_id,
2726                         vnfd_id=vnfd_id,
2727                         vdu_id=vdu_id,
2728                         kdu_name=kdu_name,
2729                         member_vnf_index=member_vnf_index,
2730                         vdu_index=vdu_index,
2731                         kdu_index=kdu_index,
2732                         vdu_name=vdu_name,
2733                         deploy_params=deploy_params,
2734                         descriptor_config=descriptor_config,
2735                         base_folder=base_folder,
2736                         task_instantiation_info=tasks_dict_info,
2737                         stage=stage,
2738                     )
2739
2740                 # Deploy charms for each VDU that supports one.
2741                 for vdud in get_vdu_list(vnfd):
2742                     vdu_id = vdud["id"]
2743                     descriptor_config = get_configuration(vnfd, vdu_id)
2744                     vdur = find_in_list(
2745                         db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
2746                     )
2747
2748                     if vdur.get("additionalParams"):
2749                         deploy_params_vdu = parse_yaml_strings(vdur["additionalParams"])
2750                     else:
2751                         deploy_params_vdu = deploy_params
2752                     deploy_params_vdu["OSM"] = get_osm_params(
2753                         db_vnfr, vdu_id, vdu_count_index=0
2754                     )
2755                     vdud_count = get_number_of_instances(vnfd, vdu_id)
2756
2757                     self.logger.debug("VDUD > {}".format(vdud))
2758                     self.logger.debug(
2759                         "Descriptor config > {}".format(descriptor_config)
2760                     )
2761                     if descriptor_config:
2762                         vdu_name = None
2763                         kdu_name = None
2764                         kdu_index = None
2765                         for vdu_index in range(vdud_count):
2766                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
2767                             self._deploy_n2vc(
2768                                 logging_text=logging_text
2769                                 + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
2770                                     member_vnf_index, vdu_id, vdu_index
2771                                 ),
2772                                 db_nsr=db_nsr,
2773                                 db_vnfr=db_vnfr,
2774                                 nslcmop_id=nslcmop_id,
2775                                 nsr_id=nsr_id,
2776                                 nsi_id=nsi_id,
2777                                 vnfd_id=vnfd_id,
2778                                 vdu_id=vdu_id,
2779                                 kdu_name=kdu_name,
2780                                 kdu_index=kdu_index,
2781                                 member_vnf_index=member_vnf_index,
2782                                 vdu_index=vdu_index,
2783                                 vdu_name=vdu_name,
2784                                 deploy_params=deploy_params_vdu,
2785                                 descriptor_config=descriptor_config,
2786                                 base_folder=base_folder,
2787                                 task_instantiation_info=tasks_dict_info,
2788                                 stage=stage,
2789                             )
2790                 for kdud in get_kdu_list(vnfd):
2791                     kdu_name = kdud["name"]
2792                     descriptor_config = get_configuration(vnfd, kdu_name)
2793                     if descriptor_config:
2794                         vdu_id = None
2795                         vdu_index = 0
2796                         vdu_name = None
2797                         kdu_index, kdur = next(
2798                             x
2799                             for x in enumerate(db_vnfr["kdur"])
2800                             if x[1]["kdu-name"] == kdu_name
2801                         )
2802                         deploy_params_kdu = {"OSM": get_osm_params(db_vnfr)}
2803                         if kdur.get("additionalParams"):
2804                             deploy_params_kdu.update(
2805                                 parse_yaml_strings(kdur["additionalParams"].copy())
2806                             )
2807
2808                         self._deploy_n2vc(
2809                             logging_text=logging_text,
2810                             db_nsr=db_nsr,
2811                             db_vnfr=db_vnfr,
2812                             nslcmop_id=nslcmop_id,
2813                             nsr_id=nsr_id,
2814                             nsi_id=nsi_id,
2815                             vnfd_id=vnfd_id,
2816                             vdu_id=vdu_id,
2817                             kdu_name=kdu_name,
2818                             member_vnf_index=member_vnf_index,
2819                             vdu_index=vdu_index,
2820                             kdu_index=kdu_index,
2821                             vdu_name=vdu_name,
2822                             deploy_params=deploy_params_kdu,
2823                             descriptor_config=descriptor_config,
2824                             base_folder=base_folder,
2825                             task_instantiation_info=tasks_dict_info,
2826                             stage=stage,
2827                         )
2828
2829             # Check if each vnf has exporter for metric collection if so update prometheus job records
2830             if "exporters-endpoints" in vnfd.get("df")[0]:
2831                 exporter_config = vnfd.get("df")[0].get("exporters-endpoints")
2832                 self.logger.debug("exporter config :{}".format(exporter_config))
2833                 artifact_path = "{}/{}/{}".format(
2834                     base_folder["folder"],
2835                     base_folder["pkg-dir"],
2836                     "exporter-endpoint",
2837                 )
2838                 ee_id = None
2839                 ee_config_descriptor = exporter_config
2840                 vnfr_id = db_vnfr["id"]
2841                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
2842                     logging_text,
2843                     nsr_id,
2844                     vnfr_id,
2845                     vdu_id=None,
2846                     vdu_index=None,
2847                     user=None,
2848                     pub_key=None,
2849                 )
2850                 self.logger.debug("rw_mgmt_ip:{}".format(rw_mgmt_ip))
2851                 self.logger.debug("Artifact_path:{}".format(artifact_path))
2852                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
2853                 vdu_id_for_prom = None
2854                 vdu_index_for_prom = None
2855                 for x in get_iterable(db_vnfr, "vdur"):
2856                     vdu_id_for_prom = x.get("vdu-id-ref")
2857                     vdu_index_for_prom = x.get("count-index")
2858                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
2859                     ee_id=ee_id,
2860                     artifact_path=artifact_path,
2861                     ee_config_descriptor=ee_config_descriptor,
2862                     vnfr_id=vnfr_id,
2863                     nsr_id=nsr_id,
2864                     target_ip=rw_mgmt_ip,
2865                     element_type="VDU",
2866                     vdu_id=vdu_id_for_prom,
2867                     vdu_index=vdu_index_for_prom,
2868                 )
2869
2870                 self.logger.debug("Prometheus job:{}".format(prometheus_jobs))
2871                 if prometheus_jobs:
2872                     db_nsr_update["_admin.deployed.prometheus_jobs"] = prometheus_jobs
2873                     self.update_db_2(
2874                         "nsrs",
2875                         nsr_id,
2876                         db_nsr_update,
2877                     )
2878
2879                     for job in prometheus_jobs:
2880                         self.db.set_one(
2881                             "prometheus_jobs",
2882                             {"job_name": job["job_name"]},
2883                             job,
2884                             upsert=True,
2885                             fail_on_empty=False,
2886                         )
2887
2888             # Check if this NS has a charm configuration
2889             descriptor_config = nsd.get("ns-configuration")
2890             if descriptor_config and descriptor_config.get("juju"):
2891                 vnfd_id = None
2892                 db_vnfr = None
2893                 member_vnf_index = None
2894                 vdu_id = None
2895                 kdu_name = None
2896                 kdu_index = None
2897                 vdu_index = 0
2898                 vdu_name = None
2899
2900                 # Get additional parameters
2901                 deploy_params = {"OSM": {"vim_account_id": ns_params["vimAccountId"]}}
2902                 if db_nsr.get("additionalParamsForNs"):
2903                     deploy_params.update(
2904                         parse_yaml_strings(db_nsr["additionalParamsForNs"].copy())
2905                     )
2906                 base_folder = nsd["_admin"]["storage"]
2907                 self._deploy_n2vc(
2908                     logging_text=logging_text,
2909                     db_nsr=db_nsr,
2910                     db_vnfr=db_vnfr,
2911                     nslcmop_id=nslcmop_id,
2912                     nsr_id=nsr_id,
2913                     nsi_id=nsi_id,
2914                     vnfd_id=vnfd_id,
2915                     vdu_id=vdu_id,
2916                     kdu_name=kdu_name,
2917                     member_vnf_index=member_vnf_index,
2918                     vdu_index=vdu_index,
2919                     kdu_index=kdu_index,
2920                     vdu_name=vdu_name,
2921                     deploy_params=deploy_params,
2922                     descriptor_config=descriptor_config,
2923                     base_folder=base_folder,
2924                     task_instantiation_info=tasks_dict_info,
2925                     stage=stage,
2926                 )
2927
2928             # rest of staff will be done at finally
2929
2930         except (
2931             ROclient.ROClientException,
2932             DbException,
2933             LcmException,
2934             N2VCException,
2935         ) as e:
2936             self.logger.error(
2937                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e)
2938             )
2939             exc = e
2940         except asyncio.CancelledError:
2941             self.logger.error(
2942                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
2943             )
2944             exc = "Operation was cancelled"
2945         except Exception as e:
2946             exc = traceback.format_exc()
2947             self.logger.critical(
2948                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
2949                 exc_info=True,
2950             )
2951         finally:
2952             if exc:
2953                 error_list.append(str(exc))
2954             try:
2955                 # wait for pending tasks
2956                 if tasks_dict_info:
2957                     stage[1] = "Waiting for instantiate pending tasks."
2958                     self.logger.debug(logging_text + stage[1])
2959                     error_list += await self._wait_for_tasks(
2960                         logging_text,
2961                         tasks_dict_info,
2962                         timeout_ns_deploy,
2963                         stage,
2964                         nslcmop_id,
2965                         nsr_id=nsr_id,
2966                     )
2967                 stage[1] = stage[2] = ""
2968             except asyncio.CancelledError:
2969                 error_list.append("Cancelled")
2970                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
2971                 await self._wait_for_tasks(
2972                     logging_text,
2973                     tasks_dict_info,
2974                     timeout_ns_deploy,
2975                     stage,
2976                     nslcmop_id,
2977                     nsr_id=nsr_id,
2978                 )
2979             except Exception as exc:
2980                 error_list.append(str(exc))
2981
2982             # update operation-status
2983             db_nsr_update["operational-status"] = "running"
2984             # let's begin with VCA 'configured' status (later we can change it)
2985             db_nsr_update["config-status"] = "configured"
2986             for task, task_name in tasks_dict_info.items():
2987                 if not task.done() or task.cancelled() or task.exception():
2988                     if task_name.startswith(self.task_name_deploy_vca):
2989                         # A N2VC task is pending
2990                         db_nsr_update["config-status"] = "failed"
2991                     else:
2992                         # RO or KDU task is pending
2993                         db_nsr_update["operational-status"] = "failed"
2994
2995             # update status at database
2996             if error_list:
2997                 error_detail = ". ".join(error_list)
2998                 self.logger.error(logging_text + error_detail)
2999                 error_description_nslcmop = "{} Detail: {}".format(
3000                     stage[0], error_detail
3001                 )
3002                 error_description_nsr = "Operation: INSTANTIATING.{}, {}".format(
3003                     nslcmop_id, stage[0]
3004                 )
3005
3006                 db_nsr_update["detailed-status"] = (
3007                     error_description_nsr + " Detail: " + error_detail
3008                 )
3009                 db_nslcmop_update["detailed-status"] = error_detail
3010                 nslcmop_operation_state = "FAILED"
3011                 ns_state = "BROKEN"
3012             else:
3013                 error_detail = None
3014                 error_description_nsr = error_description_nslcmop = None
3015                 ns_state = "READY"
3016                 db_nsr_update["detailed-status"] = "Done"
3017                 db_nslcmop_update["detailed-status"] = "Done"
3018                 nslcmop_operation_state = "COMPLETED"
3019                 # Gather auto-healing and auto-scaling alerts for each vnfr
3020                 healing_alerts = []
3021                 scaling_alerts = []
3022                 for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
3023                     vnfd = next(
3024                         (sub for sub in db_vnfds if sub["_id"] == vnfr["vnfd-id"]), None
3025                     )
3026                     healing_alerts = self._gather_vnfr_healing_alerts(vnfr, vnfd)
3027                     for alert in healing_alerts:
3028                         self.logger.info(f"Storing healing alert in MongoDB: {alert}")
3029                         self.db.create("alerts", alert)
3030
3031                     scaling_alerts = self._gather_vnfr_scaling_alerts(vnfr, vnfd)
3032                     for alert in scaling_alerts:
3033                         self.logger.info(f"Storing scaling alert in MongoDB: {alert}")
3034                         self.db.create("alerts", alert)
3035
3036                     alarm_alerts = self._gather_vnfr_alarm_alerts(vnfr, vnfd)
3037                     for alert in alarm_alerts:
3038                         self.logger.info(f"Storing VNF alarm alert in MongoDB: {alert}")
3039                         self.db.create("alerts", alert)
3040             if db_nsr:
3041                 self._write_ns_status(
3042                     nsr_id=nsr_id,
3043                     ns_state=ns_state,
3044                     current_operation="IDLE",
3045                     current_operation_id=None,
3046                     error_description=error_description_nsr,
3047                     error_detail=error_detail,
3048                     other_update=db_nsr_update,
3049                 )
3050             self._write_op_status(
3051                 op_id=nslcmop_id,
3052                 stage="",
3053                 error_message=error_description_nslcmop,
3054                 operation_state=nslcmop_operation_state,
3055                 other_update=db_nslcmop_update,
3056             )
3057
3058             if nslcmop_operation_state:
3059                 try:
3060                     await self.msg.aiowrite(
3061                         "ns",
3062                         "instantiated",
3063                         {
3064                             "nsr_id": nsr_id,
3065                             "nslcmop_id": nslcmop_id,
3066                             "operationState": nslcmop_operation_state,
3067                             "startTime": db_nslcmop["startTime"],
3068                             "links": db_nslcmop["links"],
3069                             "operationParams": {
3070                                 "nsInstanceId": nsr_id,
3071                                 "nsdId": db_nsr["nsd-id"],
3072                             },
3073                         },
3074                     )
3075                 except Exception as e:
3076                     self.logger.error(
3077                         logging_text + "kafka_write notification Exception {}".format(e)
3078                     )
3079
3080             self.logger.debug(logging_text + "Exit")
3081             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
3082
3083     def _get_vnfd(self, vnfd_id: str, projects_read: str, cached_vnfds: Dict[str, Any]):
3084         if vnfd_id not in cached_vnfds:
3085             cached_vnfds[vnfd_id] = self.db.get_one(
3086                 "vnfds", {"id": vnfd_id, "_admin.projects_read": projects_read}
3087             )
3088         return cached_vnfds[vnfd_id]
3089
3090     def _get_vnfr(self, nsr_id: str, vnf_profile_id: str, cached_vnfrs: Dict[str, Any]):
3091         if vnf_profile_id not in cached_vnfrs:
3092             cached_vnfrs[vnf_profile_id] = self.db.get_one(
3093                 "vnfrs",
3094                 {
3095                     "member-vnf-index-ref": vnf_profile_id,
3096                     "nsr-id-ref": nsr_id,
3097                 },
3098             )
3099         return cached_vnfrs[vnf_profile_id]
3100
3101     def _is_deployed_vca_in_relation(
3102         self, vca: DeployedVCA, relation: Relation
3103     ) -> bool:
3104         found = False
3105         for endpoint in (relation.provider, relation.requirer):
3106             if endpoint["kdu-resource-profile-id"]:
3107                 continue
3108             found = (
3109                 vca.vnf_profile_id == endpoint.vnf_profile_id
3110                 and vca.vdu_profile_id == endpoint.vdu_profile_id
3111                 and vca.execution_environment_ref == endpoint.execution_environment_ref
3112             )
3113             if found:
3114                 break
3115         return found
3116
3117     def _update_ee_relation_data_with_implicit_data(
3118         self, nsr_id, nsd, ee_relation_data, cached_vnfds, vnf_profile_id: str = None
3119     ):
3120         ee_relation_data = safe_get_ee_relation(
3121             nsr_id, ee_relation_data, vnf_profile_id=vnf_profile_id
3122         )
3123         ee_relation_level = EELevel.get_level(ee_relation_data)
3124         if (ee_relation_level in (EELevel.VNF, EELevel.VDU)) and not ee_relation_data[
3125             "execution-environment-ref"
3126         ]:
3127             vnf_profile = get_vnf_profile(nsd, ee_relation_data["vnf-profile-id"])
3128             vnfd_id = vnf_profile["vnfd-id"]
3129             project = nsd["_admin"]["projects_read"][0]
3130             db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3131             entity_id = (
3132                 vnfd_id
3133                 if ee_relation_level == EELevel.VNF
3134                 else ee_relation_data["vdu-profile-id"]
3135             )
3136             ee = get_juju_ee_ref(db_vnfd, entity_id)
3137             if not ee:
3138                 raise Exception(
3139                     f"not execution environments found for ee_relation {ee_relation_data}"
3140                 )
3141             ee_relation_data["execution-environment-ref"] = ee["id"]
3142         return ee_relation_data
3143
3144     def _get_ns_relations(
3145         self,
3146         nsr_id: str,
3147         nsd: Dict[str, Any],
3148         vca: DeployedVCA,
3149         cached_vnfds: Dict[str, Any],
3150     ) -> List[Relation]:
3151         relations = []
3152         db_ns_relations = get_ns_configuration_relation_list(nsd)
3153         for r in db_ns_relations:
3154             provider_dict = None
3155             requirer_dict = None
3156             if all(key in r for key in ("provider", "requirer")):
3157                 provider_dict = r["provider"]
3158                 requirer_dict = r["requirer"]
3159             elif "entities" in r:
3160                 provider_id = r["entities"][0]["id"]
3161                 provider_dict = {
3162                     "nsr-id": nsr_id,
3163                     "endpoint": r["entities"][0]["endpoint"],
3164                 }
3165                 if provider_id != nsd["id"]:
3166                     provider_dict["vnf-profile-id"] = provider_id
3167                 requirer_id = r["entities"][1]["id"]
3168                 requirer_dict = {
3169                     "nsr-id": nsr_id,
3170                     "endpoint": r["entities"][1]["endpoint"],
3171                 }
3172                 if requirer_id != nsd["id"]:
3173                     requirer_dict["vnf-profile-id"] = requirer_id
3174             else:
3175                 raise Exception(
3176                     "provider/requirer or entities must be included in the relation."
3177                 )
3178             relation_provider = self._update_ee_relation_data_with_implicit_data(
3179                 nsr_id, nsd, provider_dict, cached_vnfds
3180             )
3181             relation_requirer = self._update_ee_relation_data_with_implicit_data(
3182                 nsr_id, nsd, requirer_dict, cached_vnfds
3183             )
3184             provider = EERelation(relation_provider)
3185             requirer = EERelation(relation_requirer)
3186             relation = Relation(r["name"], provider, requirer)
3187             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
3188             if vca_in_relation:
3189                 relations.append(relation)
3190         return relations
3191
3192     def _get_vnf_relations(
3193         self,
3194         nsr_id: str,
3195         nsd: Dict[str, Any],
3196         vca: DeployedVCA,
3197         cached_vnfds: Dict[str, Any],
3198     ) -> List[Relation]:
3199         relations = []
3200         if vca.target_element == "ns":
3201             self.logger.debug("VCA is a NS charm, not a VNF.")
3202             return relations
3203         vnf_profile = get_vnf_profile(nsd, vca.vnf_profile_id)
3204         vnf_profile_id = vnf_profile["id"]
3205         vnfd_id = vnf_profile["vnfd-id"]
3206         project = nsd["_admin"]["projects_read"][0]
3207         db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3208         db_vnf_relations = get_relation_list(db_vnfd, vnfd_id)
3209         for r in db_vnf_relations:
3210             provider_dict = None
3211             requirer_dict = None
3212             if all(key in r for key in ("provider", "requirer")):
3213                 provider_dict = r["provider"]
3214                 requirer_dict = r["requirer"]
3215             elif "entities" in r:
3216                 provider_id = r["entities"][0]["id"]
3217                 provider_dict = {
3218                     "nsr-id": nsr_id,
3219                     "vnf-profile-id": vnf_profile_id,
3220                     "endpoint": r["entities"][0]["endpoint"],
3221                 }
3222                 if provider_id != vnfd_id:
3223                     provider_dict["vdu-profile-id"] = provider_id
3224                 requirer_id = r["entities"][1]["id"]
3225                 requirer_dict = {
3226                     "nsr-id": nsr_id,
3227                     "vnf-profile-id": vnf_profile_id,
3228                     "endpoint": r["entities"][1]["endpoint"],
3229                 }
3230                 if requirer_id != vnfd_id:
3231                     requirer_dict["vdu-profile-id"] = requirer_id
3232             else:
3233                 raise Exception(
3234                     "provider/requirer or entities must be included in the relation."
3235                 )
3236             relation_provider = self._update_ee_relation_data_with_implicit_data(
3237                 nsr_id, nsd, provider_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
3238             )
3239             relation_requirer = self._update_ee_relation_data_with_implicit_data(
3240                 nsr_id, nsd, requirer_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
3241             )
3242             provider = EERelation(relation_provider)
3243             requirer = EERelation(relation_requirer)
3244             relation = Relation(r["name"], provider, requirer)
3245             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
3246             if vca_in_relation:
3247                 relations.append(relation)
3248         return relations
3249
3250     def _get_kdu_resource_data(
3251         self,
3252         ee_relation: EERelation,
3253         db_nsr: Dict[str, Any],
3254         cached_vnfds: Dict[str, Any],
3255     ) -> DeployedK8sResource:
3256         nsd = get_nsd(db_nsr)
3257         vnf_profiles = get_vnf_profiles(nsd)
3258         vnfd_id = find_in_list(
3259             vnf_profiles,
3260             lambda vnf_profile: vnf_profile["id"] == ee_relation.vnf_profile_id,
3261         )["vnfd-id"]
3262         project = nsd["_admin"]["projects_read"][0]
3263         db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3264         kdu_resource_profile = get_kdu_resource_profile(
3265             db_vnfd, ee_relation.kdu_resource_profile_id
3266         )
3267         kdu_name = kdu_resource_profile["kdu-name"]
3268         deployed_kdu, _ = get_deployed_kdu(
3269             db_nsr.get("_admin", ()).get("deployed", ()),
3270             kdu_name,
3271             ee_relation.vnf_profile_id,
3272         )
3273         deployed_kdu.update({"resource-name": kdu_resource_profile["resource-name"]})
3274         return deployed_kdu
3275
3276     def _get_deployed_component(
3277         self,
3278         ee_relation: EERelation,
3279         db_nsr: Dict[str, Any],
3280         cached_vnfds: Dict[str, Any],
3281     ) -> DeployedComponent:
3282         nsr_id = db_nsr["_id"]
3283         deployed_component = None
3284         ee_level = EELevel.get_level(ee_relation)
3285         if ee_level == EELevel.NS:
3286             vca = get_deployed_vca(db_nsr, {"vdu_id": None, "member-vnf-index": None})
3287             if vca:
3288                 deployed_component = DeployedVCA(nsr_id, vca)
3289         elif ee_level == EELevel.VNF:
3290             vca = get_deployed_vca(
3291                 db_nsr,
3292                 {
3293                     "vdu_id": None,
3294                     "member-vnf-index": ee_relation.vnf_profile_id,
3295                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3296                 },
3297             )
3298             if vca:
3299                 deployed_component = DeployedVCA(nsr_id, vca)
3300         elif ee_level == EELevel.VDU:
3301             vca = get_deployed_vca(
3302                 db_nsr,
3303                 {
3304                     "vdu_id": ee_relation.vdu_profile_id,
3305                     "member-vnf-index": ee_relation.vnf_profile_id,
3306                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3307                 },
3308             )
3309             if vca:
3310                 deployed_component = DeployedVCA(nsr_id, vca)
3311         elif ee_level == EELevel.KDU:
3312             kdu_resource_data = self._get_kdu_resource_data(
3313                 ee_relation, db_nsr, cached_vnfds
3314             )
3315             if kdu_resource_data:
3316                 deployed_component = DeployedK8sResource(kdu_resource_data)
3317         return deployed_component
3318
3319     async def _add_relation(
3320         self,
3321         relation: Relation,
3322         vca_type: str,
3323         db_nsr: Dict[str, Any],
3324         cached_vnfds: Dict[str, Any],
3325         cached_vnfrs: Dict[str, Any],
3326     ) -> bool:
3327         deployed_provider = self._get_deployed_component(
3328             relation.provider, db_nsr, cached_vnfds
3329         )
3330         deployed_requirer = self._get_deployed_component(
3331             relation.requirer, db_nsr, cached_vnfds
3332         )
3333         if (
3334             deployed_provider
3335             and deployed_requirer
3336             and deployed_provider.config_sw_installed
3337             and deployed_requirer.config_sw_installed
3338         ):
3339             provider_db_vnfr = (
3340                 self._get_vnfr(
3341                     relation.provider.nsr_id,
3342                     relation.provider.vnf_profile_id,
3343                     cached_vnfrs,
3344                 )
3345                 if relation.provider.vnf_profile_id
3346                 else None
3347             )
3348             requirer_db_vnfr = (
3349                 self._get_vnfr(
3350                     relation.requirer.nsr_id,
3351                     relation.requirer.vnf_profile_id,
3352                     cached_vnfrs,
3353                 )
3354                 if relation.requirer.vnf_profile_id
3355                 else None
3356             )
3357             provider_vca_id = self.get_vca_id(provider_db_vnfr, db_nsr)
3358             requirer_vca_id = self.get_vca_id(requirer_db_vnfr, db_nsr)
3359             provider_relation_endpoint = RelationEndpoint(
3360                 deployed_provider.ee_id,
3361                 provider_vca_id,
3362                 relation.provider.endpoint,
3363             )
3364             requirer_relation_endpoint = RelationEndpoint(
3365                 deployed_requirer.ee_id,
3366                 requirer_vca_id,
3367                 relation.requirer.endpoint,
3368             )
3369             try:
3370                 await self.vca_map[vca_type].add_relation(
3371                     provider=provider_relation_endpoint,
3372                     requirer=requirer_relation_endpoint,
3373                 )
3374             except N2VCException as exception:
3375                 self.logger.error(exception)
3376                 raise LcmException(exception)
3377             return True
3378         return False
3379
3380     async def _add_vca_relations(
3381         self,
3382         logging_text,
3383         nsr_id,
3384         vca_type: str,
3385         vca_index: int,
3386         timeout: int = 3600,
3387     ) -> bool:
3388         # steps:
3389         # 1. find all relations for this VCA
3390         # 2. wait for other peers related
3391         # 3. add relations
3392
3393         try:
3394             # STEP 1: find all relations for this VCA
3395
3396             # read nsr record
3397             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3398             nsd = get_nsd(db_nsr)
3399
3400             # this VCA data
3401             deployed_vca_dict = get_deployed_vca_list(db_nsr)[vca_index]
3402             my_vca = DeployedVCA(nsr_id, deployed_vca_dict)
3403
3404             cached_vnfds = {}
3405             cached_vnfrs = {}
3406             relations = []
3407             relations.extend(self._get_ns_relations(nsr_id, nsd, my_vca, cached_vnfds))
3408             relations.extend(self._get_vnf_relations(nsr_id, nsd, my_vca, cached_vnfds))
3409
3410             # if no relations, terminate
3411             if not relations:
3412                 self.logger.debug(logging_text + " No relations")
3413                 return True
3414
3415             self.logger.debug(logging_text + " adding relations {}".format(relations))
3416
3417             # add all relations
3418             start = time()
3419             while True:
3420                 # check timeout
3421                 now = time()
3422                 if now - start >= timeout:
3423                     self.logger.error(logging_text + " : timeout adding relations")
3424                     return False
3425
3426                 # reload nsr from database (we need to update record: _admin.deployed.VCA)
3427                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3428
3429                 # for each relation, find the VCA's related
3430                 for relation in relations.copy():
3431                     added = await self._add_relation(
3432                         relation,
3433                         vca_type,
3434                         db_nsr,
3435                         cached_vnfds,
3436                         cached_vnfrs,
3437                     )
3438                     if added:
3439                         relations.remove(relation)
3440
3441                 if not relations:
3442                     self.logger.debug("Relations added")
3443                     break
3444                 await asyncio.sleep(5.0)
3445
3446             return True
3447
3448         except Exception as e:
3449             self.logger.warn(logging_text + " ERROR adding relations: {}".format(e))
3450             return False
3451
3452     async def _install_kdu(
3453         self,
3454         nsr_id: str,
3455         nsr_db_path: str,
3456         vnfr_data: dict,
3457         kdu_index: int,
3458         kdud: dict,
3459         vnfd: dict,
3460         k8s_instance_info: dict,
3461         k8params: dict = None,
3462         timeout: int = 600,
3463         vca_id: str = None,
3464     ):
3465         try:
3466             k8sclustertype = k8s_instance_info["k8scluster-type"]
3467             # Instantiate kdu
3468             db_dict_install = {
3469                 "collection": "nsrs",
3470                 "filter": {"_id": nsr_id},
3471                 "path": nsr_db_path,
3472             }
3473
3474             if k8s_instance_info.get("kdu-deployment-name"):
3475                 kdu_instance = k8s_instance_info.get("kdu-deployment-name")
3476             else:
3477                 kdu_instance = self.k8scluster_map[
3478                     k8sclustertype
3479                 ].generate_kdu_instance_name(
3480                     db_dict=db_dict_install,
3481                     kdu_model=k8s_instance_info["kdu-model"],
3482                     kdu_name=k8s_instance_info["kdu-name"],
3483                 )
3484
3485             # Update the nsrs table with the kdu-instance value
3486             self.update_db_2(
3487                 item="nsrs",
3488                 _id=nsr_id,
3489                 _desc={nsr_db_path + ".kdu-instance": kdu_instance},
3490             )
3491
3492             # Update the nsrs table with the actual namespace being used, if the k8scluster-type is `juju` or
3493             # `juju-bundle`. This verification is needed because there is not a standard/homogeneous namespace
3494             # between the Helm Charts and Juju Bundles-based KNFs. If we found a way of having an homogeneous
3495             # namespace, this first verification could be removed, and the next step would be done for any kind
3496             # of KNF.
3497             # TODO -> find a way to have an homogeneous namespace between the Helm Charts and Juju Bundles-based
3498             # KNFs (Bug 2027: https://osm.etsi.org/bugzilla/show_bug.cgi?id=2027)
3499             if k8sclustertype in ("juju", "juju-bundle"):
3500                 # First, verify if the current namespace is present in the `_admin.projects_read` (if not, it means
3501                 # that the user passed a namespace which he wants its KDU to be deployed in)
3502                 if (
3503                     self.db.count(
3504                         table="nsrs",
3505                         q_filter={
3506                             "_id": nsr_id,
3507                             "_admin.projects_write": k8s_instance_info["namespace"],
3508                             "_admin.projects_read": k8s_instance_info["namespace"],
3509                         },
3510                     )
3511                     > 0
3512                 ):
3513                     self.logger.debug(
3514                         f"Updating namespace/model for Juju Bundle from {k8s_instance_info['namespace']} to {kdu_instance}"
3515                     )
3516                     self.update_db_2(
3517                         item="nsrs",
3518                         _id=nsr_id,
3519                         _desc={f"{nsr_db_path}.namespace": kdu_instance},
3520                     )
3521                     k8s_instance_info["namespace"] = kdu_instance
3522
3523             await self.k8scluster_map[k8sclustertype].install(
3524                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3525                 kdu_model=k8s_instance_info["kdu-model"],
3526                 atomic=True,
3527                 params=k8params,
3528                 db_dict=db_dict_install,
3529                 timeout=timeout,
3530                 kdu_name=k8s_instance_info["kdu-name"],
3531                 namespace=k8s_instance_info["namespace"],
3532                 kdu_instance=kdu_instance,
3533                 vca_id=vca_id,
3534             )
3535
3536             # Obtain services to obtain management service ip
3537             services = await self.k8scluster_map[k8sclustertype].get_services(
3538                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3539                 kdu_instance=kdu_instance,
3540                 namespace=k8s_instance_info["namespace"],
3541             )
3542
3543             # Obtain management service info (if exists)
3544             vnfr_update_dict = {}
3545             kdu_config = get_configuration(vnfd, kdud["name"])
3546             if kdu_config:
3547                 target_ee_list = kdu_config.get("execution-environment-list", [])
3548             else:
3549                 target_ee_list = []
3550
3551             if services:
3552                 vnfr_update_dict["kdur.{}.services".format(kdu_index)] = services
3553                 mgmt_services = [
3554                     service
3555                     for service in kdud.get("service", [])
3556                     if service.get("mgmt-service")
3557                 ]
3558                 for mgmt_service in mgmt_services:
3559                     for service in services:
3560                         if service["name"].startswith(mgmt_service["name"]):
3561                             # Mgmt service found, Obtain service ip
3562                             ip = service.get("external_ip", service.get("cluster_ip"))
3563                             if isinstance(ip, list) and len(ip) == 1:
3564                                 ip = ip[0]
3565
3566                             vnfr_update_dict[
3567                                 "kdur.{}.ip-address".format(kdu_index)
3568                             ] = ip
3569
3570                             # Check if must update also mgmt ip at the vnf
3571                             service_external_cp = mgmt_service.get(
3572                                 "external-connection-point-ref"
3573                             )
3574                             if service_external_cp:
3575                                 if (
3576                                     deep_get(vnfd, ("mgmt-interface", "cp"))
3577                                     == service_external_cp
3578                                 ):
3579                                     vnfr_update_dict["ip-address"] = ip
3580
3581                                 if find_in_list(
3582                                     target_ee_list,
3583                                     lambda ee: ee.get(
3584                                         "external-connection-point-ref", ""
3585                                     )
3586                                     == service_external_cp,
3587                                 ):
3588                                     vnfr_update_dict[
3589                                         "kdur.{}.ip-address".format(kdu_index)
3590                                     ] = ip
3591                             break
3592                     else:
3593                         self.logger.warn(
3594                             "Mgmt service name: {} not found".format(
3595                                 mgmt_service["name"]
3596                             )
3597                         )
3598
3599             vnfr_update_dict["kdur.{}.status".format(kdu_index)] = "READY"
3600             self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
3601
3602             kdu_config = get_configuration(vnfd, k8s_instance_info["kdu-name"])
3603             if (
3604                 kdu_config
3605                 and kdu_config.get("initial-config-primitive")
3606                 and get_juju_ee_ref(vnfd, k8s_instance_info["kdu-name"]) is None
3607             ):
3608                 initial_config_primitive_list = kdu_config.get(
3609                     "initial-config-primitive"
3610                 )
3611                 initial_config_primitive_list.sort(key=lambda val: int(val["seq"]))
3612
3613                 for initial_config_primitive in initial_config_primitive_list:
3614                     primitive_params_ = self._map_primitive_params(
3615                         initial_config_primitive, {}, {}
3616                     )
3617
3618                     await asyncio.wait_for(
3619                         self.k8scluster_map[k8sclustertype].exec_primitive(
3620                             cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3621                             kdu_instance=kdu_instance,
3622                             primitive_name=initial_config_primitive["name"],
3623                             params=primitive_params_,
3624                             db_dict=db_dict_install,
3625                             vca_id=vca_id,
3626                         ),
3627                         timeout=timeout,
3628                     )
3629
3630         except Exception as e:
3631             # Prepare update db with error and raise exception
3632             try:
3633                 self.update_db_2(
3634                     "nsrs", nsr_id, {nsr_db_path + ".detailed-status": str(e)}
3635                 )
3636                 self.update_db_2(
3637                     "vnfrs",
3638                     vnfr_data.get("_id"),
3639                     {"kdur.{}.status".format(kdu_index): "ERROR"},
3640                 )
3641             except Exception as error:
3642                 # ignore to keep original exception
3643                 self.logger.warning(
3644                     f"An exception occurred while updating DB: {str(error)}"
3645                 )
3646             # reraise original error
3647             raise
3648
3649         return kdu_instance
3650
3651     async def deploy_kdus(
3652         self,
3653         logging_text,
3654         nsr_id,
3655         nslcmop_id,
3656         db_vnfrs,
3657         db_vnfds,
3658         task_instantiation_info,
3659     ):
3660         # Launch kdus if present in the descriptor
3661
3662         k8scluster_id_2_uuic = {
3663             "helm-chart-v3": {},
3664             "juju-bundle": {},
3665         }
3666
3667         async def _get_cluster_id(cluster_id, cluster_type):
3668             nonlocal k8scluster_id_2_uuic
3669             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
3670                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
3671
3672             # check if K8scluster is creating and wait look if previous tasks in process
3673             task_name, task_dependency = self.lcm_tasks.lookfor_related(
3674                 "k8scluster", cluster_id
3675             )
3676             if task_dependency:
3677                 text = "Waiting for related tasks '{}' on k8scluster {} to be completed".format(
3678                     task_name, cluster_id
3679                 )
3680                 self.logger.debug(logging_text + text)
3681                 await asyncio.wait(task_dependency, timeout=3600)
3682
3683             db_k8scluster = self.db.get_one(
3684                 "k8sclusters", {"_id": cluster_id}, fail_on_empty=False
3685             )
3686             if not db_k8scluster:
3687                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
3688
3689             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
3690             if not k8s_id:
3691                 if cluster_type == "helm-chart-v3":
3692                     try:
3693                         # backward compatibility for existing clusters that have not been initialized for helm v3
3694                         k8s_credentials = yaml.safe_dump(
3695                             db_k8scluster.get("credentials")
3696                         )
3697                         k8s_id, uninstall_sw = await self.k8sclusterhelm3.init_env(
3698                             k8s_credentials, reuse_cluster_uuid=cluster_id
3699                         )
3700                         db_k8scluster_update = {}
3701                         db_k8scluster_update["_admin.helm-chart-v3.error_msg"] = None
3702                         db_k8scluster_update["_admin.helm-chart-v3.id"] = k8s_id
3703                         db_k8scluster_update[
3704                             "_admin.helm-chart-v3.created"
3705                         ] = uninstall_sw
3706                         db_k8scluster_update[
3707                             "_admin.helm-chart-v3.operationalState"
3708                         ] = "ENABLED"
3709                         self.update_db_2(
3710                             "k8sclusters", cluster_id, db_k8scluster_update
3711                         )
3712                     except Exception as e:
3713                         self.logger.error(
3714                             logging_text
3715                             + "error initializing helm-v3 cluster: {}".format(str(e))
3716                         )
3717                         raise LcmException(
3718                             "K8s cluster '{}' has not been initialized for '{}'".format(
3719                                 cluster_id, cluster_type
3720                             )
3721                         )
3722                 else:
3723                     raise LcmException(
3724                         "K8s cluster '{}' has not been initialized for '{}'".format(
3725                             cluster_id, cluster_type
3726                         )
3727                     )
3728             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
3729             return k8s_id
3730
3731         logging_text += "Deploy kdus: "
3732         step = ""
3733         try:
3734             db_nsr_update = {"_admin.deployed.K8s": []}
3735             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3736
3737             index = 0
3738             updated_cluster_list = []
3739             updated_v3_cluster_list = []
3740
3741             for vnfr_data in db_vnfrs.values():
3742                 vca_id = self.get_vca_id(vnfr_data, {})
3743                 for kdu_index, kdur in enumerate(get_iterable(vnfr_data, "kdur")):
3744                     # Step 0: Prepare and set parameters
3745                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
3746                     vnfd_id = vnfr_data.get("vnfd-id")
3747                     vnfd_with_id = find_in_list(
3748                         db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3749                     )
3750                     kdud = next(
3751                         kdud
3752                         for kdud in vnfd_with_id["kdu"]
3753                         if kdud["name"] == kdur["kdu-name"]
3754                     )
3755                     namespace = kdur.get("k8s-namespace")
3756                     kdu_deployment_name = kdur.get("kdu-deployment-name")
3757                     if kdur.get("helm-chart"):
3758                         kdumodel = kdur["helm-chart"]
3759                         # Default version: helm3, if helm-version is v2 assign v2
3760                         k8sclustertype = "helm-chart-v3"
3761                         self.logger.debug("kdur: {}".format(kdur))
3762                     elif kdur.get("juju-bundle"):
3763                         kdumodel = kdur["juju-bundle"]
3764                         k8sclustertype = "juju-bundle"
3765                     else:
3766                         raise LcmException(
3767                             "kdu type for kdu='{}.{}' is neither helm-chart nor "
3768                             "juju-bundle. Maybe an old NBI version is running".format(
3769                                 vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]
3770                             )
3771                         )
3772                     # check if kdumodel is a file and exists
3773                     try:
3774                         vnfd_with_id = find_in_list(
3775                             db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3776                         )
3777                         storage = deep_get(vnfd_with_id, ("_admin", "storage"))
3778                         if storage:  # may be not present if vnfd has not artifacts
3779                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
3780                             if storage["pkg-dir"]:
3781                                 filename = "{}/{}/{}s/{}".format(
3782                                     storage["folder"],
3783                                     storage["pkg-dir"],
3784                                     k8sclustertype,
3785                                     kdumodel,
3786                                 )
3787                             else:
3788                                 filename = "{}/Scripts/{}s/{}".format(
3789                                     storage["folder"],
3790                                     k8sclustertype,
3791                                     kdumodel,
3792                                 )
3793                             if self.fs.file_exists(
3794                                 filename, mode="file"
3795                             ) or self.fs.file_exists(filename, mode="dir"):
3796                                 kdumodel = self.fs.path + filename
3797                     except (asyncio.TimeoutError, asyncio.CancelledError):
3798                         raise
3799                     except Exception as e:  # it is not a file
3800                         self.logger.warning(f"An exception occurred: {str(e)}")
3801
3802                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
3803                     step = "Synchronize repos for k8s cluster '{}'".format(
3804                         k8s_cluster_id
3805                     )
3806                     cluster_uuid = await _get_cluster_id(k8s_cluster_id, k8sclustertype)
3807
3808                     # Synchronize  repos
3809                     if (
3810                         k8sclustertype == "helm-chart"
3811                         and cluster_uuid not in updated_cluster_list
3812                     ) or (
3813                         k8sclustertype == "helm-chart-v3"
3814                         and cluster_uuid not in updated_v3_cluster_list
3815                     ):
3816                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
3817                             self.k8scluster_map[k8sclustertype].synchronize_repos(
3818                                 cluster_uuid=cluster_uuid
3819                             )
3820                         )
3821                         if del_repo_list or added_repo_dict:
3822                             if k8sclustertype == "helm-chart":
3823                                 unset = {
3824                                     "_admin.helm_charts_added." + item: None
3825                                     for item in del_repo_list
3826                                 }
3827                                 updated = {
3828                                     "_admin.helm_charts_added." + item: name
3829                                     for item, name in added_repo_dict.items()
3830                                 }
3831                                 updated_cluster_list.append(cluster_uuid)
3832                             elif k8sclustertype == "helm-chart-v3":
3833                                 unset = {
3834                                     "_admin.helm_charts_v3_added." + item: None
3835                                     for item in del_repo_list
3836                                 }
3837                                 updated = {
3838                                     "_admin.helm_charts_v3_added." + item: name
3839                                     for item, name in added_repo_dict.items()
3840                                 }
3841                                 updated_v3_cluster_list.append(cluster_uuid)
3842                             self.logger.debug(
3843                                 logging_text + "repos synchronized on k8s cluster "
3844                                 "'{}' to_delete: {}, to_add: {}".format(
3845                                     k8s_cluster_id, del_repo_list, added_repo_dict
3846                                 )
3847                             )
3848                             self.db.set_one(
3849                                 "k8sclusters",
3850                                 {"_id": k8s_cluster_id},
3851                                 updated,
3852                                 unset=unset,
3853                             )
3854
3855                     # Instantiate kdu
3856                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(
3857                         vnfr_data["member-vnf-index-ref"],
3858                         kdur["kdu-name"],
3859                         k8s_cluster_id,
3860                     )
3861                     k8s_instance_info = {
3862                         "kdu-instance": None,
3863                         "k8scluster-uuid": cluster_uuid,
3864                         "k8scluster-type": k8sclustertype,
3865                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
3866                         "kdu-name": kdur["kdu-name"],
3867                         "kdu-model": kdumodel,
3868                         "namespace": namespace,
3869                         "kdu-deployment-name": kdu_deployment_name,
3870                     }
3871                     db_path = "_admin.deployed.K8s.{}".format(index)
3872                     db_nsr_update[db_path] = k8s_instance_info
3873                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
3874                     vnfd_with_id = find_in_list(
3875                         db_vnfds, lambda vnf: vnf["_id"] == vnfd_id
3876                     )
3877                     task = asyncio.ensure_future(
3878                         self._install_kdu(
3879                             nsr_id,
3880                             db_path,
3881                             vnfr_data,
3882                             kdu_index,
3883                             kdud,
3884                             vnfd_with_id,
3885                             k8s_instance_info,
3886                             k8params=desc_params,
3887                             timeout=1800,
3888                             vca_id=vca_id,
3889                         )
3890                     )
3891                     self.lcm_tasks.register(
3892                         "ns",
3893                         nsr_id,
3894                         nslcmop_id,
3895                         "instantiate_KDU-{}".format(index),
3896                         task,
3897                     )
3898                     task_instantiation_info[task] = "Deploying KDU {}".format(
3899                         kdur["kdu-name"]
3900                     )
3901
3902                     index += 1
3903
3904         except (LcmException, asyncio.CancelledError):
3905             raise
3906         except Exception as e:
3907             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
3908             if isinstance(e, (N2VCException, DbException)):
3909                 self.logger.error(logging_text + msg)
3910             else:
3911                 self.logger.critical(logging_text + msg, exc_info=True)
3912             raise LcmException(msg)
3913         finally:
3914             if db_nsr_update:
3915                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3916
3917     def _deploy_n2vc(
3918         self,
3919         logging_text,
3920         db_nsr,
3921         db_vnfr,
3922         nslcmop_id,
3923         nsr_id,
3924         nsi_id,
3925         vnfd_id,
3926         vdu_id,
3927         kdu_name,
3928         member_vnf_index,
3929         vdu_index,
3930         kdu_index,
3931         vdu_name,
3932         deploy_params,
3933         descriptor_config,
3934         base_folder,
3935         task_instantiation_info,
3936         stage,
3937     ):
3938         # launch instantiate_N2VC in a asyncio task and register task object
3939         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
3940         # if not found, create one entry and update database
3941         # fill db_nsr._admin.deployed.VCA.<index>
3942
3943         self.logger.debug(
3944             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
3945         )
3946
3947         charm_name = ""
3948         get_charm_name = False
3949         if "execution-environment-list" in descriptor_config:
3950             ee_list = descriptor_config.get("execution-environment-list", [])
3951         elif "juju" in descriptor_config:
3952             ee_list = [descriptor_config]  # ns charms
3953             if "execution-environment-list" not in descriptor_config:
3954                 # charm name is only required for ns charms
3955                 get_charm_name = True
3956         else:  # other types as script are not supported
3957             ee_list = []
3958
3959         for ee_item in ee_list:
3960             self.logger.debug(
3961                 logging_text
3962                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
3963                     ee_item.get("juju"), ee_item.get("helm-chart")
3964                 )
3965             )
3966             ee_descriptor_id = ee_item.get("id")
3967             vca_name, charm_name, vca_type = self.get_vca_info(
3968                 ee_item, db_nsr, get_charm_name
3969             )
3970             if not vca_type:
3971                 self.logger.debug(
3972                     logging_text + "skipping, non juju/charm/helm configuration"
3973                 )
3974                 continue
3975
3976             vca_index = -1
3977             for vca_index, vca_deployed in enumerate(
3978                 db_nsr["_admin"]["deployed"]["VCA"]
3979             ):
3980                 if not vca_deployed:
3981                     continue
3982                 if (
3983                     vca_deployed.get("member-vnf-index") == member_vnf_index
3984                     and vca_deployed.get("vdu_id") == vdu_id
3985                     and vca_deployed.get("kdu_name") == kdu_name
3986                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
3987                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
3988                 ):
3989                     break
3990             else:
3991                 # not found, create one.
3992                 target = (
3993                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
3994                 )
3995                 if vdu_id:
3996                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
3997                 elif kdu_name:
3998                     target += "/kdu/{}".format(kdu_name)
3999                 vca_deployed = {
4000                     "target_element": target,
4001                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
4002                     "member-vnf-index": member_vnf_index,
4003                     "vdu_id": vdu_id,
4004                     "kdu_name": kdu_name,
4005                     "vdu_count_index": vdu_index,
4006                     "operational-status": "init",  # TODO revise
4007                     "detailed-status": "",  # TODO revise
4008                     "step": "initial-deploy",  # TODO revise
4009                     "vnfd_id": vnfd_id,
4010                     "vdu_name": vdu_name,
4011                     "type": vca_type,
4012                     "ee_descriptor_id": ee_descriptor_id,
4013                     "charm_name": charm_name,
4014                 }
4015                 vca_index += 1
4016
4017                 # create VCA and configurationStatus in db
4018                 db_dict = {
4019                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
4020                     "configurationStatus.{}".format(vca_index): dict(),
4021                 }
4022                 self.update_db_2("nsrs", nsr_id, db_dict)
4023
4024                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
4025
4026             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
4027             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
4028             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
4029
4030             # Launch task
4031             task_n2vc = asyncio.ensure_future(
4032                 self.instantiate_N2VC(
4033                     logging_text=logging_text,
4034                     vca_index=vca_index,
4035                     nsi_id=nsi_id,
4036                     db_nsr=db_nsr,
4037                     db_vnfr=db_vnfr,
4038                     vdu_id=vdu_id,
4039                     kdu_name=kdu_name,
4040                     vdu_index=vdu_index,
4041                     kdu_index=kdu_index,
4042                     deploy_params=deploy_params,
4043                     config_descriptor=descriptor_config,
4044                     base_folder=base_folder,
4045                     nslcmop_id=nslcmop_id,
4046                     stage=stage,
4047                     vca_type=vca_type,
4048                     vca_name=vca_name,
4049                     ee_config_descriptor=ee_item,
4050                 )
4051             )
4052             self.lcm_tasks.register(
4053                 "ns",
4054                 nsr_id,
4055                 nslcmop_id,
4056                 "instantiate_N2VC-{}".format(vca_index),
4057                 task_n2vc,
4058             )
4059             task_instantiation_info[
4060                 task_n2vc
4061             ] = self.task_name_deploy_vca + " {}.{}".format(
4062                 member_vnf_index or "", vdu_id or ""
4063             )
4064
4065     def _format_additional_params(self, params):
4066         params = params or {}
4067         for key, value in params.items():
4068             if str(value).startswith("!!yaml "):
4069                 params[key] = yaml.safe_load(value[7:])
4070         return params
4071
4072     def _get_terminate_primitive_params(self, seq, vnf_index):
4073         primitive = seq.get("name")
4074         primitive_params = {}
4075         params = {
4076             "member_vnf_index": vnf_index,
4077             "primitive": primitive,
4078             "primitive_params": primitive_params,
4079         }
4080         desc_params = {}
4081         return self._map_primitive_params(seq, params, desc_params)
4082
4083     # sub-operations
4084
4085     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
4086         op = deep_get(db_nslcmop, ("_admin", "operations"), [])[op_index]
4087         if op.get("operationState") == "COMPLETED":
4088             # b. Skip sub-operation
4089             # _ns_execute_primitive() or RO.create_action() will NOT be executed
4090             return self.SUBOPERATION_STATUS_SKIP
4091         else:
4092             # c. retry executing sub-operation
4093             # The sub-operation exists, and operationState != 'COMPLETED'
4094             # Update operationState = 'PROCESSING' to indicate a retry.
4095             operationState = "PROCESSING"
4096             detailed_status = "In progress"
4097             self._update_suboperation_status(
4098                 db_nslcmop, op_index, operationState, detailed_status
4099             )
4100             # Return the sub-operation index
4101             # _ns_execute_primitive() or RO.create_action() will be called from scale()
4102             # with arguments extracted from the sub-operation
4103             return op_index
4104
4105     # Find a sub-operation where all keys in a matching dictionary must match
4106     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
4107     def _find_suboperation(self, db_nslcmop, match):
4108         if db_nslcmop and match:
4109             op_list = db_nslcmop.get("_admin", {}).get("operations", [])
4110             for i, op in enumerate(op_list):
4111                 if all(op.get(k) == match[k] for k in match):
4112                     return i
4113         return self.SUBOPERATION_STATUS_NOT_FOUND
4114
4115     # Update status for a sub-operation given its index
4116     def _update_suboperation_status(
4117         self, db_nslcmop, op_index, operationState, detailed_status
4118     ):
4119         # Update DB for HA tasks
4120         q_filter = {"_id": db_nslcmop["_id"]}
4121         update_dict = {
4122             "_admin.operations.{}.operationState".format(op_index): operationState,
4123             "_admin.operations.{}.detailed-status".format(op_index): detailed_status,
4124         }
4125         self.db.set_one(
4126             "nslcmops", q_filter=q_filter, update_dict=update_dict, fail_on_empty=False
4127         )
4128
4129     # Add sub-operation, return the index of the added sub-operation
4130     # Optionally, set operationState, detailed-status, and operationType
4131     # Status and type are currently set for 'scale' sub-operations:
4132     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
4133     # 'detailed-status' : status message
4134     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
4135     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
4136     def _add_suboperation(
4137         self,
4138         db_nslcmop,
4139         vnf_index,
4140         vdu_id,
4141         vdu_count_index,
4142         vdu_name,
4143         primitive,
4144         mapped_primitive_params,
4145         operationState=None,
4146         detailed_status=None,
4147         operationType=None,
4148         RO_nsr_id=None,
4149         RO_scaling_info=None,
4150     ):
4151         if not db_nslcmop:
4152             return self.SUBOPERATION_STATUS_NOT_FOUND
4153         # Get the "_admin.operations" list, if it exists
4154         db_nslcmop_admin = db_nslcmop.get("_admin", {})
4155         op_list = db_nslcmop_admin.get("operations")
4156         # Create or append to the "_admin.operations" list
4157         new_op = {
4158             "member_vnf_index": vnf_index,
4159             "vdu_id": vdu_id,
4160             "vdu_count_index": vdu_count_index,
4161             "primitive": primitive,
4162             "primitive_params": mapped_primitive_params,
4163         }
4164         if operationState:
4165             new_op["operationState"] = operationState
4166         if detailed_status:
4167             new_op["detailed-status"] = detailed_status
4168         if operationType:
4169             new_op["lcmOperationType"] = operationType
4170         if RO_nsr_id:
4171             new_op["RO_nsr_id"] = RO_nsr_id
4172         if RO_scaling_info:
4173             new_op["RO_scaling_info"] = RO_scaling_info
4174         if not op_list:
4175             # No existing operations, create key 'operations' with current operation as first list element
4176             db_nslcmop_admin.update({"operations": [new_op]})
4177             op_list = db_nslcmop_admin.get("operations")
4178         else:
4179             # Existing operations, append operation to list
4180             op_list.append(new_op)
4181
4182         db_nslcmop_update = {"_admin.operations": op_list}
4183         self.update_db_2("nslcmops", db_nslcmop["_id"], db_nslcmop_update)
4184         op_index = len(op_list) - 1
4185         return op_index
4186
4187     # Helper methods for scale() sub-operations
4188
4189     # pre-scale/post-scale:
4190     # Check for 3 different cases:
4191     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
4192     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
4193     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
4194     def _check_or_add_scale_suboperation(
4195         self,
4196         db_nslcmop,
4197         vnf_index,
4198         vnf_config_primitive,
4199         primitive_params,
4200         operationType,
4201         RO_nsr_id=None,
4202         RO_scaling_info=None,
4203     ):
4204         # Find this sub-operation
4205         if RO_nsr_id and RO_scaling_info:
4206             operationType = "SCALE-RO"
4207             match = {
4208                 "member_vnf_index": vnf_index,
4209                 "RO_nsr_id": RO_nsr_id,
4210                 "RO_scaling_info": RO_scaling_info,
4211             }
4212         else:
4213             match = {
4214                 "member_vnf_index": vnf_index,
4215                 "primitive": vnf_config_primitive,
4216                 "primitive_params": primitive_params,
4217                 "lcmOperationType": operationType,
4218             }
4219         op_index = self._find_suboperation(db_nslcmop, match)
4220         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
4221             # a. New sub-operation
4222             # The sub-operation does not exist, add it.
4223             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
4224             # The following parameters are set to None for all kind of scaling:
4225             vdu_id = None
4226             vdu_count_index = None
4227             vdu_name = None
4228             if RO_nsr_id and RO_scaling_info:
4229                 vnf_config_primitive = None
4230                 primitive_params = None
4231             else:
4232                 RO_nsr_id = None
4233                 RO_scaling_info = None
4234             # Initial status for sub-operation
4235             operationState = "PROCESSING"
4236             detailed_status = "In progress"
4237             # Add sub-operation for pre/post-scaling (zero or more operations)
4238             self._add_suboperation(
4239                 db_nslcmop,
4240                 vnf_index,
4241                 vdu_id,
4242                 vdu_count_index,
4243                 vdu_name,
4244                 vnf_config_primitive,
4245                 primitive_params,
4246                 operationState,
4247                 detailed_status,
4248                 operationType,
4249                 RO_nsr_id,
4250                 RO_scaling_info,
4251             )
4252             return self.SUBOPERATION_STATUS_NEW
4253         else:
4254             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
4255             # or op_index (operationState != 'COMPLETED')
4256             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
4257
4258     # Function to return execution_environment id
4259
4260     async def destroy_N2VC(
4261         self,
4262         logging_text,
4263         db_nslcmop,
4264         vca_deployed,
4265         config_descriptor,
4266         vca_index,
4267         destroy_ee=True,
4268         exec_primitives=True,
4269         scaling_in=False,
4270         vca_id: str = None,
4271     ):
4272         """
4273         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
4274         :param logging_text:
4275         :param db_nslcmop:
4276         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
4277         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
4278         :param vca_index: index in the database _admin.deployed.VCA
4279         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
4280         :param exec_primitives: False to do not execute terminate primitives, because the config is not completed or has
4281                             not executed properly
4282         :param scaling_in: True destroys the application, False destroys the model
4283         :return: None or exception
4284         """
4285
4286         self.logger.debug(
4287             logging_text
4288             + " vca_index: {}, vca_deployed: {}, config_descriptor: {}, destroy_ee: {}".format(
4289                 vca_index, vca_deployed, config_descriptor, destroy_ee
4290             )
4291         )
4292
4293         vca_type = vca_deployed.get("type", "lxc_proxy_charm")
4294
4295         # execute terminate_primitives
4296         if exec_primitives:
4297             terminate_primitives = get_ee_sorted_terminate_config_primitive_list(
4298                 config_descriptor.get("terminate-config-primitive"),
4299                 vca_deployed.get("ee_descriptor_id"),
4300             )
4301             vdu_id = vca_deployed.get("vdu_id")
4302             vdu_count_index = vca_deployed.get("vdu_count_index")
4303             vdu_name = vca_deployed.get("vdu_name")
4304             vnf_index = vca_deployed.get("member-vnf-index")
4305             if terminate_primitives and vca_deployed.get("needed_terminate"):
4306                 for seq in terminate_primitives:
4307                     # For each sequence in list, get primitive and call _ns_execute_primitive()
4308                     step = "Calling terminate action for vnf_member_index={} primitive={}".format(
4309                         vnf_index, seq.get("name")
4310                     )
4311                     self.logger.debug(logging_text + step)
4312                     # Create the primitive for each sequence, i.e. "primitive": "touch"
4313                     primitive = seq.get("name")
4314                     mapped_primitive_params = self._get_terminate_primitive_params(
4315                         seq, vnf_index
4316                     )
4317
4318                     # Add sub-operation
4319                     self._add_suboperation(
4320                         db_nslcmop,
4321                         vnf_index,
4322                         vdu_id,
4323                         vdu_count_index,
4324                         vdu_name,
4325                         primitive,
4326                         mapped_primitive_params,
4327                     )
4328                     # Sub-operations: Call _ns_execute_primitive() instead of action()
4329                     try:
4330                         result, result_detail = await self._ns_execute_primitive(
4331                             vca_deployed["ee_id"],
4332                             primitive,
4333                             mapped_primitive_params,
4334                             vca_type=vca_type,
4335                             vca_id=vca_id,
4336                         )
4337                     except LcmException:
4338                         # this happens when VCA is not deployed. In this case it is not needed to terminate
4339                         continue
4340                     result_ok = ["COMPLETED", "PARTIALLY_COMPLETED"]
4341                     if result not in result_ok:
4342                         raise LcmException(
4343                             "terminate_primitive {}  for vnf_member_index={} fails with "
4344                             "error {}".format(seq.get("name"), vnf_index, result_detail)
4345                         )
4346                 # set that this VCA do not need terminated
4347                 db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(
4348                     vca_index
4349                 )
4350                 self.update_db_2(
4351                     "nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False}
4352                 )
4353
4354         # Delete Prometheus Jobs if any
4355         # This uses NSR_ID, so it will destroy any jobs under this index
4356         self.db.del_list("prometheus_jobs", {"nsr_id": db_nslcmop["nsInstanceId"]})
4357
4358         if destroy_ee:
4359             await self.vca_map[vca_type].delete_execution_environment(
4360                 vca_deployed["ee_id"],
4361                 scaling_in=scaling_in,
4362                 vca_type=vca_type,
4363                 vca_id=vca_id,
4364             )
4365
4366     async def _delete_all_N2VC(self, db_nsr: dict, vca_id: str = None):
4367         self._write_all_config_status(db_nsr=db_nsr, status="TERMINATING")
4368         namespace = "." + db_nsr["_id"]
4369         try:
4370             await self.n2vc.delete_namespace(
4371                 namespace=namespace,
4372                 total_timeout=self.timeout.charm_delete,
4373                 vca_id=vca_id,
4374             )
4375         except N2VCNotFound:  # already deleted. Skip
4376             pass
4377         self._write_all_config_status(db_nsr=db_nsr, status="DELETED")
4378
4379     async def terminate(self, nsr_id, nslcmop_id):
4380         # Try to lock HA task here
4381         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
4382         if not task_is_locked_by_me:
4383             return
4384
4385         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
4386         self.logger.debug(logging_text + "Enter")
4387         timeout_ns_terminate = self.timeout.ns_terminate
4388         db_nsr = None
4389         db_nslcmop = None
4390         operation_params = None
4391         exc = None
4392         error_list = []  # annotates all failed error messages
4393         db_nslcmop_update = {}
4394         autoremove = False  # autoremove after terminated
4395         tasks_dict_info = {}
4396         db_nsr_update = {}
4397         stage = [
4398             "Stage 1/3: Preparing task.",
4399             "Waiting for previous operations to terminate.",
4400             "",
4401         ]
4402         # ^ contains [stage, step, VIM-status]
4403         try:
4404             # wait for any previous tasks in process
4405             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
4406
4407             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
4408             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
4409             operation_params = db_nslcmop.get("operationParams") or {}
4410             if operation_params.get("timeout_ns_terminate"):
4411                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
4412             stage[1] = "Getting nsr={} from db.".format(nsr_id)
4413             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4414
4415             db_nsr_update["operational-status"] = "terminating"
4416             db_nsr_update["config-status"] = "terminating"
4417             self._write_ns_status(
4418                 nsr_id=nsr_id,
4419                 ns_state="TERMINATING",
4420                 current_operation="TERMINATING",
4421                 current_operation_id=nslcmop_id,
4422                 other_update=db_nsr_update,
4423             )
4424             self._write_op_status(op_id=nslcmop_id, queuePosition=0, stage=stage)
4425             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
4426             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
4427                 return
4428
4429             stage[1] = "Getting vnf descriptors from db."
4430             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
4431             db_vnfrs_dict = {
4432                 db_vnfr["member-vnf-index-ref"]: db_vnfr for db_vnfr in db_vnfrs_list
4433             }
4434             db_vnfds_from_id = {}
4435             db_vnfds_from_member_index = {}
4436             # Loop over VNFRs
4437             for vnfr in db_vnfrs_list:
4438                 vnfd_id = vnfr["vnfd-id"]
4439                 if vnfd_id not in db_vnfds_from_id:
4440                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
4441                     db_vnfds_from_id[vnfd_id] = vnfd
4442                 db_vnfds_from_member_index[
4443                     vnfr["member-vnf-index-ref"]
4444                 ] = db_vnfds_from_id[vnfd_id]
4445
4446             # Destroy individual execution environments when there are terminating primitives.
4447             # Rest of EE will be deleted at once
4448             # TODO - check before calling _destroy_N2VC
4449             # if not operation_params.get("skip_terminate_primitives"):#
4450             # or not vca.get("needed_terminate"):
4451             stage[0] = "Stage 2/3 execute terminating primitives."
4452             self.logger.debug(logging_text + stage[0])
4453             stage[1] = "Looking execution environment that needs terminate."
4454             self.logger.debug(logging_text + stage[1])
4455
4456             for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
4457                 config_descriptor = None
4458                 vca_member_vnf_index = vca.get("member-vnf-index")
4459                 vca_id = self.get_vca_id(
4460                     db_vnfrs_dict.get(vca_member_vnf_index)
4461                     if vca_member_vnf_index
4462                     else None,
4463                     db_nsr,
4464                 )
4465                 if not vca or not vca.get("ee_id"):
4466                     continue
4467                 if not vca.get("member-vnf-index"):
4468                     # ns
4469                     config_descriptor = db_nsr.get("ns-configuration")
4470                 elif vca.get("vdu_id"):
4471                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4472                     config_descriptor = get_configuration(db_vnfd, vca.get("vdu_id"))
4473                 elif vca.get("kdu_name"):
4474                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4475                     config_descriptor = get_configuration(db_vnfd, vca.get("kdu_name"))
4476                 else:
4477                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4478                     config_descriptor = get_configuration(db_vnfd, db_vnfd["id"])
4479                 vca_type = vca.get("type")
4480                 exec_terminate_primitives = not operation_params.get(
4481                     "skip_terminate_primitives"
4482                 ) and vca.get("needed_terminate")
4483                 # For helm we must destroy_ee. Also for native_charm, as juju_model cannot be deleted if there are
4484                 # pending native charms
4485                 destroy_ee = True if vca_type in ("helm-v3", "native_charm") else False
4486                 # self.logger.debug(logging_text + "vca_index: {}, ee_id: {}, vca_type: {} destroy_ee: {}".format(
4487                 #     vca_index, vca.get("ee_id"), vca_type, destroy_ee))
4488                 task = asyncio.ensure_future(
4489                     self.destroy_N2VC(
4490                         logging_text,
4491                         db_nslcmop,
4492                         vca,
4493                         config_descriptor,
4494                         vca_index,
4495                         destroy_ee,
4496                         exec_terminate_primitives,
4497                         vca_id=vca_id,
4498                     )
4499                 )
4500                 tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
4501
4502             # wait for pending tasks of terminate primitives
4503             if tasks_dict_info:
4504                 self.logger.debug(
4505                     logging_text
4506                     + "Waiting for tasks {}".format(list(tasks_dict_info.keys()))
4507                 )
4508                 error_list = await self._wait_for_tasks(
4509                     logging_text,
4510                     tasks_dict_info,
4511                     min(self.timeout.charm_delete, timeout_ns_terminate),
4512                     stage,
4513                     nslcmop_id,
4514                 )
4515                 tasks_dict_info.clear()
4516                 if error_list:
4517                     return  # raise LcmException("; ".join(error_list))
4518
4519             # remove All execution environments at once
4520             stage[0] = "Stage 3/3 delete all."
4521
4522             if nsr_deployed.get("VCA"):
4523                 stage[1] = "Deleting all execution environments."
4524                 self.logger.debug(logging_text + stage[1])
4525                 helm_vca_list = get_deployed_vca(db_nsr, {"type": "helm-v3"})
4526                 if helm_vca_list:
4527                     # Delete Namespace and Certificates
4528                     await self.vca_map["helm-v3"].delete_tls_certificate(
4529                         namespace=db_nslcmop["nsInstanceId"],
4530                         certificate_name=self.EE_TLS_NAME,
4531                     )
4532                     await self.vca_map["helm-v3"].delete_namespace(
4533                         namespace=db_nslcmop["nsInstanceId"],
4534                     )
4535                 else:
4536                     vca_id = self.get_vca_id({}, db_nsr)
4537                     task_delete_ee = asyncio.ensure_future(
4538                         asyncio.wait_for(
4539                             self._delete_all_N2VC(db_nsr=db_nsr, vca_id=vca_id),
4540                             timeout=self.timeout.charm_delete,
4541                         )
4542                     )
4543                     tasks_dict_info[task_delete_ee] = "Terminating all VCA"
4544
4545             # Delete from k8scluster
4546             stage[1] = "Deleting KDUs."
4547             self.logger.debug(logging_text + stage[1])
4548             # print(nsr_deployed)
4549             for kdu in get_iterable(nsr_deployed, "K8s"):
4550                 if not kdu or not kdu.get("kdu-instance"):
4551                     continue
4552                 kdu_instance = kdu.get("kdu-instance")
4553                 if kdu.get("k8scluster-type") in self.k8scluster_map:
4554                     # TODO: Uninstall kdu instances taking into account they could be deployed in different VIMs
4555                     vca_id = self.get_vca_id({}, db_nsr)
4556                     task_delete_kdu_instance = asyncio.ensure_future(
4557                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
4558                             cluster_uuid=kdu.get("k8scluster-uuid"),
4559                             kdu_instance=kdu_instance,
4560                             vca_id=vca_id,
4561                             namespace=kdu.get("namespace"),
4562                         )
4563                     )
4564                 else:
4565                     self.logger.error(
4566                         logging_text
4567                         + "Unknown k8s deployment type {}".format(
4568                             kdu.get("k8scluster-type")
4569                         )
4570                     )
4571                     continue
4572                 tasks_dict_info[
4573                     task_delete_kdu_instance
4574                 ] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
4575
4576             # remove from RO
4577             stage[1] = "Deleting ns from VIM."
4578             if self.ro_config.ng:
4579                 task_delete_ro = asyncio.ensure_future(
4580                     self._terminate_ng_ro(
4581                         logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
4582                     )
4583                 )
4584                 tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
4585
4586             # rest of staff will be done at finally
4587
4588         except (
4589             ROclient.ROClientException,
4590             DbException,
4591             LcmException,
4592             N2VCException,
4593         ) as e:
4594             self.logger.error(logging_text + "Exit Exception {}".format(e))
4595             exc = e
4596         except asyncio.CancelledError:
4597             self.logger.error(
4598                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
4599             )
4600             exc = "Operation was cancelled"
4601         except Exception as e:
4602             exc = traceback.format_exc()
4603             self.logger.critical(
4604                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
4605                 exc_info=True,
4606             )
4607         finally:
4608             if exc:
4609                 error_list.append(str(exc))
4610             try:
4611                 # wait for pending tasks
4612                 if tasks_dict_info:
4613                     stage[1] = "Waiting for terminate pending tasks."
4614                     self.logger.debug(logging_text + stage[1])
4615                     error_list += await self._wait_for_tasks(
4616                         logging_text,
4617                         tasks_dict_info,
4618                         timeout_ns_terminate,
4619                         stage,
4620                         nslcmop_id,
4621                     )
4622                 stage[1] = stage[2] = ""
4623             except asyncio.CancelledError:
4624                 error_list.append("Cancelled")
4625                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
4626                 await self._wait_for_tasks(
4627                     logging_text,
4628                     tasks_dict_info,
4629                     timeout_ns_terminate,
4630                     stage,
4631                     nslcmop_id,
4632                 )
4633             except Exception as exc:
4634                 error_list.append(str(exc))
4635             # update status at database
4636             if error_list:
4637                 error_detail = "; ".join(error_list)
4638                 # self.logger.error(logging_text + error_detail)
4639                 error_description_nslcmop = "{} Detail: {}".format(
4640                     stage[0], error_detail
4641                 )
4642                 error_description_nsr = "Operation: TERMINATING.{}, {}.".format(
4643                     nslcmop_id, stage[0]
4644                 )
4645
4646                 db_nsr_update["operational-status"] = "failed"
4647                 db_nsr_update["detailed-status"] = (
4648                     error_description_nsr + " Detail: " + error_detail
4649                 )
4650                 db_nslcmop_update["detailed-status"] = error_detail
4651                 nslcmop_operation_state = "FAILED"
4652                 ns_state = "BROKEN"
4653             else:
4654                 error_detail = None
4655                 error_description_nsr = error_description_nslcmop = None
4656                 ns_state = "NOT_INSTANTIATED"
4657                 db_nsr_update["operational-status"] = "terminated"
4658                 db_nsr_update["detailed-status"] = "Done"
4659                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
4660                 db_nslcmop_update["detailed-status"] = "Done"
4661                 nslcmop_operation_state = "COMPLETED"
4662
4663             if db_nsr:
4664                 self._write_ns_status(
4665                     nsr_id=nsr_id,
4666                     ns_state=ns_state,
4667                     current_operation="IDLE",
4668                     current_operation_id=None,
4669                     error_description=error_description_nsr,
4670                     error_detail=error_detail,
4671                     other_update=db_nsr_update,
4672                 )
4673             self._write_op_status(
4674                 op_id=nslcmop_id,
4675                 stage="",
4676                 error_message=error_description_nslcmop,
4677                 operation_state=nslcmop_operation_state,
4678                 other_update=db_nslcmop_update,
4679             )
4680             if ns_state == "NOT_INSTANTIATED":
4681                 try:
4682                     self.db.set_list(
4683                         "vnfrs",
4684                         {"nsr-id-ref": nsr_id},
4685                         {"_admin.nsState": "NOT_INSTANTIATED"},
4686                     )
4687                 except DbException as e:
4688                     self.logger.warn(
4689                         logging_text
4690                         + "Error writing VNFR status for nsr-id-ref: {} -> {}".format(
4691                             nsr_id, e
4692                         )
4693                     )
4694             if operation_params:
4695                 autoremove = operation_params.get("autoremove", False)
4696             if nslcmop_operation_state:
4697                 try:
4698                     await self.msg.aiowrite(
4699                         "ns",
4700                         "terminated",
4701                         {
4702                             "nsr_id": nsr_id,
4703                             "nslcmop_id": nslcmop_id,
4704                             "operationState": nslcmop_operation_state,
4705                             "autoremove": autoremove,
4706                         },
4707                     )
4708                 except Exception as e:
4709                     self.logger.error(
4710                         logging_text + "kafka_write notification Exception {}".format(e)
4711                     )
4712                 self.logger.debug(f"Deleting alerts: ns_id={nsr_id}")
4713                 self.db.del_list("alerts", {"tags.ns_id": nsr_id})
4714
4715             self.logger.debug(logging_text + "Exit")
4716             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
4717
4718     async def _wait_for_tasks(
4719         self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None
4720     ):
4721         time_start = time()
4722         error_detail_list = []
4723         error_list = []
4724         pending_tasks = list(created_tasks_info.keys())
4725         num_tasks = len(pending_tasks)
4726         num_done = 0
4727         stage[1] = "{}/{}.".format(num_done, num_tasks)
4728         self._write_op_status(nslcmop_id, stage)
4729         while pending_tasks:
4730             new_error = None
4731             _timeout = timeout + time_start - time()
4732             done, pending_tasks = await asyncio.wait(
4733                 pending_tasks, timeout=_timeout, return_when=asyncio.FIRST_COMPLETED
4734             )
4735             num_done += len(done)
4736             if not done:  # Timeout
4737                 for task in pending_tasks:
4738                     new_error = created_tasks_info[task] + ": Timeout"
4739                     error_detail_list.append(new_error)
4740                     error_list.append(new_error)
4741                 break
4742             for task in done:
4743                 if task.cancelled():
4744                     exc = "Cancelled"
4745                 else:
4746                     exc = task.exception()
4747                 if exc:
4748                     if isinstance(exc, asyncio.TimeoutError):
4749                         exc = "Timeout"
4750                     new_error = created_tasks_info[task] + ": {}".format(exc)
4751                     error_list.append(created_tasks_info[task])
4752                     error_detail_list.append(new_error)
4753                     if isinstance(
4754                         exc,
4755                         (
4756                             str,
4757                             DbException,
4758                             N2VCException,
4759                             ROclient.ROClientException,
4760                             LcmException,
4761                             K8sException,
4762                             NgRoException,
4763                         ),
4764                     ):
4765                         self.logger.error(logging_text + new_error)
4766                     else:
4767                         exc_traceback = "".join(
4768                             traceback.format_exception(None, exc, exc.__traceback__)
4769                         )
4770                         self.logger.error(
4771                             logging_text
4772                             + created_tasks_info[task]
4773                             + " "
4774                             + exc_traceback
4775                         )
4776                 else:
4777                     self.logger.debug(
4778                         logging_text + created_tasks_info[task] + ": Done"
4779                     )
4780             stage[1] = "{}/{}.".format(num_done, num_tasks)
4781             if new_error:
4782                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
4783                 if nsr_id:  # update also nsr
4784                     self.update_db_2(
4785                         "nsrs",
4786                         nsr_id,
4787                         {
4788                             "errorDescription": "Error at: " + ", ".join(error_list),
4789                             "errorDetail": ". ".join(error_detail_list),
4790                         },
4791                     )
4792             self._write_op_status(nslcmop_id, stage)
4793         return error_detail_list
4794
4795     async def _cancel_pending_tasks(self, logging_text, created_tasks_info):
4796         for task, name in created_tasks_info.items():
4797             self.logger.debug(logging_text + "Cancelling task: " + name)
4798             task.cancel()
4799
4800     @staticmethod
4801     def _map_primitive_params(primitive_desc, params, instantiation_params):
4802         """
4803         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
4804         The default-value is used. If it is between < > it look for a value at instantiation_params
4805         :param primitive_desc: portion of VNFD/NSD that describes primitive
4806         :param params: Params provided by user
4807         :param instantiation_params: Instantiation params provided by user
4808         :return: a dictionary with the calculated params
4809         """
4810         calculated_params = {}
4811         for parameter in primitive_desc.get("parameter", ()):
4812             param_name = parameter["name"]
4813             if param_name in params:
4814                 calculated_params[param_name] = params[param_name]
4815             elif "default-value" in parameter or "value" in parameter:
4816                 if "value" in parameter:
4817                     calculated_params[param_name] = parameter["value"]
4818                 else:
4819                     calculated_params[param_name] = parameter["default-value"]
4820                 if (
4821                     isinstance(calculated_params[param_name], str)
4822                     and calculated_params[param_name].startswith("<")
4823                     and calculated_params[param_name].endswith(">")
4824                 ):
4825                     if calculated_params[param_name][1:-1] in instantiation_params:
4826                         calculated_params[param_name] = instantiation_params[
4827                             calculated_params[param_name][1:-1]
4828                         ]
4829                     else:
4830                         raise LcmException(
4831                             "Parameter {} needed to execute primitive {} not provided".format(
4832                                 calculated_params[param_name], primitive_desc["name"]
4833                             )
4834                         )
4835             else:
4836                 raise LcmException(
4837                     "Parameter {} needed to execute primitive {} not provided".format(
4838                         param_name, primitive_desc["name"]
4839                     )
4840                 )
4841
4842             if isinstance(calculated_params[param_name], (dict, list, tuple)):
4843                 calculated_params[param_name] = yaml.safe_dump(
4844                     calculated_params[param_name], default_flow_style=True, width=256
4845                 )
4846             elif isinstance(calculated_params[param_name], str) and calculated_params[
4847                 param_name
4848             ].startswith("!!yaml "):
4849                 calculated_params[param_name] = calculated_params[param_name][7:]
4850             if parameter.get("data-type") == "INTEGER":
4851                 try:
4852                     calculated_params[param_name] = int(calculated_params[param_name])
4853                 except ValueError:  # error converting string to int
4854                     raise LcmException(
4855                         "Parameter {} of primitive {} must be integer".format(
4856                             param_name, primitive_desc["name"]
4857                         )
4858                     )
4859             elif parameter.get("data-type") == "BOOLEAN":
4860                 calculated_params[param_name] = not (
4861                     (str(calculated_params[param_name])).lower() == "false"
4862                 )
4863
4864         # add always ns_config_info if primitive name is config
4865         if primitive_desc["name"] == "config":
4866             if "ns_config_info" in instantiation_params:
4867                 calculated_params["ns_config_info"] = instantiation_params[
4868                     "ns_config_info"
4869                 ]
4870         return calculated_params
4871
4872     def _look_for_deployed_vca(
4873         self,
4874         deployed_vca,
4875         member_vnf_index,
4876         vdu_id,
4877         vdu_count_index,
4878         kdu_name=None,
4879         ee_descriptor_id=None,
4880     ):
4881         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
4882         for vca in deployed_vca:
4883             if not vca:
4884                 continue
4885             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
4886                 continue
4887             if (
4888                 vdu_count_index is not None
4889                 and vdu_count_index != vca["vdu_count_index"]
4890             ):
4891                 continue
4892             if kdu_name and kdu_name != vca["kdu_name"]:
4893                 continue
4894             if ee_descriptor_id and ee_descriptor_id != vca["ee_descriptor_id"]:
4895                 continue
4896             break
4897         else:
4898             # vca_deployed not found
4899             raise LcmException(
4900                 "charm for member_vnf_index={} vdu_id={}.{} kdu_name={} execution-environment-list.id={}"
4901                 " is not deployed".format(
4902                     member_vnf_index,
4903                     vdu_id,
4904                     vdu_count_index,
4905                     kdu_name,
4906                     ee_descriptor_id,
4907                 )
4908             )
4909         # get ee_id
4910         ee_id = vca.get("ee_id")
4911         vca_type = vca.get(
4912             "type", "lxc_proxy_charm"
4913         )  # default value for backward compatibility - proxy charm
4914         if not ee_id:
4915             raise LcmException(
4916                 "charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
4917                 "execution environment".format(
4918                     member_vnf_index, vdu_id, kdu_name, vdu_count_index
4919                 )
4920             )
4921         return ee_id, vca_type
4922
4923     async def _ns_execute_primitive(
4924         self,
4925         ee_id,
4926         primitive,
4927         primitive_params,
4928         retries=0,
4929         retries_interval=30,
4930         timeout=None,
4931         vca_type=None,
4932         db_dict=None,
4933         vca_id: str = None,
4934     ) -> (str, str):
4935         try:
4936             if primitive == "config":
4937                 primitive_params = {"params": primitive_params}
4938
4939             vca_type = vca_type or "lxc_proxy_charm"
4940
4941             while retries >= 0:
4942                 try:
4943                     output = await asyncio.wait_for(
4944                         self.vca_map[vca_type].exec_primitive(
4945                             ee_id=ee_id,
4946                             primitive_name=primitive,
4947                             params_dict=primitive_params,
4948                             progress_timeout=self.timeout.progress_primitive,
4949                             total_timeout=self.timeout.primitive,
4950                             db_dict=db_dict,
4951                             vca_id=vca_id,
4952                             vca_type=vca_type,
4953                         ),
4954                         timeout=timeout or self.timeout.primitive,
4955                     )
4956                     # execution was OK
4957                     break
4958                 except asyncio.CancelledError:
4959                     raise
4960                 except Exception as e:
4961                     retries -= 1
4962                     if retries >= 0:
4963                         self.logger.debug(
4964                             "Error executing action {} on {} -> {}".format(
4965                                 primitive, ee_id, e
4966                             )
4967                         )
4968                         # wait and retry
4969                         await asyncio.sleep(retries_interval)
4970                     else:
4971                         if isinstance(e, asyncio.TimeoutError):
4972                             e = N2VCException(
4973                                 message="Timed out waiting for action to complete"
4974                             )
4975                         return "FAILED", getattr(e, "message", repr(e))
4976
4977             return "COMPLETED", output
4978
4979         except (LcmException, asyncio.CancelledError):
4980             raise
4981         except Exception as e:
4982             return "FAIL", "Error executing action {}: {}".format(primitive, e)
4983
4984     async def vca_status_refresh(self, nsr_id, nslcmop_id):
4985         """
4986         Updating the vca_status with latest juju information in nsrs record
4987         :param: nsr_id: Id of the nsr
4988         :param: nslcmop_id: Id of the nslcmop
4989         :return: None
4990         """
4991
4992         self.logger.debug("Task ns={} action={} Enter".format(nsr_id, nslcmop_id))
4993         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4994         vca_id = self.get_vca_id({}, db_nsr)
4995         if db_nsr["_admin"]["deployed"]["K8s"]:
4996             for _, k8s in enumerate(db_nsr["_admin"]["deployed"]["K8s"]):
4997                 cluster_uuid, kdu_instance, cluster_type = (
4998                     k8s["k8scluster-uuid"],
4999                     k8s["kdu-instance"],
5000                     k8s["k8scluster-type"],
5001                 )
5002                 await self._on_update_k8s_db(
5003                     cluster_uuid=cluster_uuid,
5004                     kdu_instance=kdu_instance,
5005                     filter={"_id": nsr_id},
5006                     vca_id=vca_id,
5007                     cluster_type=cluster_type,
5008                 )
5009         else:
5010             for vca_index, _ in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
5011                 table, filter = "nsrs", {"_id": nsr_id}
5012                 path = "_admin.deployed.VCA.{}.".format(vca_index)
5013                 await self._on_update_n2vc_db(table, filter, path, {})
5014
5015         self.logger.debug("Task ns={} action={} Exit".format(nsr_id, nslcmop_id))
5016         self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_vca_status_refresh")
5017
5018     async def action(self, nsr_id, nslcmop_id):
5019         # Try to lock HA task here
5020         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5021         if not task_is_locked_by_me:
5022             return
5023
5024         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
5025         self.logger.debug(logging_text + "Enter")
5026         # get all needed from database
5027         db_nsr = None
5028         db_nslcmop = None
5029         db_nsr_update = {}
5030         db_nslcmop_update = {}
5031         nslcmop_operation_state = None
5032         error_description_nslcmop = None
5033         exc = None
5034         step = ""
5035         try:
5036             # wait for any previous tasks in process
5037             step = "Waiting for previous operations to terminate"
5038             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5039
5040             self._write_ns_status(
5041                 nsr_id=nsr_id,
5042                 ns_state=None,
5043                 current_operation="RUNNING ACTION",
5044                 current_operation_id=nslcmop_id,
5045             )
5046
5047             step = "Getting information from database"
5048             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5049             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5050             if db_nslcmop["operationParams"].get("primitive_params"):
5051                 db_nslcmop["operationParams"]["primitive_params"] = json.loads(
5052                     db_nslcmop["operationParams"]["primitive_params"]
5053                 )
5054
5055             nsr_deployed = db_nsr["_admin"].get("deployed")
5056             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
5057             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
5058             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
5059             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
5060             primitive = db_nslcmop["operationParams"]["primitive"]
5061             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
5062             timeout_ns_action = db_nslcmop["operationParams"].get(
5063                 "timeout_ns_action", self.timeout.primitive
5064             )
5065
5066             if vnf_index:
5067                 step = "Getting vnfr from database"
5068                 db_vnfr = self.db.get_one(
5069                     "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
5070                 )
5071                 if db_vnfr.get("kdur"):
5072                     kdur_list = []
5073                     for kdur in db_vnfr["kdur"]:
5074                         if kdur.get("additionalParams"):
5075                             kdur["additionalParams"] = json.loads(
5076                                 kdur["additionalParams"]
5077                             )
5078                         kdur_list.append(kdur)
5079                     db_vnfr["kdur"] = kdur_list
5080                 step = "Getting vnfd from database"
5081                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
5082
5083                 # Sync filesystem before running a primitive
5084                 self.fs.sync(db_vnfr["vnfd-id"])
5085             else:
5086                 step = "Getting nsd from database"
5087                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
5088
5089             vca_id = self.get_vca_id(db_vnfr, db_nsr)
5090             # for backward compatibility
5091             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
5092                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
5093                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
5094                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5095
5096             # look for primitive
5097             config_primitive_desc = descriptor_configuration = None
5098             if vdu_id:
5099                 descriptor_configuration = get_configuration(db_vnfd, vdu_id)
5100             elif kdu_name:
5101                 descriptor_configuration = get_configuration(db_vnfd, kdu_name)
5102             elif vnf_index:
5103                 descriptor_configuration = get_configuration(db_vnfd, db_vnfd["id"])
5104             else:
5105                 descriptor_configuration = db_nsd.get("ns-configuration")
5106
5107             if descriptor_configuration and descriptor_configuration.get(
5108                 "config-primitive"
5109             ):
5110                 for config_primitive in descriptor_configuration["config-primitive"]:
5111                     if config_primitive["name"] == primitive:
5112                         config_primitive_desc = config_primitive
5113                         break
5114
5115             if not config_primitive_desc:
5116                 if not (kdu_name and primitive in ("upgrade", "rollback", "status")):
5117                     raise LcmException(
5118                         "Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".format(
5119                             primitive
5120                         )
5121                     )
5122                 primitive_name = primitive
5123                 ee_descriptor_id = None
5124             else:
5125                 primitive_name = config_primitive_desc.get(
5126                     "execution-environment-primitive", primitive
5127                 )
5128                 ee_descriptor_id = config_primitive_desc.get(
5129                     "execution-environment-ref"
5130                 )
5131
5132             if vnf_index:
5133                 if vdu_id:
5134                     vdur = next(
5135                         (x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None
5136                     )
5137                     desc_params = parse_yaml_strings(vdur.get("additionalParams"))
5138                 elif kdu_name:
5139                     kdur = next(
5140                         (x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None
5141                     )
5142                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
5143                 else:
5144                     desc_params = parse_yaml_strings(
5145                         db_vnfr.get("additionalParamsForVnf")
5146                     )
5147             else:
5148                 desc_params = parse_yaml_strings(db_nsr.get("additionalParamsForNs"))
5149             if kdu_name and get_configuration(db_vnfd, kdu_name):
5150                 kdu_configuration = get_configuration(db_vnfd, kdu_name)
5151                 actions = set()
5152                 for primitive in kdu_configuration.get("initial-config-primitive", []):
5153                     actions.add(primitive["name"])
5154                 for primitive in kdu_configuration.get("config-primitive", []):
5155                     actions.add(primitive["name"])
5156                 kdu = find_in_list(
5157                     nsr_deployed["K8s"],
5158                     lambda kdu: kdu_name == kdu["kdu-name"]
5159                     and kdu["member-vnf-index"] == vnf_index,
5160                 )
5161                 kdu_action = (
5162                     True
5163                     if primitive_name in actions
5164                     and kdu["k8scluster-type"] != "helm-chart-v3"
5165                     else False
5166                 )
5167
5168             # TODO check if ns is in a proper status
5169             if kdu_name and (
5170                 primitive_name in ("upgrade", "rollback", "status") or kdu_action
5171             ):
5172                 # kdur and desc_params already set from before
5173                 if primitive_params:
5174                     desc_params.update(primitive_params)
5175                 # TODO Check if we will need something at vnf level
5176                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
5177                     if (
5178                         kdu_name == kdu["kdu-name"]
5179                         and kdu["member-vnf-index"] == vnf_index
5180                     ):
5181                         break
5182                 else:
5183                     raise LcmException(
5184                         "KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index)
5185                     )
5186
5187                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
5188                     msg = "unknown k8scluster-type '{}'".format(
5189                         kdu.get("k8scluster-type")
5190                     )
5191                     raise LcmException(msg)
5192
5193                 db_dict = {
5194                     "collection": "nsrs",
5195                     "filter": {"_id": nsr_id},
5196                     "path": "_admin.deployed.K8s.{}".format(index),
5197                 }
5198                 self.logger.debug(
5199                     logging_text
5200                     + "Exec k8s {} on {}.{}".format(primitive_name, vnf_index, kdu_name)
5201                 )
5202                 step = "Executing kdu {}".format(primitive_name)
5203                 if primitive_name == "upgrade":
5204                     if desc_params.get("kdu_model"):
5205                         kdu_model = desc_params.get("kdu_model")
5206                         del desc_params["kdu_model"]
5207                     else:
5208                         kdu_model = kdu.get("kdu-model")
5209                         if kdu_model.count("/") < 2:  # helm chart is not embedded
5210                             parts = kdu_model.split(sep=":")
5211                             if len(parts) == 2:
5212                                 kdu_model = parts[0]
5213                     if desc_params.get("kdu_atomic_upgrade"):
5214                         atomic_upgrade = desc_params.get(
5215                             "kdu_atomic_upgrade"
5216                         ).lower() in ("yes", "true", "1")
5217                         del desc_params["kdu_atomic_upgrade"]
5218                     else:
5219                         atomic_upgrade = True
5220
5221                     detailed_status = await asyncio.wait_for(
5222                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
5223                             cluster_uuid=kdu.get("k8scluster-uuid"),
5224                             kdu_instance=kdu.get("kdu-instance"),
5225                             atomic=atomic_upgrade,
5226                             kdu_model=kdu_model,
5227                             params=desc_params,
5228                             db_dict=db_dict,
5229                             timeout=timeout_ns_action,
5230                         ),
5231                         timeout=timeout_ns_action + 10,
5232                     )
5233                     self.logger.debug(
5234                         logging_text + " Upgrade of kdu {} done".format(detailed_status)
5235                     )
5236                 elif primitive_name == "rollback":
5237                     detailed_status = await asyncio.wait_for(
5238                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
5239                             cluster_uuid=kdu.get("k8scluster-uuid"),
5240                             kdu_instance=kdu.get("kdu-instance"),
5241                             db_dict=db_dict,
5242                         ),
5243                         timeout=timeout_ns_action,
5244                     )
5245                 elif primitive_name == "status":
5246                     detailed_status = await asyncio.wait_for(
5247                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
5248                             cluster_uuid=kdu.get("k8scluster-uuid"),
5249                             kdu_instance=kdu.get("kdu-instance"),
5250                             vca_id=vca_id,
5251                         ),
5252                         timeout=timeout_ns_action,
5253                     )
5254                 else:
5255                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(
5256                         kdu["kdu-name"], nsr_id
5257                     )
5258                     params = self._map_primitive_params(
5259                         config_primitive_desc, primitive_params, desc_params
5260                     )
5261
5262                     detailed_status = await asyncio.wait_for(
5263                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
5264                             cluster_uuid=kdu.get("k8scluster-uuid"),
5265                             kdu_instance=kdu_instance,
5266                             primitive_name=primitive_name,
5267                             params=params,
5268                             db_dict=db_dict,
5269                             timeout=timeout_ns_action,
5270                             vca_id=vca_id,
5271                         ),
5272                         timeout=timeout_ns_action,
5273                     )
5274
5275                 if detailed_status:
5276                     nslcmop_operation_state = "COMPLETED"
5277                 else:
5278                     detailed_status = ""
5279                     nslcmop_operation_state = "FAILED"
5280             else:
5281                 ee_id, vca_type = self._look_for_deployed_vca(
5282                     nsr_deployed["VCA"],
5283                     member_vnf_index=vnf_index,
5284                     vdu_id=vdu_id,
5285                     vdu_count_index=vdu_count_index,
5286                     ee_descriptor_id=ee_descriptor_id,
5287                 )
5288                 for vca_index, vca_deployed in enumerate(
5289                     db_nsr["_admin"]["deployed"]["VCA"]
5290                 ):
5291                     if vca_deployed.get("member-vnf-index") == vnf_index:
5292                         db_dict = {
5293                             "collection": "nsrs",
5294                             "filter": {"_id": nsr_id},
5295                             "path": "_admin.deployed.VCA.{}.".format(vca_index),
5296                         }
5297                         break
5298                 (
5299                     nslcmop_operation_state,
5300                     detailed_status,
5301                 ) = await self._ns_execute_primitive(
5302                     ee_id,
5303                     primitive=primitive_name,
5304                     primitive_params=self._map_primitive_params(
5305                         config_primitive_desc, primitive_params, desc_params
5306                     ),
5307                     timeout=timeout_ns_action,
5308                     vca_type=vca_type,
5309                     db_dict=db_dict,
5310                     vca_id=vca_id,
5311                 )
5312
5313             db_nslcmop_update["detailed-status"] = detailed_status
5314             error_description_nslcmop = (
5315                 detailed_status if nslcmop_operation_state == "FAILED" else ""
5316             )
5317             self.logger.debug(
5318                 logging_text
5319                 + "Done with result {} {}".format(
5320                     nslcmop_operation_state, detailed_status
5321                 )
5322             )
5323             return  # database update is called inside finally
5324
5325         except (DbException, LcmException, N2VCException, K8sException) as e:
5326             self.logger.error(logging_text + "Exit Exception {}".format(e))
5327             exc = e
5328         except asyncio.CancelledError:
5329             self.logger.error(
5330                 logging_text + "Cancelled Exception while '{}'".format(step)
5331             )
5332             exc = "Operation was cancelled"
5333         except asyncio.TimeoutError:
5334             self.logger.error(logging_text + "Timeout while '{}'".format(step))
5335             exc = "Timeout"
5336         except Exception as e:
5337             exc = traceback.format_exc()
5338             self.logger.critical(
5339                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
5340                 exc_info=True,
5341             )
5342         finally:
5343             if exc:
5344                 db_nslcmop_update[
5345                     "detailed-status"
5346                 ] = (
5347                     detailed_status
5348                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
5349                 nslcmop_operation_state = "FAILED"
5350             if db_nsr:
5351                 self._write_ns_status(
5352                     nsr_id=nsr_id,
5353                     ns_state=db_nsr[
5354                         "nsState"
5355                     ],  # TODO check if degraded. For the moment use previous status
5356                     current_operation="IDLE",
5357                     current_operation_id=None,
5358                     # error_description=error_description_nsr,
5359                     # error_detail=error_detail,
5360                     other_update=db_nsr_update,
5361                 )
5362
5363             self._write_op_status(
5364                 op_id=nslcmop_id,
5365                 stage="",
5366                 error_message=error_description_nslcmop,
5367                 operation_state=nslcmop_operation_state,
5368                 other_update=db_nslcmop_update,
5369             )
5370
5371             if nslcmop_operation_state:
5372                 try:
5373                     await self.msg.aiowrite(
5374                         "ns",
5375                         "actioned",
5376                         {
5377                             "nsr_id": nsr_id,
5378                             "nslcmop_id": nslcmop_id,
5379                             "operationState": nslcmop_operation_state,
5380                         },
5381                     )
5382                 except Exception as e:
5383                     self.logger.error(
5384                         logging_text + "kafka_write notification Exception {}".format(e)
5385                     )
5386             self.logger.debug(logging_text + "Exit")
5387             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
5388             return nslcmop_operation_state, detailed_status
5389
5390     async def terminate_vdus(
5391         self, db_vnfr, member_vnf_index, db_nsr, update_db_nslcmops, stage, logging_text
5392     ):
5393         """This method terminates VDUs
5394
5395         Args:
5396             db_vnfr: VNF instance record
5397             member_vnf_index: VNF index to identify the VDUs to be removed
5398             db_nsr: NS instance record
5399             update_db_nslcmops: Nslcmop update record
5400         """
5401         vca_scaling_info = []
5402         scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5403         scaling_info["scaling_direction"] = "IN"
5404         scaling_info["vdu-delete"] = {}
5405         scaling_info["kdu-delete"] = {}
5406         db_vdur = db_vnfr.get("vdur")
5407         vdur_list = copy(db_vdur)
5408         count_index = 0
5409         for index, vdu in enumerate(vdur_list):
5410             vca_scaling_info.append(
5411                 {
5412                     "osm_vdu_id": vdu["vdu-id-ref"],
5413                     "member-vnf-index": member_vnf_index,
5414                     "type": "delete",
5415                     "vdu_index": count_index,
5416                 }
5417             )
5418             scaling_info["vdu-delete"][vdu["vdu-id-ref"]] = count_index
5419             scaling_info["vdu"].append(
5420                 {
5421                     "name": vdu.get("name") or vdu.get("vdu-name"),
5422                     "vdu_id": vdu["vdu-id-ref"],
5423                     "interface": [],
5424                 }
5425             )
5426             for interface in vdu["interfaces"]:
5427                 scaling_info["vdu"][index]["interface"].append(
5428                     {
5429                         "name": interface["name"],
5430                         "ip_address": interface["ip-address"],
5431                         "mac_address": interface.get("mac-address"),
5432                     }
5433                 )
5434             self.logger.info("NS update scaling info{}".format(scaling_info))
5435             stage[2] = "Terminating VDUs"
5436             if scaling_info.get("vdu-delete"):
5437                 # scale_process = "RO"
5438                 if self.ro_config.ng:
5439                     await self._scale_ng_ro(
5440                         logging_text,
5441                         db_nsr,
5442                         update_db_nslcmops,
5443                         db_vnfr,
5444                         scaling_info,
5445                         stage,
5446                     )
5447
5448     async def remove_vnf(self, nsr_id, nslcmop_id, vnf_instance_id):
5449         """This method is to Remove VNF instances from NS.
5450
5451         Args:
5452             nsr_id: NS instance id
5453             nslcmop_id: nslcmop id of update
5454             vnf_instance_id: id of the VNF instance to be removed
5455
5456         Returns:
5457             result: (str, str) COMPLETED/FAILED, details
5458         """
5459         try:
5460             db_nsr_update = {}
5461             logging_text = "Task ns={} update ".format(nsr_id)
5462             check_vnfr_count = len(self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}))
5463             self.logger.info("check_vnfr_count {}".format(check_vnfr_count))
5464             if check_vnfr_count > 1:
5465                 stage = ["", "", ""]
5466                 step = "Getting nslcmop from database"
5467                 self.logger.debug(
5468                     step + " after having waited for previous tasks to be completed"
5469                 )
5470                 # db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5471                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5472                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
5473                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
5474                 """ db_vnfr = self.db.get_one(
5475                     "vnfrs", {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id}) """
5476
5477                 update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5478                 await self.terminate_vdus(
5479                     db_vnfr,
5480                     member_vnf_index,
5481                     db_nsr,
5482                     update_db_nslcmops,
5483                     stage,
5484                     logging_text,
5485                 )
5486
5487                 constituent_vnfr = db_nsr.get("constituent-vnfr-ref")
5488                 constituent_vnfr.remove(db_vnfr.get("_id"))
5489                 db_nsr_update["constituent-vnfr-ref"] = db_nsr.get(
5490                     "constituent-vnfr-ref"
5491                 )
5492                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5493                 self.db.del_one("vnfrs", {"_id": db_vnfr.get("_id")})
5494                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5495                 return "COMPLETED", "Done"
5496             else:
5497                 step = "Terminate VNF Failed with"
5498                 raise LcmException(
5499                     "{} Cannot terminate the last VNF in this NS.".format(
5500                         vnf_instance_id
5501                     )
5502                 )
5503         except (LcmException, asyncio.CancelledError):
5504             raise
5505         except Exception as e:
5506             self.logger.debug("Error removing VNF {}".format(e))
5507             return "FAILED", "Error removing VNF {}".format(e)
5508
5509     async def _ns_redeploy_vnf(
5510         self,
5511         nsr_id,
5512         nslcmop_id,
5513         db_vnfd,
5514         db_vnfr,
5515         db_nsr,
5516     ):
5517         """This method updates and redeploys VNF instances
5518
5519         Args:
5520             nsr_id: NS instance id
5521             nslcmop_id:   nslcmop id
5522             db_vnfd: VNF descriptor
5523             db_vnfr: VNF instance record
5524             db_nsr: NS instance record
5525
5526         Returns:
5527             result: (str, str) COMPLETED/FAILED, details
5528         """
5529         try:
5530             count_index = 0
5531             stage = ["", "", ""]
5532             logging_text = "Task ns={} update ".format(nsr_id)
5533             latest_vnfd_revision = db_vnfd["_admin"].get("revision")
5534             member_vnf_index = db_vnfr["member-vnf-index-ref"]
5535
5536             # Terminate old VNF resources
5537             update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5538             await self.terminate_vdus(
5539                 db_vnfr,
5540                 member_vnf_index,
5541                 db_nsr,
5542                 update_db_nslcmops,
5543                 stage,
5544                 logging_text,
5545             )
5546
5547             # old_vnfd_id = db_vnfr["vnfd-id"]
5548             # new_db_vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
5549             new_db_vnfd = db_vnfd
5550             # new_vnfd_ref = new_db_vnfd["id"]
5551             # new_vnfd_id = vnfd_id
5552
5553             # Create VDUR
5554             new_vnfr_cp = []
5555             for cp in new_db_vnfd.get("ext-cpd", ()):
5556                 vnf_cp = {
5557                     "name": cp.get("id"),
5558                     "connection-point-id": cp.get("int-cpd", {}).get("cpd"),
5559                     "connection-point-vdu-id": cp.get("int-cpd", {}).get("vdu-id"),
5560                     "id": cp.get("id"),
5561                 }
5562                 new_vnfr_cp.append(vnf_cp)
5563             new_vdur = update_db_nslcmops["operationParams"]["newVdur"]
5564             # new_vdur = self._create_vdur_descriptor_from_vnfd(db_nsd, db_vnfd, old_db_vnfd, vnfd_id, db_nsr, member_vnf_index)
5565             # new_vnfr_update = {"vnfd-ref": new_vnfd_ref, "vnfd-id": new_vnfd_id, "connection-point": new_vnfr_cp, "vdur": new_vdur, "ip-address": ""}
5566             new_vnfr_update = {
5567                 "revision": latest_vnfd_revision,
5568                 "connection-point": new_vnfr_cp,
5569                 "vdur": new_vdur,
5570                 "ip-address": "",
5571             }
5572             self.update_db_2("vnfrs", db_vnfr["_id"], new_vnfr_update)
5573             updated_db_vnfr = self.db.get_one(
5574                 "vnfrs",
5575                 {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id},
5576             )
5577
5578             # Instantiate new VNF resources
5579             # update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5580             vca_scaling_info = []
5581             scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5582             scaling_info["scaling_direction"] = "OUT"
5583             scaling_info["vdu-create"] = {}
5584             scaling_info["kdu-create"] = {}
5585             vdud_instantiate_list = db_vnfd["vdu"]
5586             for index, vdud in enumerate(vdud_instantiate_list):
5587                 cloud_init_text = self._get_vdu_cloud_init_content(vdud, db_vnfd)
5588                 if cloud_init_text:
5589                     additional_params = (
5590                         self._get_vdu_additional_params(updated_db_vnfr, vdud["id"])
5591                         or {}
5592                     )
5593                 cloud_init_list = []
5594                 if cloud_init_text:
5595                     # TODO Information of its own ip is not available because db_vnfr is not updated.
5596                     additional_params["OSM"] = get_osm_params(
5597                         updated_db_vnfr, vdud["id"], 1
5598                     )
5599                     cloud_init_list.append(
5600                         self._parse_cloud_init(
5601                             cloud_init_text,
5602                             additional_params,
5603                             db_vnfd["id"],
5604                             vdud["id"],
5605                         )
5606                     )
5607                     vca_scaling_info.append(
5608                         {
5609                             "osm_vdu_id": vdud["id"],
5610                             "member-vnf-index": member_vnf_index,
5611                             "type": "create",
5612                             "vdu_index": count_index,
5613                         }
5614                     )
5615                 scaling_info["vdu-create"][vdud["id"]] = count_index
5616             if self.ro_config.ng:
5617                 self.logger.debug(
5618                     "New Resources to be deployed: {}".format(scaling_info)
5619                 )
5620                 await self._scale_ng_ro(
5621                     logging_text,
5622                     db_nsr,
5623                     update_db_nslcmops,
5624                     updated_db_vnfr,
5625                     scaling_info,
5626                     stage,
5627                 )
5628                 return "COMPLETED", "Done"
5629         except (LcmException, asyncio.CancelledError):
5630             raise
5631         except Exception as e:
5632             self.logger.debug("Error updating VNF {}".format(e))
5633             return "FAILED", "Error updating VNF {}".format(e)
5634
5635     async def _ns_charm_upgrade(
5636         self,
5637         ee_id,
5638         charm_id,
5639         charm_type,
5640         path,
5641         timeout: float = None,
5642     ) -> (str, str):
5643         """This method upgrade charms in VNF instances
5644
5645         Args:
5646             ee_id:  Execution environment id
5647             path:   Local path to the charm
5648             charm_id: charm-id
5649             charm_type: Charm type can be lxc-proxy-charm, native-charm or k8s-proxy-charm
5650             timeout: (Float)    Timeout for the ns update operation
5651
5652         Returns:
5653             result: (str, str) COMPLETED/FAILED, details
5654         """
5655         try:
5656             charm_type = charm_type or "lxc_proxy_charm"
5657             output = await self.vca_map[charm_type].upgrade_charm(
5658                 ee_id=ee_id,
5659                 path=path,
5660                 charm_id=charm_id,
5661                 charm_type=charm_type,
5662                 timeout=timeout or self.timeout.ns_update,
5663             )
5664
5665             if output:
5666                 return "COMPLETED", output
5667
5668         except (LcmException, asyncio.CancelledError):
5669             raise
5670
5671         except Exception as e:
5672             self.logger.debug("Error upgrading charm {}".format(path))
5673
5674             return "FAILED", "Error upgrading charm {}: {}".format(path, e)
5675
5676     async def update(self, nsr_id, nslcmop_id):
5677         """Update NS according to different update types
5678
5679         This method performs upgrade of VNF instances then updates the revision
5680         number in VNF record
5681
5682         Args:
5683             nsr_id: Network service will be updated
5684             nslcmop_id: ns lcm operation id
5685
5686         Returns:
5687              It may raise DbException, LcmException, N2VCException, K8sException
5688
5689         """
5690         # Try to lock HA task here
5691         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5692         if not task_is_locked_by_me:
5693             return
5694
5695         logging_text = "Task ns={} update={} ".format(nsr_id, nslcmop_id)
5696         self.logger.debug(logging_text + "Enter")
5697
5698         # Set the required variables to be filled up later
5699         db_nsr = None
5700         db_nslcmop_update = {}
5701         vnfr_update = {}
5702         nslcmop_operation_state = None
5703         db_nsr_update = {}
5704         error_description_nslcmop = ""
5705         exc = None
5706         change_type = "updated"
5707         detailed_status = ""
5708         member_vnf_index = None
5709
5710         try:
5711             # wait for any previous tasks in process
5712             step = "Waiting for previous operations to terminate"
5713             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5714             self._write_ns_status(
5715                 nsr_id=nsr_id,
5716                 ns_state=None,
5717                 current_operation="UPDATING",
5718                 current_operation_id=nslcmop_id,
5719             )
5720
5721             step = "Getting nslcmop from database"
5722             db_nslcmop = self.db.get_one(
5723                 "nslcmops", {"_id": nslcmop_id}, fail_on_empty=False
5724             )
5725             update_type = db_nslcmop["operationParams"]["updateType"]
5726
5727             step = "Getting nsr from database"
5728             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5729             old_operational_status = db_nsr["operational-status"]
5730             db_nsr_update["operational-status"] = "updating"
5731             self.update_db_2("nsrs", nsr_id, db_nsr_update)
5732             nsr_deployed = db_nsr["_admin"].get("deployed")
5733
5734             if update_type == "CHANGE_VNFPKG":
5735                 # Get the input parameters given through update request
5736                 vnf_instance_id = db_nslcmop["operationParams"][
5737                     "changeVnfPackageData"
5738                 ].get("vnfInstanceId")
5739
5740                 vnfd_id = db_nslcmop["operationParams"]["changeVnfPackageData"].get(
5741                     "vnfdId"
5742                 )
5743                 timeout_seconds = db_nslcmop["operationParams"].get("timeout_ns_update")
5744
5745                 step = "Getting vnfr from database"
5746                 db_vnfr = self.db.get_one(
5747                     "vnfrs", {"_id": vnf_instance_id}, fail_on_empty=False
5748                 )
5749
5750                 step = "Getting vnfds from database"
5751                 # Latest VNFD
5752                 latest_vnfd = self.db.get_one(
5753                     "vnfds", {"_id": vnfd_id}, fail_on_empty=False
5754                 )
5755                 latest_vnfd_revision = latest_vnfd["_admin"].get("revision")
5756
5757                 # Current VNFD
5758                 current_vnf_revision = db_vnfr.get("revision", 1)
5759                 current_vnfd = self.db.get_one(
5760                     "vnfds_revisions",
5761                     {"_id": vnfd_id + ":" + str(current_vnf_revision)},
5762                     fail_on_empty=False,
5763                 )
5764                 # Charm artifact paths will be filled up later
5765                 (
5766                     current_charm_artifact_path,
5767                     target_charm_artifact_path,
5768                     charm_artifact_paths,
5769                     helm_artifacts,
5770                 ) = ([], [], [], [])
5771
5772                 step = "Checking if revision has changed in VNFD"
5773                 if current_vnf_revision != latest_vnfd_revision:
5774                     change_type = "policy_updated"
5775
5776                     # There is new revision of VNFD, update operation is required
5777                     current_vnfd_path = vnfd_id + ":" + str(current_vnf_revision)
5778                     latest_vnfd_path = vnfd_id + ":" + str(latest_vnfd_revision)
5779
5780                     step = "Removing the VNFD packages if they exist in the local path"
5781                     shutil.rmtree(self.fs.path + current_vnfd_path, ignore_errors=True)
5782                     shutil.rmtree(self.fs.path + latest_vnfd_path, ignore_errors=True)
5783
5784                     step = "Get the VNFD packages from FSMongo"
5785                     self.fs.sync(from_path=latest_vnfd_path)
5786                     self.fs.sync(from_path=current_vnfd_path)
5787
5788                     step = (
5789                         "Get the charm-type, charm-id, ee-id if there is deployed VCA"
5790                     )
5791                     current_base_folder = current_vnfd["_admin"]["storage"]
5792                     latest_base_folder = latest_vnfd["_admin"]["storage"]
5793
5794                     for vca_index, vca_deployed in enumerate(
5795                         get_iterable(nsr_deployed, "VCA")
5796                     ):
5797                         vnf_index = db_vnfr.get("member-vnf-index-ref")
5798
5799                         # Getting charm-id and charm-type
5800                         if vca_deployed.get("member-vnf-index") == vnf_index:
5801                             vca_id = self.get_vca_id(db_vnfr, db_nsr)
5802                             vca_type = vca_deployed.get("type")
5803                             vdu_count_index = vca_deployed.get("vdu_count_index")
5804
5805                             # Getting ee-id
5806                             ee_id = vca_deployed.get("ee_id")
5807
5808                             step = "Getting descriptor config"
5809                             if current_vnfd.get("kdu"):
5810                                 search_key = "kdu_name"
5811                             else:
5812                                 search_key = "vnfd_id"
5813
5814                             entity_id = vca_deployed.get(search_key)
5815
5816                             descriptor_config = get_configuration(
5817                                 current_vnfd, entity_id
5818                             )
5819
5820                             if "execution-environment-list" in descriptor_config:
5821                                 ee_list = descriptor_config.get(
5822                                     "execution-environment-list", []
5823                                 )
5824                             else:
5825                                 ee_list = []
5826
5827                             # There could be several charm used in the same VNF
5828                             for ee_item in ee_list:
5829                                 if ee_item.get("juju"):
5830                                     step = "Getting charm name"
5831                                     charm_name = ee_item["juju"].get("charm")
5832
5833                                     step = "Setting Charm artifact paths"
5834                                     current_charm_artifact_path.append(
5835                                         get_charm_artifact_path(
5836                                             current_base_folder,
5837                                             charm_name,
5838                                             vca_type,
5839                                             current_vnf_revision,
5840                                         )
5841                                     )
5842                                     target_charm_artifact_path.append(
5843                                         get_charm_artifact_path(
5844                                             latest_base_folder,
5845                                             charm_name,
5846                                             vca_type,
5847                                             latest_vnfd_revision,
5848                                         )
5849                                     )
5850                                 elif ee_item.get("helm-chart"):
5851                                     # add chart to list and all parameters
5852                                     step = "Getting helm chart name"
5853                                     chart_name = ee_item.get("helm-chart")
5854                                     vca_type = "helm-v3"
5855                                     step = "Setting Helm chart artifact paths"
5856
5857                                     helm_artifacts.append(
5858                                         {
5859                                             "current_artifact_path": get_charm_artifact_path(
5860                                                 current_base_folder,
5861                                                 chart_name,
5862                                                 vca_type,
5863                                                 current_vnf_revision,
5864                                             ),
5865                                             "target_artifact_path": get_charm_artifact_path(
5866                                                 latest_base_folder,
5867                                                 chart_name,
5868                                                 vca_type,
5869                                                 latest_vnfd_revision,
5870                                             ),
5871                                             "ee_id": ee_id,
5872                                             "vca_index": vca_index,
5873                                             "vdu_index": vdu_count_index,
5874                                         }
5875                                     )
5876
5877                             charm_artifact_paths = zip(
5878                                 current_charm_artifact_path, target_charm_artifact_path
5879                             )
5880
5881                     step = "Checking if software version has changed in VNFD"
5882                     if find_software_version(current_vnfd) != find_software_version(
5883                         latest_vnfd
5884                     ):
5885                         step = "Checking if existing VNF has charm"
5886                         for current_charm_path, target_charm_path in list(
5887                             charm_artifact_paths
5888                         ):
5889                             if current_charm_path:
5890                                 raise LcmException(
5891                                     "Software version change is not supported as VNF instance {} has charm.".format(
5892                                         vnf_instance_id
5893                                     )
5894                                 )
5895
5896                         step = "Checking whether the descriptor has SFC"
5897                         if db_nsr.get("nsd", {}).get("vnffgd"):
5898                             raise LcmException(
5899                                 "Ns update is not allowed for NS with SFC"
5900                             )
5901
5902                         # There is no change in the charm package, then redeploy the VNF
5903                         # based on new descriptor
5904                         step = "Redeploying VNF"
5905                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5906                         (result, detailed_status) = await self._ns_redeploy_vnf(
5907                             nsr_id, nslcmop_id, latest_vnfd, db_vnfr, db_nsr
5908                         )
5909                         if result == "FAILED":
5910                             nslcmop_operation_state = result
5911                             error_description_nslcmop = detailed_status
5912                             old_operational_status = "failed"
5913                         db_nslcmop_update["detailed-status"] = detailed_status
5914                         db_nsr_update["detailed-status"] = detailed_status
5915                         scaling_aspect = get_scaling_aspect(latest_vnfd)
5916                         scaling_group_desc = db_nsr.get("_admin").get(
5917                             "scaling-group", None
5918                         )
5919                         if scaling_group_desc:
5920                             for aspect in scaling_aspect:
5921                                 scaling_group_id = aspect.get("id")
5922                                 for scale_index, scaling_group in enumerate(
5923                                     scaling_group_desc
5924                                 ):
5925                                     if scaling_group.get("name") == scaling_group_id:
5926                                         db_nsr_update[
5927                                             "_admin.scaling-group.{}.nb-scale-op".format(
5928                                                 scale_index
5929                                             )
5930                                         ] = 0
5931                         self.logger.debug(
5932                             logging_text
5933                             + " step {} Done with result {} {}".format(
5934                                 step, nslcmop_operation_state, detailed_status
5935                             )
5936                         )
5937
5938                     else:
5939                         step = "Checking if any charm package has changed or not"
5940                         for current_charm_path, target_charm_path in list(
5941                             charm_artifact_paths
5942                         ):
5943                             if (
5944                                 current_charm_path
5945                                 and target_charm_path
5946                                 and self.check_charm_hash_changed(
5947                                     current_charm_path, target_charm_path
5948                                 )
5949                             ):
5950                                 step = "Checking whether VNF uses juju bundle"
5951                                 if check_juju_bundle_existence(current_vnfd):
5952                                     raise LcmException(
5953                                         "Charm upgrade is not supported for the instance which"
5954                                         " uses juju-bundle: {}".format(
5955                                             check_juju_bundle_existence(current_vnfd)
5956                                         )
5957                                     )
5958
5959                                 step = "Upgrading Charm"
5960                                 (
5961                                     result,
5962                                     detailed_status,
5963                                 ) = await self._ns_charm_upgrade(
5964                                     ee_id=ee_id,
5965                                     charm_id=vca_id,
5966                                     charm_type=vca_type,
5967                                     path=self.fs.path + target_charm_path,
5968                                     timeout=timeout_seconds,
5969                                 )
5970
5971                                 if result == "FAILED":
5972                                     nslcmop_operation_state = result
5973                                     error_description_nslcmop = detailed_status
5974
5975                                 db_nslcmop_update["detailed-status"] = detailed_status
5976                                 self.logger.debug(
5977                                     logging_text
5978                                     + " step {} Done with result {} {}".format(
5979                                         step, nslcmop_operation_state, detailed_status
5980                                     )
5981                                 )
5982
5983                         step = "Updating policies"
5984                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5985                         result = "COMPLETED"
5986                         detailed_status = "Done"
5987                         db_nslcmop_update["detailed-status"] = "Done"
5988
5989                     # helm base EE
5990                     for item in helm_artifacts:
5991                         if not (
5992                             item["current_artifact_path"]
5993                             and item["target_artifact_path"]
5994                             and self.check_charm_hash_changed(
5995                                 item["current_artifact_path"],
5996                                 item["target_artifact_path"],
5997                             )
5998                         ):
5999                             continue
6000                         db_update_entry = "_admin.deployed.VCA.{}.".format(
6001                             item["vca_index"]
6002                         )
6003                         vnfr_id = db_vnfr["_id"]
6004                         osm_config = {"osm": {"ns_id": nsr_id, "vnf_id": vnfr_id}}
6005                         db_dict = {
6006                             "collection": "nsrs",
6007                             "filter": {"_id": nsr_id},
6008                             "path": db_update_entry,
6009                         }
6010                         vca_type, namespace, helm_id = get_ee_id_parts(item["ee_id"])
6011                         await self.vca_map[vca_type].upgrade_execution_environment(
6012                             namespace=namespace,
6013                             helm_id=helm_id,
6014                             db_dict=db_dict,
6015                             config=osm_config,
6016                             artifact_path=item["target_artifact_path"],
6017                             vca_type=vca_type,
6018                         )
6019                         vnf_id = db_vnfr.get("vnfd-ref")
6020                         config_descriptor = get_configuration(latest_vnfd, vnf_id)
6021                         self.logger.debug("get ssh key block")
6022                         rw_mgmt_ip = None
6023                         if deep_get(
6024                             config_descriptor,
6025                             ("config-access", "ssh-access", "required"),
6026                         ):
6027                             # Needed to inject a ssh key
6028                             user = deep_get(
6029                                 config_descriptor,
6030                                 ("config-access", "ssh-access", "default-user"),
6031                             )
6032                             step = (
6033                                 "Install configuration Software, getting public ssh key"
6034                             )
6035                             pub_key = await self.vca_map[
6036                                 vca_type
6037                             ].get_ee_ssh_public__key(
6038                                 ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
6039                             )
6040
6041                             step = (
6042                                 "Insert public key into VM user={} ssh_key={}".format(
6043                                     user, pub_key
6044                                 )
6045                             )
6046                             self.logger.debug(logging_text + step)
6047
6048                             # wait for RO (ip-address) Insert pub_key into VM
6049                             rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
6050                                 logging_text,
6051                                 nsr_id,
6052                                 vnfr_id,
6053                                 None,
6054                                 item["vdu_index"],
6055                                 user=user,
6056                                 pub_key=pub_key,
6057                             )
6058
6059                         initial_config_primitive_list = config_descriptor.get(
6060                             "initial-config-primitive"
6061                         )
6062                         config_primitive = next(
6063                             (
6064                                 p
6065                                 for p in initial_config_primitive_list
6066                                 if p["name"] == "config"
6067                             ),
6068                             None,
6069                         )
6070                         if not config_primitive:
6071                             continue
6072
6073                         deploy_params = {"OSM": get_osm_params(db_vnfr)}
6074                         if rw_mgmt_ip:
6075                             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
6076                         if db_vnfr.get("additionalParamsForVnf"):
6077                             deploy_params.update(
6078                                 parse_yaml_strings(
6079                                     db_vnfr["additionalParamsForVnf"].copy()
6080                                 )
6081                             )
6082                         primitive_params_ = self._map_primitive_params(
6083                             config_primitive, {}, deploy_params
6084                         )
6085
6086                         step = "execute primitive '{}' params '{}'".format(
6087                             config_primitive["name"], primitive_params_
6088                         )
6089                         self.logger.debug(logging_text + step)
6090                         await self.vca_map[vca_type].exec_primitive(
6091                             ee_id=ee_id,
6092                             primitive_name=config_primitive["name"],
6093                             params_dict=primitive_params_,
6094                             db_dict=db_dict,
6095                             vca_id=vca_id,
6096                             vca_type=vca_type,
6097                         )
6098
6099                         step = "Updating policies"
6100                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
6101                         detailed_status = "Done"
6102                         db_nslcmop_update["detailed-status"] = "Done"
6103
6104                     #  If nslcmop_operation_state is None, so any operation is not failed.
6105                     if not nslcmop_operation_state:
6106                         nslcmop_operation_state = "COMPLETED"
6107
6108                         # If update CHANGE_VNFPKG nslcmop_operation is successful
6109                         # vnf revision need to be updated
6110                         vnfr_update["revision"] = latest_vnfd_revision
6111                         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
6112
6113                     self.logger.debug(
6114                         logging_text
6115                         + " task Done with result {} {}".format(
6116                             nslcmop_operation_state, detailed_status
6117                         )
6118                     )
6119             elif update_type == "REMOVE_VNF":
6120                 # This part is included in https://osm.etsi.org/gerrit/11876
6121                 vnf_instance_id = db_nslcmop["operationParams"]["removeVnfInstanceId"]
6122                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
6123                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
6124                 step = "Removing VNF"
6125                 (result, detailed_status) = await self.remove_vnf(
6126                     nsr_id, nslcmop_id, vnf_instance_id
6127                 )
6128                 if result == "FAILED":
6129                     nslcmop_operation_state = result
6130                     error_description_nslcmop = detailed_status
6131                 db_nslcmop_update["detailed-status"] = detailed_status
6132                 change_type = "vnf_terminated"
6133                 if not nslcmop_operation_state:
6134                     nslcmop_operation_state = "COMPLETED"
6135                 self.logger.debug(
6136                     logging_text
6137                     + " task Done with result {} {}".format(
6138                         nslcmop_operation_state, detailed_status
6139                     )
6140                 )
6141
6142             elif update_type == "OPERATE_VNF":
6143                 vnf_id = db_nslcmop["operationParams"]["operateVnfData"][
6144                     "vnfInstanceId"
6145                 ]
6146                 operation_type = db_nslcmop["operationParams"]["operateVnfData"][
6147                     "changeStateTo"
6148                 ]
6149                 additional_param = db_nslcmop["operationParams"]["operateVnfData"][
6150                     "additionalParam"
6151                 ]
6152                 (result, detailed_status) = await self.rebuild_start_stop(
6153                     nsr_id, nslcmop_id, vnf_id, additional_param, operation_type
6154                 )
6155                 if result == "FAILED":
6156                     nslcmop_operation_state = result
6157                     error_description_nslcmop = detailed_status
6158                 db_nslcmop_update["detailed-status"] = detailed_status
6159                 if not nslcmop_operation_state:
6160                     nslcmop_operation_state = "COMPLETED"
6161                 self.logger.debug(
6162                     logging_text
6163                     + " task Done with result {} {}".format(
6164                         nslcmop_operation_state, detailed_status
6165                     )
6166                 )
6167
6168             #  If nslcmop_operation_state is None, so any operation is not failed.
6169             #  All operations are executed in overall.
6170             if not nslcmop_operation_state:
6171                 nslcmop_operation_state = "COMPLETED"
6172             db_nsr_update["operational-status"] = old_operational_status
6173
6174         except (DbException, LcmException, N2VCException, K8sException) as e:
6175             self.logger.error(logging_text + "Exit Exception {}".format(e))
6176             exc = e
6177         except asyncio.CancelledError:
6178             self.logger.error(
6179                 logging_text + "Cancelled Exception while '{}'".format(step)
6180             )
6181             exc = "Operation was cancelled"
6182         except asyncio.TimeoutError:
6183             self.logger.error(logging_text + "Timeout while '{}'".format(step))
6184             exc = "Timeout"
6185         except Exception as e:
6186             exc = traceback.format_exc()
6187             self.logger.critical(
6188                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
6189                 exc_info=True,
6190             )
6191         finally:
6192             if exc:
6193                 db_nslcmop_update[
6194                     "detailed-status"
6195                 ] = (
6196                     detailed_status
6197                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
6198                 nslcmop_operation_state = "FAILED"
6199                 db_nsr_update["operational-status"] = old_operational_status
6200             if db_nsr:
6201                 self._write_ns_status(
6202                     nsr_id=nsr_id,
6203                     ns_state=db_nsr["nsState"],
6204                     current_operation="IDLE",
6205                     current_operation_id=None,
6206                     other_update=db_nsr_update,
6207                 )
6208
6209             self._write_op_status(
6210                 op_id=nslcmop_id,
6211                 stage="",
6212                 error_message=error_description_nslcmop,
6213                 operation_state=nslcmop_operation_state,
6214                 other_update=db_nslcmop_update,
6215             )
6216
6217             if nslcmop_operation_state:
6218                 try:
6219                     msg = {
6220                         "nsr_id": nsr_id,
6221                         "nslcmop_id": nslcmop_id,
6222                         "operationState": nslcmop_operation_state,
6223                     }
6224                     if (
6225                         change_type in ("vnf_terminated", "policy_updated")
6226                         and member_vnf_index
6227                     ):
6228                         msg.update({"vnf_member_index": member_vnf_index})
6229                     await self.msg.aiowrite("ns", change_type, msg)
6230                 except Exception as e:
6231                     self.logger.error(
6232                         logging_text + "kafka_write notification Exception {}".format(e)
6233                     )
6234             self.logger.debug(logging_text + "Exit")
6235             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_update")
6236             return nslcmop_operation_state, detailed_status
6237
6238     async def scale(self, nsr_id, nslcmop_id):
6239         # Try to lock HA task here
6240         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
6241         if not task_is_locked_by_me:
6242             return
6243
6244         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
6245         stage = ["", "", ""]
6246         tasks_dict_info = {}
6247         # ^ stage, step, VIM progress
6248         self.logger.debug(logging_text + "Enter")
6249         # get all needed from database
6250         db_nsr = None
6251         db_nslcmop_update = {}
6252         db_nsr_update = {}
6253         exc = None
6254         # in case of error, indicates what part of scale was failed to put nsr at error status
6255         scale_process = None
6256         old_operational_status = ""
6257         old_config_status = ""
6258         nsi_id = None
6259         prom_job_name = ""
6260         try:
6261             # wait for any previous tasks in process
6262             step = "Waiting for previous operations to terminate"
6263             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
6264             self._write_ns_status(
6265                 nsr_id=nsr_id,
6266                 ns_state=None,
6267                 current_operation="SCALING",
6268                 current_operation_id=nslcmop_id,
6269             )
6270
6271             step = "Getting nslcmop from database"
6272             self.logger.debug(
6273                 step + " after having waited for previous tasks to be completed"
6274             )
6275             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
6276
6277             step = "Getting nsr from database"
6278             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
6279             old_operational_status = db_nsr["operational-status"]
6280             old_config_status = db_nsr["config-status"]
6281
6282             step = "Checking whether the descriptor has SFC"
6283             if db_nsr.get("nsd", {}).get("vnffgd"):
6284                 raise LcmException("Scaling is not allowed for NS with SFC")
6285
6286             step = "Parsing scaling parameters"
6287             db_nsr_update["operational-status"] = "scaling"
6288             self.update_db_2("nsrs", nsr_id, db_nsr_update)
6289             nsr_deployed = db_nsr["_admin"].get("deployed")
6290
6291             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"][
6292                 "scaleByStepData"
6293             ]["member-vnf-index"]
6294             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"][
6295                 "scaleByStepData"
6296             ]["scaling-group-descriptor"]
6297             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
6298             # for backward compatibility
6299             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
6300                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
6301                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
6302                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6303
6304             step = "Getting vnfr from database"
6305             db_vnfr = self.db.get_one(
6306                 "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
6307             )
6308
6309             vca_id = self.get_vca_id(db_vnfr, db_nsr)
6310
6311             step = "Getting vnfd from database"
6312             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
6313
6314             base_folder = db_vnfd["_admin"]["storage"]
6315
6316             step = "Getting scaling-group-descriptor"
6317             scaling_descriptor = find_in_list(
6318                 get_scaling_aspect(db_vnfd),
6319                 lambda scale_desc: scale_desc["name"] == scaling_group,
6320             )
6321             if not scaling_descriptor:
6322                 raise LcmException(
6323                     "input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
6324                     "at vnfd:scaling-group-descriptor".format(scaling_group)
6325                 )
6326
6327             step = "Sending scale order to VIM"
6328             # TODO check if ns is in a proper status
6329             nb_scale_op = 0
6330             if not db_nsr["_admin"].get("scaling-group"):
6331                 self.update_db_2(
6332                     "nsrs",
6333                     nsr_id,
6334                     {
6335                         "_admin.scaling-group": [
6336                             {"name": scaling_group, "nb-scale-op": 0}
6337                         ]
6338                     },
6339                 )
6340                 admin_scale_index = 0
6341             else:
6342                 for admin_scale_index, admin_scale_info in enumerate(
6343                     db_nsr["_admin"]["scaling-group"]
6344                 ):
6345                     if admin_scale_info["name"] == scaling_group:
6346                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
6347                         break
6348                 else:  # not found, set index one plus last element and add new entry with the name
6349                     admin_scale_index += 1
6350                     db_nsr_update[
6351                         "_admin.scaling-group.{}.name".format(admin_scale_index)
6352                     ] = scaling_group
6353
6354             vca_scaling_info = []
6355             scaling_info = {"scaling_group_name": scaling_group, "vdu": [], "kdu": []}
6356             if scaling_type == "SCALE_OUT":
6357                 if "aspect-delta-details" not in scaling_descriptor:
6358                     raise LcmException(
6359                         "Aspect delta details not fount in scaling descriptor {}".format(
6360                             scaling_descriptor["name"]
6361                         )
6362                     )
6363                 # count if max-instance-count is reached
6364                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6365
6366                 scaling_info["scaling_direction"] = "OUT"
6367                 scaling_info["vdu-create"] = {}
6368                 scaling_info["kdu-create"] = {}
6369                 for delta in deltas:
6370                     for vdu_delta in delta.get("vdu-delta", {}):
6371                         vdud = get_vdu(db_vnfd, vdu_delta["id"])
6372                         # vdu_index also provides the number of instance of the targeted vdu
6373                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6374                         if vdu_index <= len(db_vnfr["vdur"]):
6375                             vdu_name_id = db_vnfr["vdur"][vdu_index - 1]["vdu-name"]
6376                             prom_job_name = (
6377                                 db_vnfr["_id"] + vdu_name_id + str(vdu_index - 1)
6378                             )
6379                             prom_job_name = prom_job_name.replace("_", "")
6380                             prom_job_name = prom_job_name.replace("-", "")
6381                         else:
6382                             prom_job_name = None
6383                         cloud_init_text = self._get_vdu_cloud_init_content(
6384                             vdud, db_vnfd
6385                         )
6386                         if cloud_init_text:
6387                             additional_params = (
6388                                 self._get_vdu_additional_params(db_vnfr, vdud["id"])
6389                                 or {}
6390                             )
6391                         cloud_init_list = []
6392
6393                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6394                         max_instance_count = 10
6395                         if vdu_profile and "max-number-of-instances" in vdu_profile:
6396                             max_instance_count = vdu_profile.get(
6397                                 "max-number-of-instances", 10
6398                             )
6399
6400                         default_instance_num = get_number_of_instances(
6401                             db_vnfd, vdud["id"]
6402                         )
6403                         instances_number = vdu_delta.get("number-of-instances", 1)
6404                         nb_scale_op += instances_number
6405
6406                         new_instance_count = nb_scale_op + default_instance_num
6407                         # Control if new count is over max and vdu count is less than max.
6408                         # Then assign new instance count
6409                         if new_instance_count > max_instance_count > vdu_count:
6410                             instances_number = new_instance_count - max_instance_count
6411                         else:
6412                             instances_number = instances_number
6413
6414                         if new_instance_count > max_instance_count:
6415                             raise LcmException(
6416                                 "reached the limit of {} (max-instance-count) "
6417                                 "scaling-out operations for the "
6418                                 "scaling-group-descriptor '{}'".format(
6419                                     nb_scale_op, scaling_group
6420                                 )
6421                             )
6422                         for x in range(vdu_delta.get("number-of-instances", 1)):
6423                             if cloud_init_text:
6424                                 # TODO Information of its own ip is not available because db_vnfr is not updated.
6425                                 additional_params["OSM"] = get_osm_params(
6426                                     db_vnfr, vdu_delta["id"], vdu_index + x
6427                                 )
6428                                 cloud_init_list.append(
6429                                     self._parse_cloud_init(
6430                                         cloud_init_text,
6431                                         additional_params,
6432                                         db_vnfd["id"],
6433                                         vdud["id"],
6434                                     )
6435                                 )
6436                                 vca_scaling_info.append(
6437                                     {
6438                                         "osm_vdu_id": vdu_delta["id"],
6439                                         "member-vnf-index": vnf_index,
6440                                         "type": "create",
6441                                         "vdu_index": vdu_index + x,
6442                                     }
6443                                 )
6444                         scaling_info["vdu-create"][vdu_delta["id"]] = instances_number
6445                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6446                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6447                         kdu_name = kdu_profile["kdu-name"]
6448                         resource_name = kdu_profile.get("resource-name", "")
6449
6450                         # Might have different kdus in the same delta
6451                         # Should have list for each kdu
6452                         if not scaling_info["kdu-create"].get(kdu_name, None):
6453                             scaling_info["kdu-create"][kdu_name] = []
6454
6455                         kdur = get_kdur(db_vnfr, kdu_name)
6456                         if kdur.get("helm-chart"):
6457                             k8s_cluster_type = "helm-chart-v3"
6458                             self.logger.debug("kdur: {}".format(kdur))
6459                         elif kdur.get("juju-bundle"):
6460                             k8s_cluster_type = "juju-bundle"
6461                         else:
6462                             raise LcmException(
6463                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6464                                 "juju-bundle. Maybe an old NBI version is running".format(
6465                                     db_vnfr["member-vnf-index-ref"], kdu_name
6466                                 )
6467                             )
6468
6469                         max_instance_count = 10
6470                         if kdu_profile and "max-number-of-instances" in kdu_profile:
6471                             max_instance_count = kdu_profile.get(
6472                                 "max-number-of-instances", 10
6473                             )
6474
6475                         nb_scale_op += kdu_delta.get("number-of-instances", 1)
6476                         deployed_kdu, _ = get_deployed_kdu(
6477                             nsr_deployed, kdu_name, vnf_index
6478                         )
6479                         if deployed_kdu is None:
6480                             raise LcmException(
6481                                 "KDU '{}' for vnf '{}' not deployed".format(
6482                                     kdu_name, vnf_index
6483                                 )
6484                             )
6485                         kdu_instance = deployed_kdu.get("kdu-instance")
6486                         instance_num = await self.k8scluster_map[
6487                             k8s_cluster_type
6488                         ].get_scale_count(
6489                             resource_name,
6490                             kdu_instance,
6491                             vca_id=vca_id,
6492                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6493                             kdu_model=deployed_kdu.get("kdu-model"),
6494                         )
6495                         kdu_replica_count = instance_num + kdu_delta.get(
6496                             "number-of-instances", 1
6497                         )
6498
6499                         # Control if new count is over max and instance_num is less than max.
6500                         # Then assign max instance number to kdu replica count
6501                         if kdu_replica_count > max_instance_count > instance_num:
6502                             kdu_replica_count = max_instance_count
6503                         if kdu_replica_count > max_instance_count:
6504                             raise LcmException(
6505                                 "reached the limit of {} (max-instance-count) "
6506                                 "scaling-out operations for the "
6507                                 "scaling-group-descriptor '{}'".format(
6508                                     instance_num, scaling_group
6509                                 )
6510                             )
6511
6512                         for x in range(kdu_delta.get("number-of-instances", 1)):
6513                             vca_scaling_info.append(
6514                                 {
6515                                     "osm_kdu_id": kdu_name,
6516                                     "member-vnf-index": vnf_index,
6517                                     "type": "create",
6518                                     "kdu_index": instance_num + x - 1,
6519                                 }
6520                             )
6521                         scaling_info["kdu-create"][kdu_name].append(
6522                             {
6523                                 "member-vnf-index": vnf_index,
6524                                 "type": "create",
6525                                 "k8s-cluster-type": k8s_cluster_type,
6526                                 "resource-name": resource_name,
6527                                 "scale": kdu_replica_count,
6528                             }
6529                         )
6530             elif scaling_type == "SCALE_IN":
6531                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6532
6533                 scaling_info["scaling_direction"] = "IN"
6534                 scaling_info["vdu-delete"] = {}
6535                 scaling_info["kdu-delete"] = {}
6536
6537                 for delta in deltas:
6538                     for vdu_delta in delta.get("vdu-delta", {}):
6539                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6540                         min_instance_count = 0
6541                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6542                         if vdu_profile and "min-number-of-instances" in vdu_profile:
6543                             min_instance_count = vdu_profile["min-number-of-instances"]
6544
6545                         default_instance_num = get_number_of_instances(
6546                             db_vnfd, vdu_delta["id"]
6547                         )
6548                         instance_num = vdu_delta.get("number-of-instances", 1)
6549                         nb_scale_op -= instance_num
6550
6551                         new_instance_count = nb_scale_op + default_instance_num
6552
6553                         if new_instance_count < min_instance_count < vdu_count:
6554                             instances_number = min_instance_count - new_instance_count
6555                         else:
6556                             instances_number = instance_num
6557
6558                         if new_instance_count < min_instance_count:
6559                             raise LcmException(
6560                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6561                                 "scaling-group-descriptor '{}'".format(
6562                                     nb_scale_op, scaling_group
6563                                 )
6564                             )
6565                         for x in range(vdu_delta.get("number-of-instances", 1)):
6566                             vca_scaling_info.append(
6567                                 {
6568                                     "osm_vdu_id": vdu_delta["id"],
6569                                     "member-vnf-index": vnf_index,
6570                                     "type": "delete",
6571                                     "vdu_index": vdu_index - 1 - x,
6572                                 }
6573                             )
6574                         scaling_info["vdu-delete"][vdu_delta["id"]] = instances_number
6575                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6576                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6577                         kdu_name = kdu_profile["kdu-name"]
6578                         resource_name = kdu_profile.get("resource-name", "")
6579
6580                         if not scaling_info["kdu-delete"].get(kdu_name, None):
6581                             scaling_info["kdu-delete"][kdu_name] = []
6582
6583                         kdur = get_kdur(db_vnfr, kdu_name)
6584                         if kdur.get("helm-chart"):
6585                             k8s_cluster_type = "helm-chart-v3"
6586                             self.logger.debug("kdur: {}".format(kdur))
6587                         elif kdur.get("juju-bundle"):
6588                             k8s_cluster_type = "juju-bundle"
6589                         else:
6590                             raise LcmException(
6591                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6592                                 "juju-bundle. Maybe an old NBI version is running".format(
6593                                     db_vnfr["member-vnf-index-ref"], kdur["kdu-name"]
6594                                 )
6595                             )
6596
6597                         min_instance_count = 0
6598                         if kdu_profile and "min-number-of-instances" in kdu_profile:
6599                             min_instance_count = kdu_profile["min-number-of-instances"]
6600
6601                         nb_scale_op -= kdu_delta.get("number-of-instances", 1)
6602                         deployed_kdu, _ = get_deployed_kdu(
6603                             nsr_deployed, kdu_name, vnf_index
6604                         )
6605                         if deployed_kdu is None:
6606                             raise LcmException(
6607                                 "KDU '{}' for vnf '{}' not deployed".format(
6608                                     kdu_name, vnf_index
6609                                 )
6610                             )
6611                         kdu_instance = deployed_kdu.get("kdu-instance")
6612                         instance_num = await self.k8scluster_map[
6613                             k8s_cluster_type
6614                         ].get_scale_count(
6615                             resource_name,
6616                             kdu_instance,
6617                             vca_id=vca_id,
6618                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6619                             kdu_model=deployed_kdu.get("kdu-model"),
6620                         )
6621                         kdu_replica_count = instance_num - kdu_delta.get(
6622                             "number-of-instances", 1
6623                         )
6624
6625                         if kdu_replica_count < min_instance_count < instance_num:
6626                             kdu_replica_count = min_instance_count
6627                         if kdu_replica_count < min_instance_count:
6628                             raise LcmException(
6629                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6630                                 "scaling-group-descriptor '{}'".format(
6631                                     instance_num, scaling_group
6632                                 )
6633                             )
6634
6635                         for x in range(kdu_delta.get("number-of-instances", 1)):
6636                             vca_scaling_info.append(
6637                                 {
6638                                     "osm_kdu_id": kdu_name,
6639                                     "member-vnf-index": vnf_index,
6640                                     "type": "delete",
6641                                     "kdu_index": instance_num - x - 1,
6642                                 }
6643                             )
6644                         scaling_info["kdu-delete"][kdu_name].append(
6645                             {
6646                                 "member-vnf-index": vnf_index,
6647                                 "type": "delete",
6648                                 "k8s-cluster-type": k8s_cluster_type,
6649                                 "resource-name": resource_name,
6650                                 "scale": kdu_replica_count,
6651                             }
6652                         )
6653
6654             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
6655             vdu_delete = copy(scaling_info.get("vdu-delete"))
6656             if scaling_info["scaling_direction"] == "IN":
6657                 for vdur in reversed(db_vnfr["vdur"]):
6658                     if vdu_delete.get(vdur["vdu-id-ref"]):
6659                         vdu_delete[vdur["vdu-id-ref"]] -= 1
6660                         scaling_info["vdu"].append(
6661                             {
6662                                 "name": vdur.get("name") or vdur.get("vdu-name"),
6663                                 "vdu_id": vdur["vdu-id-ref"],
6664                                 "interface": [],
6665                             }
6666                         )
6667                         for interface in vdur["interfaces"]:
6668                             scaling_info["vdu"][-1]["interface"].append(
6669                                 {
6670                                     "name": interface["name"],
6671                                     "ip_address": interface["ip-address"],
6672                                     "mac_address": interface.get("mac-address"),
6673                                 }
6674                             )
6675                 # vdu_delete = vdu_scaling_info.pop("vdu-delete")
6676
6677             # PRE-SCALE BEGIN
6678             step = "Executing pre-scale vnf-config-primitive"
6679             if scaling_descriptor.get("scaling-config-action"):
6680                 for scaling_config_action in scaling_descriptor[
6681                     "scaling-config-action"
6682                 ]:
6683                     if (
6684                         scaling_config_action.get("trigger") == "pre-scale-in"
6685                         and scaling_type == "SCALE_IN"
6686                     ) or (
6687                         scaling_config_action.get("trigger") == "pre-scale-out"
6688                         and scaling_type == "SCALE_OUT"
6689                     ):
6690                         vnf_config_primitive = scaling_config_action[
6691                             "vnf-config-primitive-name-ref"
6692                         ]
6693                         step = db_nslcmop_update[
6694                             "detailed-status"
6695                         ] = "executing pre-scale scaling-config-action '{}'".format(
6696                             vnf_config_primitive
6697                         )
6698
6699                         # look for primitive
6700                         for config_primitive in (
6701                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
6702                         ).get("config-primitive", ()):
6703                             if config_primitive["name"] == vnf_config_primitive:
6704                                 break
6705                         else:
6706                             raise LcmException(
6707                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
6708                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
6709                                 "primitive".format(scaling_group, vnf_config_primitive)
6710                             )
6711
6712                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
6713                         if db_vnfr.get("additionalParamsForVnf"):
6714                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
6715
6716                         scale_process = "VCA"
6717                         db_nsr_update["config-status"] = "configuring pre-scaling"
6718                         primitive_params = self._map_primitive_params(
6719                             config_primitive, {}, vnfr_params
6720                         )
6721
6722                         # Pre-scale retry check: Check if this sub-operation has been executed before
6723                         op_index = self._check_or_add_scale_suboperation(
6724                             db_nslcmop,
6725                             vnf_index,
6726                             vnf_config_primitive,
6727                             primitive_params,
6728                             "PRE-SCALE",
6729                         )
6730                         if op_index == self.SUBOPERATION_STATUS_SKIP:
6731                             # Skip sub-operation
6732                             result = "COMPLETED"
6733                             result_detail = "Done"
6734                             self.logger.debug(
6735                                 logging_text
6736                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
6737                                     vnf_config_primitive, result, result_detail
6738                                 )
6739                             )
6740                         else:
6741                             if op_index == self.SUBOPERATION_STATUS_NEW:
6742                                 # New sub-operation: Get index of this sub-operation
6743                                 op_index = (
6744                                     len(db_nslcmop.get("_admin", {}).get("operations"))
6745                                     - 1
6746                                 )
6747                                 self.logger.debug(
6748                                     logging_text
6749                                     + "vnf_config_primitive={} New sub-operation".format(
6750                                         vnf_config_primitive
6751                                     )
6752                                 )
6753                             else:
6754                                 # retry:  Get registered params for this existing sub-operation
6755                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
6756                                     op_index
6757                                 ]
6758                                 vnf_index = op.get("member_vnf_index")
6759                                 vnf_config_primitive = op.get("primitive")
6760                                 primitive_params = op.get("primitive_params")
6761                                 self.logger.debug(
6762                                     logging_text
6763                                     + "vnf_config_primitive={} Sub-operation retry".format(
6764                                         vnf_config_primitive
6765                                     )
6766                                 )
6767                             # Execute the primitive, either with new (first-time) or registered (reintent) args
6768                             ee_descriptor_id = config_primitive.get(
6769                                 "execution-environment-ref"
6770                             )
6771                             primitive_name = config_primitive.get(
6772                                 "execution-environment-primitive", vnf_config_primitive
6773                             )
6774                             ee_id, vca_type = self._look_for_deployed_vca(
6775                                 nsr_deployed["VCA"],
6776                                 member_vnf_index=vnf_index,
6777                                 vdu_id=None,
6778                                 vdu_count_index=None,
6779                                 ee_descriptor_id=ee_descriptor_id,
6780                             )
6781                             result, result_detail = await self._ns_execute_primitive(
6782                                 ee_id,
6783                                 primitive_name,
6784                                 primitive_params,
6785                                 vca_type=vca_type,
6786                                 vca_id=vca_id,
6787                             )
6788                             self.logger.debug(
6789                                 logging_text
6790                                 + "vnf_config_primitive={} Done with result {} {}".format(
6791                                     vnf_config_primitive, result, result_detail
6792                                 )
6793                             )
6794                             # Update operationState = COMPLETED | FAILED
6795                             self._update_suboperation_status(
6796                                 db_nslcmop, op_index, result, result_detail
6797                             )
6798
6799                         if result == "FAILED":
6800                             raise LcmException(result_detail)
6801                         db_nsr_update["config-status"] = old_config_status
6802                         scale_process = None
6803             # PRE-SCALE END
6804
6805             db_nsr_update[
6806                 "_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)
6807             ] = nb_scale_op
6808             db_nsr_update[
6809                 "_admin.scaling-group.{}.time".format(admin_scale_index)
6810             ] = time()
6811
6812             # SCALE-IN VCA - BEGIN
6813             if vca_scaling_info:
6814                 step = db_nslcmop_update[
6815                     "detailed-status"
6816                 ] = "Deleting the execution environments"
6817                 scale_process = "VCA"
6818                 for vca_info in vca_scaling_info:
6819                     if vca_info["type"] == "delete" and not vca_info.get("osm_kdu_id"):
6820                         member_vnf_index = str(vca_info["member-vnf-index"])
6821                         self.logger.debug(
6822                             logging_text + "vdu info: {}".format(vca_info)
6823                         )
6824                         if vca_info.get("osm_vdu_id"):
6825                             vdu_id = vca_info["osm_vdu_id"]
6826                             vdu_index = int(vca_info["vdu_index"])
6827                             stage[
6828                                 1
6829                             ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
6830                                 member_vnf_index, vdu_id, vdu_index
6831                             )
6832                         stage[2] = step = "Scaling in VCA"
6833                         self._write_op_status(op_id=nslcmop_id, stage=stage)
6834                         vca_update = db_nsr["_admin"]["deployed"]["VCA"]
6835                         config_update = db_nsr["configurationStatus"]
6836                         for vca_index, vca in enumerate(vca_update):
6837                             if (
6838                                 (vca or vca.get("ee_id"))
6839                                 and vca["member-vnf-index"] == member_vnf_index
6840                                 and vca["vdu_count_index"] == vdu_index
6841                             ):
6842                                 if vca.get("vdu_id"):
6843                                     config_descriptor = get_configuration(
6844                                         db_vnfd, vca.get("vdu_id")
6845                                     )
6846                                 elif vca.get("kdu_name"):
6847                                     config_descriptor = get_configuration(
6848                                         db_vnfd, vca.get("kdu_name")
6849                                     )
6850                                 else:
6851                                     config_descriptor = get_configuration(
6852                                         db_vnfd, db_vnfd["id"]
6853                                     )
6854                                 operation_params = (
6855                                     db_nslcmop.get("operationParams") or {}
6856                                 )
6857                                 exec_terminate_primitives = not operation_params.get(
6858                                     "skip_terminate_primitives"
6859                                 ) and vca.get("needed_terminate")
6860                                 task = asyncio.ensure_future(
6861                                     asyncio.wait_for(
6862                                         self.destroy_N2VC(
6863                                             logging_text,
6864                                             db_nslcmop,
6865                                             vca,
6866                                             config_descriptor,
6867                                             vca_index,
6868                                             destroy_ee=True,
6869                                             exec_primitives=exec_terminate_primitives,
6870                                             scaling_in=True,
6871                                             vca_id=vca_id,
6872                                         ),
6873                                         timeout=self.timeout.charm_delete,
6874                                     )
6875                                 )
6876                                 tasks_dict_info[task] = "Terminating VCA {}".format(
6877                                     vca.get("ee_id")
6878                                 )
6879                                 del vca_update[vca_index]
6880                                 del config_update[vca_index]
6881                         # wait for pending tasks of terminate primitives
6882                         if tasks_dict_info:
6883                             self.logger.debug(
6884                                 logging_text
6885                                 + "Waiting for tasks {}".format(
6886                                     list(tasks_dict_info.keys())
6887                                 )
6888                             )
6889                             error_list = await self._wait_for_tasks(
6890                                 logging_text,
6891                                 tasks_dict_info,
6892                                 min(
6893                                     self.timeout.charm_delete, self.timeout.ns_terminate
6894                                 ),
6895                                 stage,
6896                                 nslcmop_id,
6897                             )
6898                             tasks_dict_info.clear()
6899                             if error_list:
6900                                 raise LcmException("; ".join(error_list))
6901
6902                         db_vca_and_config_update = {
6903                             "_admin.deployed.VCA": vca_update,
6904                             "configurationStatus": config_update,
6905                         }
6906                         self.update_db_2(
6907                             "nsrs", db_nsr["_id"], db_vca_and_config_update
6908                         )
6909             scale_process = None
6910             # SCALE-IN VCA - END
6911
6912             # SCALE RO - BEGIN
6913             if scaling_info.get("vdu-create") or scaling_info.get("vdu-delete"):
6914                 scale_process = "RO"
6915                 if self.ro_config.ng:
6916                     await self._scale_ng_ro(
6917                         logging_text, db_nsr, db_nslcmop, db_vnfr, scaling_info, stage
6918                     )
6919             scaling_info.pop("vdu-create", None)
6920             scaling_info.pop("vdu-delete", None)
6921
6922             scale_process = None
6923             # SCALE RO - END
6924
6925             # SCALE KDU - BEGIN
6926             if scaling_info.get("kdu-create") or scaling_info.get("kdu-delete"):
6927                 scale_process = "KDU"
6928                 await self._scale_kdu(
6929                     logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
6930                 )
6931             scaling_info.pop("kdu-create", None)
6932             scaling_info.pop("kdu-delete", None)
6933
6934             scale_process = None
6935             # SCALE KDU - END
6936
6937             if db_nsr_update:
6938                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6939
6940             # SCALE-UP VCA - BEGIN
6941             if vca_scaling_info:
6942                 step = db_nslcmop_update[
6943                     "detailed-status"
6944                 ] = "Creating new execution environments"
6945                 scale_process = "VCA"
6946                 for vca_info in vca_scaling_info:
6947                     if vca_info["type"] == "create" and not vca_info.get("osm_kdu_id"):
6948                         member_vnf_index = str(vca_info["member-vnf-index"])
6949                         self.logger.debug(
6950                             logging_text + "vdu info: {}".format(vca_info)
6951                         )
6952                         vnfd_id = db_vnfr["vnfd-ref"]
6953                         if vca_info.get("osm_vdu_id"):
6954                             vdu_index = int(vca_info["vdu_index"])
6955                             deploy_params = {"OSM": get_osm_params(db_vnfr)}
6956                             if db_vnfr.get("additionalParamsForVnf"):
6957                                 deploy_params.update(
6958                                     parse_yaml_strings(
6959                                         db_vnfr["additionalParamsForVnf"].copy()
6960                                     )
6961                                 )
6962                             descriptor_config = get_configuration(
6963                                 db_vnfd, db_vnfd["id"]
6964                             )
6965                             if descriptor_config:
6966                                 vdu_id = None
6967                                 vdu_name = None
6968                                 kdu_name = None
6969                                 kdu_index = None
6970                                 self._deploy_n2vc(
6971                                     logging_text=logging_text
6972                                     + "member_vnf_index={} ".format(member_vnf_index),
6973                                     db_nsr=db_nsr,
6974                                     db_vnfr=db_vnfr,
6975                                     nslcmop_id=nslcmop_id,
6976                                     nsr_id=nsr_id,
6977                                     nsi_id=nsi_id,
6978                                     vnfd_id=vnfd_id,
6979                                     vdu_id=vdu_id,
6980                                     kdu_name=kdu_name,
6981                                     kdu_index=kdu_index,
6982                                     member_vnf_index=member_vnf_index,
6983                                     vdu_index=vdu_index,
6984                                     vdu_name=vdu_name,
6985                                     deploy_params=deploy_params,
6986                                     descriptor_config=descriptor_config,
6987                                     base_folder=base_folder,
6988                                     task_instantiation_info=tasks_dict_info,
6989                                     stage=stage,
6990                                 )
6991                             vdu_id = vca_info["osm_vdu_id"]
6992                             vdur = find_in_list(
6993                                 db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
6994                             )
6995                             descriptor_config = get_configuration(db_vnfd, vdu_id)
6996                             if vdur.get("additionalParams"):
6997                                 deploy_params_vdu = parse_yaml_strings(
6998                                     vdur["additionalParams"]
6999                                 )
7000                             else:
7001                                 deploy_params_vdu = deploy_params
7002                             deploy_params_vdu["OSM"] = get_osm_params(
7003                                 db_vnfr, vdu_id, vdu_count_index=vdu_index
7004                             )
7005                             if descriptor_config:
7006                                 vdu_name = None
7007                                 kdu_name = None
7008                                 kdu_index = None
7009                                 stage[
7010                                     1
7011                                 ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
7012                                     member_vnf_index, vdu_id, vdu_index
7013                                 )
7014                                 stage[2] = step = "Scaling out VCA"
7015                                 self._write_op_status(op_id=nslcmop_id, stage=stage)
7016                                 self._deploy_n2vc(
7017                                     logging_text=logging_text
7018                                     + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
7019                                         member_vnf_index, vdu_id, vdu_index
7020                                     ),
7021                                     db_nsr=db_nsr,
7022                                     db_vnfr=db_vnfr,
7023                                     nslcmop_id=nslcmop_id,
7024                                     nsr_id=nsr_id,
7025                                     nsi_id=nsi_id,
7026                                     vnfd_id=vnfd_id,
7027                                     vdu_id=vdu_id,
7028                                     kdu_name=kdu_name,
7029                                     member_vnf_index=member_vnf_index,
7030                                     vdu_index=vdu_index,
7031                                     kdu_index=kdu_index,
7032                                     vdu_name=vdu_name,
7033                                     deploy_params=deploy_params_vdu,
7034                                     descriptor_config=descriptor_config,
7035                                     base_folder=base_folder,
7036                                     task_instantiation_info=tasks_dict_info,
7037                                     stage=stage,
7038                                 )
7039             # SCALE-UP VCA - END
7040             scale_process = None
7041
7042             # POST-SCALE BEGIN
7043             # execute primitive service POST-SCALING
7044             step = "Executing post-scale vnf-config-primitive"
7045             if scaling_descriptor.get("scaling-config-action"):
7046                 for scaling_config_action in scaling_descriptor[
7047                     "scaling-config-action"
7048                 ]:
7049                     if (
7050                         scaling_config_action.get("trigger") == "post-scale-in"
7051                         and scaling_type == "SCALE_IN"
7052                     ) or (
7053                         scaling_config_action.get("trigger") == "post-scale-out"
7054                         and scaling_type == "SCALE_OUT"
7055                     ):
7056                         vnf_config_primitive = scaling_config_action[
7057                             "vnf-config-primitive-name-ref"
7058                         ]
7059                         step = db_nslcmop_update[
7060                             "detailed-status"
7061                         ] = "executing post-scale scaling-config-action '{}'".format(
7062                             vnf_config_primitive
7063                         )
7064
7065                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
7066                         if db_vnfr.get("additionalParamsForVnf"):
7067                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
7068
7069                         # look for primitive
7070                         for config_primitive in (
7071                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
7072                         ).get("config-primitive", ()):
7073                             if config_primitive["name"] == vnf_config_primitive:
7074                                 break
7075                         else:
7076                             raise LcmException(
7077                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-"
7078                                 "action[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:"
7079                                 "config-primitive".format(
7080                                     scaling_group, vnf_config_primitive
7081                                 )
7082                             )
7083                         scale_process = "VCA"
7084                         db_nsr_update["config-status"] = "configuring post-scaling"
7085                         primitive_params = self._map_primitive_params(
7086                             config_primitive, {}, vnfr_params
7087                         )
7088
7089                         # Post-scale retry check: Check if this sub-operation has been executed before
7090                         op_index = self._check_or_add_scale_suboperation(
7091                             db_nslcmop,
7092                             vnf_index,
7093                             vnf_config_primitive,
7094                             primitive_params,
7095                             "POST-SCALE",
7096                         )
7097                         if op_index == self.SUBOPERATION_STATUS_SKIP:
7098                             # Skip sub-operation
7099                             result = "COMPLETED"
7100                             result_detail = "Done"
7101                             self.logger.debug(
7102                                 logging_text
7103                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
7104                                     vnf_config_primitive, result, result_detail
7105                                 )
7106                             )
7107                         else:
7108                             if op_index == self.SUBOPERATION_STATUS_NEW:
7109                                 # New sub-operation: Get index of this sub-operation
7110                                 op_index = (
7111                                     len(db_nslcmop.get("_admin", {}).get("operations"))
7112                                     - 1
7113                                 )
7114                                 self.logger.debug(
7115                                     logging_text
7116                                     + "vnf_config_primitive={} New sub-operation".format(
7117                                         vnf_config_primitive
7118                                     )
7119                                 )
7120                             else:
7121                                 # retry:  Get registered params for this existing sub-operation
7122                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
7123                                     op_index
7124                                 ]
7125                                 vnf_index = op.get("member_vnf_index")
7126                                 vnf_config_primitive = op.get("primitive")
7127                                 primitive_params = op.get("primitive_params")
7128                                 self.logger.debug(
7129                                     logging_text
7130                                     + "vnf_config_primitive={} Sub-operation retry".format(
7131                                         vnf_config_primitive
7132                                     )
7133                                 )
7134                             # Execute the primitive, either with new (first-time) or registered (reintent) args
7135                             ee_descriptor_id = config_primitive.get(
7136                                 "execution-environment-ref"
7137                             )
7138                             primitive_name = config_primitive.get(
7139                                 "execution-environment-primitive", vnf_config_primitive
7140                             )
7141                             ee_id, vca_type = self._look_for_deployed_vca(
7142                                 nsr_deployed["VCA"],
7143                                 member_vnf_index=vnf_index,
7144                                 vdu_id=None,
7145                                 vdu_count_index=None,
7146                                 ee_descriptor_id=ee_descriptor_id,
7147                             )
7148                             result, result_detail = await self._ns_execute_primitive(
7149                                 ee_id,
7150                                 primitive_name,
7151                                 primitive_params,
7152                                 vca_type=vca_type,
7153                                 vca_id=vca_id,
7154                             )
7155                             self.logger.debug(
7156                                 logging_text
7157                                 + "vnf_config_primitive={} Done with result {} {}".format(
7158                                     vnf_config_primitive, result, result_detail
7159                                 )
7160                             )
7161                             # Update operationState = COMPLETED | FAILED
7162                             self._update_suboperation_status(
7163                                 db_nslcmop, op_index, result, result_detail
7164                             )
7165
7166                         if result == "FAILED":
7167                             raise LcmException(result_detail)
7168                         db_nsr_update["config-status"] = old_config_status
7169                         scale_process = None
7170             # POST-SCALE END
7171             # Check if each vnf has exporter for metric collection if so update prometheus job records
7172             if scaling_type == "SCALE_OUT":
7173                 if "exporters-endpoints" in db_vnfd.get("df")[0]:
7174                     vnfr_id = db_vnfr["id"]
7175                     db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
7176                     exporter_config = db_vnfd.get("df")[0].get("exporters-endpoints")
7177                     self.logger.debug("exporter config :{}".format(exporter_config))
7178                     artifact_path = "{}/{}/{}".format(
7179                         base_folder["folder"],
7180                         base_folder["pkg-dir"],
7181                         "exporter-endpoint",
7182                     )
7183                     ee_id = None
7184                     ee_config_descriptor = exporter_config
7185                     rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
7186                         logging_text,
7187                         nsr_id,
7188                         vnfr_id,
7189                         vdu_id=db_vnfr["vdur"][-1]["vdu-id-ref"],
7190                         vdu_index=db_vnfr["vdur"][-1]["count-index"],
7191                         user=None,
7192                         pub_key=None,
7193                     )
7194                     self.logger.debug("rw_mgmt_ip:{}".format(rw_mgmt_ip))
7195                     self.logger.debug("Artifact_path:{}".format(artifact_path))
7196                     vdu_id_for_prom = None
7197                     vdu_index_for_prom = None
7198                     for x in get_iterable(db_vnfr, "vdur"):
7199                         vdu_id_for_prom = x.get("vdu-id-ref")
7200                         vdu_index_for_prom = x.get("count-index")
7201                     vnfr_id = vnfr_id + vdu_id + str(vdu_index)
7202                     vnfr_id = vnfr_id.replace("_", "")
7203                     prometheus_jobs = await self.extract_prometheus_scrape_jobs(
7204                         ee_id=ee_id,
7205                         artifact_path=artifact_path,
7206                         ee_config_descriptor=ee_config_descriptor,
7207                         vnfr_id=vnfr_id,
7208                         nsr_id=nsr_id,
7209                         target_ip=rw_mgmt_ip,
7210                         element_type="VDU",
7211                         vdu_id=vdu_id_for_prom,
7212                         vdu_index=vdu_index_for_prom,
7213                     )
7214
7215                     self.logger.debug("Prometheus job:{}".format(prometheus_jobs))
7216                     if prometheus_jobs:
7217                         db_nsr_update[
7218                             "_admin.deployed.prometheus_jobs"
7219                         ] = prometheus_jobs
7220                         self.update_db_2(
7221                             "nsrs",
7222                             nsr_id,
7223                             db_nsr_update,
7224                         )
7225
7226                         for job in prometheus_jobs:
7227                             self.db.set_one(
7228                                 "prometheus_jobs",
7229                                 {"job_name": ""},
7230                                 job,
7231                                 upsert=True,
7232                                 fail_on_empty=False,
7233                             )
7234             db_nsr_update[
7235                 "detailed-status"
7236             ] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
7237             db_nsr_update["operational-status"] = (
7238                 "running"
7239                 if old_operational_status == "failed"
7240                 else old_operational_status
7241             )
7242             db_nsr_update["config-status"] = old_config_status
7243             return
7244         except (
7245             ROclient.ROClientException,
7246             DbException,
7247             LcmException,
7248             NgRoException,
7249         ) as e:
7250             self.logger.error(logging_text + "Exit Exception {}".format(e))
7251             exc = e
7252         except asyncio.CancelledError:
7253             self.logger.error(
7254                 logging_text + "Cancelled Exception while '{}'".format(step)
7255             )
7256             exc = "Operation was cancelled"
7257         except Exception as e:
7258             exc = traceback.format_exc()
7259             self.logger.critical(
7260                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
7261                 exc_info=True,
7262             )
7263         finally:
7264             error_list = list()
7265             if exc:
7266                 error_list.append(str(exc))
7267             self._write_ns_status(
7268                 nsr_id=nsr_id,
7269                 ns_state=None,
7270                 current_operation="IDLE",
7271                 current_operation_id=None,
7272             )
7273             try:
7274                 if tasks_dict_info:
7275                     stage[1] = "Waiting for instantiate pending tasks."
7276                     self.logger.debug(logging_text + stage[1])
7277                     exc = await self._wait_for_tasks(
7278                         logging_text,
7279                         tasks_dict_info,
7280                         self.timeout.ns_deploy,
7281                         stage,
7282                         nslcmop_id,
7283                         nsr_id=nsr_id,
7284                     )
7285             except asyncio.CancelledError:
7286                 error_list.append("Cancelled")
7287                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
7288                 await self._wait_for_tasks(
7289                     logging_text,
7290                     tasks_dict_info,
7291                     self.timeout.ns_deploy,
7292                     stage,
7293                     nslcmop_id,
7294                     nsr_id=nsr_id,
7295                 )
7296             if error_list:
7297                 error_detail = "; ".join(error_list)
7298                 db_nslcmop_update[
7299                     "detailed-status"
7300                 ] = error_description_nslcmop = "FAILED {}: {}".format(
7301                     step, error_detail
7302                 )
7303                 nslcmop_operation_state = "FAILED"
7304                 if db_nsr:
7305                     db_nsr_update["operational-status"] = old_operational_status
7306                     db_nsr_update["config-status"] = old_config_status
7307                     db_nsr_update["detailed-status"] = ""
7308                     if scale_process:
7309                         if "VCA" in scale_process:
7310                             db_nsr_update["config-status"] = "failed"
7311                         if "RO" in scale_process:
7312                             db_nsr_update["operational-status"] = "failed"
7313                         db_nsr_update[
7314                             "detailed-status"
7315                         ] = "FAILED scaling nslcmop={} {}: {}".format(
7316                             nslcmop_id, step, error_detail
7317                         )
7318             else:
7319                 error_description_nslcmop = None
7320                 nslcmop_operation_state = "COMPLETED"
7321                 db_nslcmop_update["detailed-status"] = "Done"
7322                 if scaling_type == "SCALE_IN" and prom_job_name is not None:
7323                     self.db.del_one(
7324                         "prometheus_jobs",
7325                         {"job_name": prom_job_name},
7326                         fail_on_empty=False,
7327                     )
7328
7329             self._write_op_status(
7330                 op_id=nslcmop_id,
7331                 stage="",
7332                 error_message=error_description_nslcmop,
7333                 operation_state=nslcmop_operation_state,
7334                 other_update=db_nslcmop_update,
7335             )
7336             if db_nsr:
7337                 self._write_ns_status(
7338                     nsr_id=nsr_id,
7339                     ns_state=None,
7340                     current_operation="IDLE",
7341                     current_operation_id=None,
7342                     other_update=db_nsr_update,
7343                 )
7344
7345             if nslcmop_operation_state:
7346                 try:
7347                     msg = {
7348                         "nsr_id": nsr_id,
7349                         "nslcmop_id": nslcmop_id,
7350                         "operationState": nslcmop_operation_state,
7351                     }
7352                     await self.msg.aiowrite("ns", "scaled", msg)
7353                 except Exception as e:
7354                     self.logger.error(
7355                         logging_text + "kafka_write notification Exception {}".format(e)
7356                     )
7357             self.logger.debug(logging_text + "Exit")
7358             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")
7359
7360     async def _scale_kdu(
7361         self, logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
7362     ):
7363         _scaling_info = scaling_info.get("kdu-create") or scaling_info.get("kdu-delete")
7364         for kdu_name in _scaling_info:
7365             for kdu_scaling_info in _scaling_info[kdu_name]:
7366                 deployed_kdu, index = get_deployed_kdu(
7367                     nsr_deployed, kdu_name, kdu_scaling_info["member-vnf-index"]
7368                 )
7369                 cluster_uuid = deployed_kdu["k8scluster-uuid"]
7370                 kdu_instance = deployed_kdu["kdu-instance"]
7371                 kdu_model = deployed_kdu.get("kdu-model")
7372                 scale = int(kdu_scaling_info["scale"])
7373                 k8s_cluster_type = kdu_scaling_info["k8s-cluster-type"]
7374
7375                 db_dict = {
7376                     "collection": "nsrs",
7377                     "filter": {"_id": nsr_id},
7378                     "path": "_admin.deployed.K8s.{}".format(index),
7379                 }
7380
7381                 step = "scaling application {}".format(
7382                     kdu_scaling_info["resource-name"]
7383                 )
7384                 self.logger.debug(logging_text + step)
7385
7386                 if kdu_scaling_info["type"] == "delete":
7387                     kdu_config = get_configuration(db_vnfd, kdu_name)
7388                     if (
7389                         kdu_config
7390                         and kdu_config.get("terminate-config-primitive")
7391                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7392                     ):
7393                         terminate_config_primitive_list = kdu_config.get(
7394                             "terminate-config-primitive"
7395                         )
7396                         terminate_config_primitive_list.sort(
7397                             key=lambda val: int(val["seq"])
7398                         )
7399
7400                         for (
7401                             terminate_config_primitive
7402                         ) in terminate_config_primitive_list:
7403                             primitive_params_ = self._map_primitive_params(
7404                                 terminate_config_primitive, {}, {}
7405                             )
7406                             step = "execute terminate config primitive"
7407                             self.logger.debug(logging_text + step)
7408                             await asyncio.wait_for(
7409                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7410                                     cluster_uuid=cluster_uuid,
7411                                     kdu_instance=kdu_instance,
7412                                     primitive_name=terminate_config_primitive["name"],
7413                                     params=primitive_params_,
7414                                     db_dict=db_dict,
7415                                     total_timeout=self.timeout.primitive,
7416                                     vca_id=vca_id,
7417                                 ),
7418                                 timeout=self.timeout.primitive
7419                                 * self.timeout.primitive_outer_factor,
7420                             )
7421
7422                 await asyncio.wait_for(
7423                     self.k8scluster_map[k8s_cluster_type].scale(
7424                         kdu_instance=kdu_instance,
7425                         scale=scale,
7426                         resource_name=kdu_scaling_info["resource-name"],
7427                         total_timeout=self.timeout.scale_on_error,
7428                         vca_id=vca_id,
7429                         cluster_uuid=cluster_uuid,
7430                         kdu_model=kdu_model,
7431                         atomic=True,
7432                         db_dict=db_dict,
7433                     ),
7434                     timeout=self.timeout.scale_on_error
7435                     * self.timeout.scale_on_error_outer_factor,
7436                 )
7437
7438                 if kdu_scaling_info["type"] == "create":
7439                     kdu_config = get_configuration(db_vnfd, kdu_name)
7440                     if (
7441                         kdu_config
7442                         and kdu_config.get("initial-config-primitive")
7443                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7444                     ):
7445                         initial_config_primitive_list = kdu_config.get(
7446                             "initial-config-primitive"
7447                         )
7448                         initial_config_primitive_list.sort(
7449                             key=lambda val: int(val["seq"])
7450                         )
7451
7452                         for initial_config_primitive in initial_config_primitive_list:
7453                             primitive_params_ = self._map_primitive_params(
7454                                 initial_config_primitive, {}, {}
7455                             )
7456                             step = "execute initial config primitive"
7457                             self.logger.debug(logging_text + step)
7458                             await asyncio.wait_for(
7459                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7460                                     cluster_uuid=cluster_uuid,
7461                                     kdu_instance=kdu_instance,
7462                                     primitive_name=initial_config_primitive["name"],
7463                                     params=primitive_params_,
7464                                     db_dict=db_dict,
7465                                     vca_id=vca_id,
7466                                 ),
7467                                 timeout=600,
7468                             )
7469
7470     async def _scale_ng_ro(
7471         self, logging_text, db_nsr, db_nslcmop, db_vnfr, vdu_scaling_info, stage
7472     ):
7473         nsr_id = db_nslcmop["nsInstanceId"]
7474         db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7475         db_vnfrs = {}
7476
7477         # read from db: vnfd's for every vnf
7478         db_vnfds = []
7479
7480         # for each vnf in ns, read vnfd
7481         for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
7482             db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
7483             vnfd_id = vnfr["vnfd-id"]  # vnfd uuid for this vnf
7484             # if we haven't this vnfd, read it from db
7485             if not find_in_list(db_vnfds, lambda a_vnfd: a_vnfd["id"] == vnfd_id):
7486                 # read from db
7487                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7488                 db_vnfds.append(vnfd)
7489         n2vc_key = self.n2vc.get_public_key()
7490         n2vc_key_list = [n2vc_key]
7491         self.scale_vnfr(
7492             db_vnfr,
7493             vdu_scaling_info.get("vdu-create"),
7494             vdu_scaling_info.get("vdu-delete"),
7495             mark_delete=True,
7496         )
7497         # db_vnfr has been updated, update db_vnfrs to use it
7498         db_vnfrs[db_vnfr["member-vnf-index-ref"]] = db_vnfr
7499         await self._instantiate_ng_ro(
7500             logging_text,
7501             nsr_id,
7502             db_nsd,
7503             db_nsr,
7504             db_nslcmop,
7505             db_vnfrs,
7506             db_vnfds,
7507             n2vc_key_list,
7508             stage=stage,
7509             start_deploy=time(),
7510             timeout_ns_deploy=self.timeout.ns_deploy,
7511         )
7512         if vdu_scaling_info.get("vdu-delete"):
7513             self.scale_vnfr(
7514                 db_vnfr, None, vdu_scaling_info["vdu-delete"], mark_delete=False
7515             )
7516
7517     async def extract_prometheus_scrape_jobs(
7518         self,
7519         ee_id: str,
7520         artifact_path: str,
7521         ee_config_descriptor: dict,
7522         vnfr_id: str,
7523         nsr_id: str,
7524         target_ip: str,
7525         element_type: str,
7526         vnf_member_index: str = "",
7527         vdu_id: str = "",
7528         vdu_index: int = None,
7529         kdu_name: str = "",
7530         kdu_index: int = None,
7531     ) -> dict:
7532         """Method to extract prometheus scrape jobs from EE's Prometheus template job file
7533             This method will wait until the corresponding VDU or KDU is fully instantiated
7534
7535         Args:
7536             ee_id (str): Execution Environment ID
7537             artifact_path (str): Path where the EE's content is (including the Prometheus template file)
7538             ee_config_descriptor (dict): Execution Environment's configuration descriptor
7539             vnfr_id (str): VNFR ID where this EE applies
7540             nsr_id (str): NSR ID where this EE applies
7541             target_ip (str): VDU/KDU instance IP address
7542             element_type (str): NS or VNF or VDU or KDU
7543             vnf_member_index (str, optional): VNF index where this EE applies. Defaults to "".
7544             vdu_id (str, optional): VDU ID where this EE applies. Defaults to "".
7545             vdu_index (int, optional): VDU index where this EE applies. Defaults to None.
7546             kdu_name (str, optional): KDU name where this EE applies. Defaults to "".
7547             kdu_index (int, optional): KDU index where this EE applies. Defaults to None.
7548
7549         Raises:
7550             LcmException: When the VDU or KDU instance was not found in an hour
7551
7552         Returns:
7553             _type_: Prometheus jobs
7554         """
7555         # default the vdur and kdur names to an empty string, to avoid any later
7556         # problem with Prometheus when the element type is not VDU or KDU
7557         vdur_name = ""
7558         kdur_name = ""
7559
7560         # look if exist a file called 'prometheus*.j2' and
7561         artifact_content = self.fs.dir_ls(artifact_path)
7562         job_file = next(
7563             (
7564                 f
7565                 for f in artifact_content
7566                 if f.startswith("prometheus") and f.endswith(".j2")
7567             ),
7568             None,
7569         )
7570         if not job_file:
7571             return
7572         self.logger.debug("Artifact path{}".format(artifact_path))
7573         self.logger.debug("job file{}".format(job_file))
7574         with self.fs.file_open((artifact_path, job_file), "r") as f:
7575             job_data = f.read()
7576
7577         # obtain the VDUR or KDUR, if the element type is VDU or KDU
7578         if element_type in ("VDU", "KDU"):
7579             for _ in range(360):
7580                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
7581                 if vdu_id and vdu_index is not None:
7582                     vdur = next(
7583                         (
7584                             x
7585                             for x in get_iterable(db_vnfr, "vdur")
7586                             if (
7587                                 x.get("vdu-id-ref") == vdu_id
7588                                 and x.get("count-index") == vdu_index
7589                             )
7590                         ),
7591                         {},
7592                     )
7593                     if vdur.get("name"):
7594                         vdur_name = vdur.get("name")
7595                         break
7596                 if kdu_name and kdu_index is not None:
7597                     kdur = next(
7598                         (
7599                             x
7600                             for x in get_iterable(db_vnfr, "kdur")
7601                             if (
7602                                 x.get("kdu-name") == kdu_name
7603                                 and x.get("count-index") == kdu_index
7604                             )
7605                         ),
7606                         {},
7607                     )
7608                     if kdur.get("name"):
7609                         kdur_name = kdur.get("name")
7610                         break
7611
7612                 await asyncio.sleep(10)
7613             else:
7614                 if vdu_id and vdu_index is not None:
7615                     raise LcmException(
7616                         f"Timeout waiting VDU with name={vdu_id} and index={vdu_index} to be intantiated"
7617                     )
7618                 if kdu_name and kdu_index is not None:
7619                     raise LcmException(
7620                         f"Timeout waiting KDU with name={kdu_name} and index={kdu_index} to be intantiated"
7621                     )
7622
7623         if ee_id is not None:
7624             _, namespace, helm_id = get_ee_id_parts(
7625                 ee_id
7626             )  # get namespace and EE gRPC service name
7627             host_name = f'{helm_id}-{ee_config_descriptor["metric-service"]}.{namespace}.svc'  # svc_name.namespace.svc
7628             host_port = "80"
7629             vnfr_id = vnfr_id.replace("-", "")
7630             variables = {
7631                 "JOB_NAME": vnfr_id,
7632                 "TARGET_IP": target_ip,
7633                 "EXPORTER_POD_IP": host_name,
7634                 "EXPORTER_POD_PORT": host_port,
7635                 "NSR_ID": nsr_id,
7636                 "VNF_MEMBER_INDEX": vnf_member_index,
7637                 "VDUR_NAME": vdur_name,
7638                 "KDUR_NAME": kdur_name,
7639                 "ELEMENT_TYPE": element_type,
7640             }
7641         else:
7642             metric_path = ee_config_descriptor["metric-path"]
7643             target_port = ee_config_descriptor["metric-port"]
7644             vnfr_id = vnfr_id.replace("-", "")
7645             variables = {
7646                 "JOB_NAME": vnfr_id,
7647                 "TARGET_IP": target_ip,
7648                 "TARGET_PORT": target_port,
7649                 "METRIC_PATH": metric_path,
7650             }
7651
7652         job_list = parse_job(job_data, variables)
7653         # ensure job_name is using the vnfr_id. Adding the metadata nsr_id
7654         for job in job_list:
7655             if (
7656                 not isinstance(job.get("job_name"), str)
7657                 or vnfr_id not in job["job_name"]
7658             ):
7659                 job["job_name"] = vnfr_id + "_" + str(SystemRandom().randint(1, 10000))
7660             job["nsr_id"] = nsr_id
7661             job["vnfr_id"] = vnfr_id
7662         return job_list
7663
7664     async def rebuild_start_stop(
7665         self, nsr_id, nslcmop_id, vnf_id, additional_param, operation_type
7666     ):
7667         logging_text = "Task ns={} {}={} ".format(nsr_id, operation_type, nslcmop_id)
7668         self.logger.info(logging_text + "Enter")
7669         stage = ["Preparing the environment", ""]
7670         # database nsrs record
7671         db_nsr_update = {}
7672         vdu_vim_name = None
7673         vim_vm_id = None
7674         # in case of error, indicates what part of scale was failed to put nsr at error status
7675         start_deploy = time()
7676         try:
7677             db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_id})
7678             vim_account_id = db_vnfr.get("vim-account-id")
7679             vim_info_key = "vim:" + vim_account_id
7680             vdu_id = additional_param["vdu_id"]
7681             vdurs = [item for item in db_vnfr["vdur"] if item["vdu-id-ref"] == vdu_id]
7682             vdur = find_in_list(
7683                 vdurs, lambda vdu: vdu["count-index"] == additional_param["count-index"]
7684             )
7685             if vdur:
7686                 vdu_vim_name = vdur["name"]
7687                 vim_vm_id = vdur["vim_info"][vim_info_key]["vim_id"]
7688                 target_vim, _ = next(k_v for k_v in vdur["vim_info"].items())
7689             else:
7690                 raise LcmException("Target vdu is not found")
7691             self.logger.info("vdu_vim_name >> {} ".format(vdu_vim_name))
7692             # wait for any previous tasks in process
7693             stage[1] = "Waiting for previous operations to terminate"
7694             self.logger.info(stage[1])
7695             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7696
7697             stage[1] = "Reading from database."
7698             self.logger.info(stage[1])
7699             self._write_ns_status(
7700                 nsr_id=nsr_id,
7701                 ns_state=None,
7702                 current_operation=operation_type.upper(),
7703                 current_operation_id=nslcmop_id,
7704             )
7705             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
7706
7707             # read from db: ns
7708             stage[1] = "Getting nsr={} from db.".format(nsr_id)
7709             db_nsr_update["operational-status"] = operation_type
7710             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7711             # Payload for RO
7712             desc = {
7713                 operation_type: {
7714                     "vim_vm_id": vim_vm_id,
7715                     "vnf_id": vnf_id,
7716                     "vdu_index": additional_param["count-index"],
7717                     "vdu_id": vdur["id"],
7718                     "target_vim": target_vim,
7719                     "vim_account_id": vim_account_id,
7720                 }
7721             }
7722             stage[1] = "Sending rebuild request to RO... {}".format(desc)
7723             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
7724             self.logger.info("ro nsr id: {}".format(nsr_id))
7725             result_dict = await self.RO.operate(nsr_id, desc, operation_type)
7726             self.logger.info("response from RO: {}".format(result_dict))
7727             action_id = result_dict["action_id"]
7728             await self._wait_ng_ro(
7729                 nsr_id,
7730                 action_id,
7731                 nslcmop_id,
7732                 start_deploy,
7733                 self.timeout.operate,
7734                 None,
7735                 "start_stop_rebuild",
7736             )
7737             return "COMPLETED", "Done"
7738         except (ROclient.ROClientException, DbException, LcmException) as e:
7739             self.logger.error("Exit Exception {}".format(e))
7740             exc = e
7741         except asyncio.CancelledError:
7742             self.logger.error("Cancelled Exception while '{}'".format(stage))
7743             exc = "Operation was cancelled"
7744         except Exception as e:
7745             exc = traceback.format_exc()
7746             self.logger.critical(
7747                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
7748             )
7749             return "FAILED", "Error in operate VNF {}".format(exc)
7750
7751     async def migrate(self, nsr_id, nslcmop_id):
7752         """
7753         Migrate VNFs and VDUs instances in a NS
7754
7755         :param: nsr_id: NS Instance ID
7756         :param: nslcmop_id: nslcmop ID of migrate
7757
7758         """
7759         # Try to lock HA task here
7760         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7761         if not task_is_locked_by_me:
7762             return
7763         logging_text = "Task ns={} migrate ".format(nsr_id)
7764         self.logger.debug(logging_text + "Enter")
7765         # get all needed from database
7766         db_nslcmop = None
7767         db_nslcmop_update = {}
7768         nslcmop_operation_state = None
7769         db_nsr_update = {}
7770         target = {}
7771         exc = None
7772         # in case of error, indicates what part of scale was failed to put nsr at error status
7773         start_deploy = time()
7774
7775         try:
7776             # wait for any previous tasks in process
7777             step = "Waiting for previous operations to terminate"
7778             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7779
7780             self._write_ns_status(
7781                 nsr_id=nsr_id,
7782                 ns_state=None,
7783                 current_operation="MIGRATING",
7784                 current_operation_id=nslcmop_id,
7785             )
7786             step = "Getting nslcmop from database"
7787             self.logger.debug(
7788                 step + " after having waited for previous tasks to be completed"
7789             )
7790             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7791             migrate_params = db_nslcmop.get("operationParams")
7792
7793             target = {}
7794             target.update(migrate_params)
7795             desc = await self.RO.migrate(nsr_id, target)
7796             self.logger.debug("RO return > {}".format(desc))
7797             action_id = desc["action_id"]
7798             await self._wait_ng_ro(
7799                 nsr_id,
7800                 action_id,
7801                 nslcmop_id,
7802                 start_deploy,
7803                 self.timeout.migrate,
7804                 operation="migrate",
7805             )
7806         except (ROclient.ROClientException, DbException, LcmException) as e:
7807             self.logger.error("Exit Exception {}".format(e))
7808             exc = e
7809         except asyncio.CancelledError:
7810             self.logger.error("Cancelled Exception while '{}'".format(step))
7811             exc = "Operation was cancelled"
7812         except Exception as e:
7813             exc = traceback.format_exc()
7814             self.logger.critical(
7815                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
7816             )
7817         finally:
7818             self._write_ns_status(
7819                 nsr_id=nsr_id,
7820                 ns_state=None,
7821                 current_operation="IDLE",
7822                 current_operation_id=None,
7823             )
7824             if exc:
7825                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
7826                 nslcmop_operation_state = "FAILED"
7827             else:
7828                 nslcmop_operation_state = "COMPLETED"
7829                 db_nslcmop_update["detailed-status"] = "Done"
7830                 db_nsr_update["detailed-status"] = "Done"
7831
7832             self._write_op_status(
7833                 op_id=nslcmop_id,
7834                 stage="",
7835                 error_message="",
7836                 operation_state=nslcmop_operation_state,
7837                 other_update=db_nslcmop_update,
7838             )
7839             if nslcmop_operation_state:
7840                 try:
7841                     msg = {
7842                         "nsr_id": nsr_id,
7843                         "nslcmop_id": nslcmop_id,
7844                         "operationState": nslcmop_operation_state,
7845                     }
7846                     await self.msg.aiowrite("ns", "migrated", msg)
7847                 except Exception as e:
7848                     self.logger.error(
7849                         logging_text + "kafka_write notification Exception {}".format(e)
7850                     )
7851             self.logger.debug(logging_text + "Exit")
7852             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_migrate")
7853
7854     async def heal(self, nsr_id, nslcmop_id):
7855         """
7856         Heal NS
7857
7858         :param nsr_id: ns instance to heal
7859         :param nslcmop_id: operation to run
7860         :return:
7861         """
7862
7863         # Try to lock HA task here
7864         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7865         if not task_is_locked_by_me:
7866             return
7867
7868         logging_text = "Task ns={} heal={} ".format(nsr_id, nslcmop_id)
7869         stage = ["", "", ""]
7870         tasks_dict_info = {}
7871         # ^ stage, step, VIM progress
7872         self.logger.debug(logging_text + "Enter")
7873         # get all needed from database
7874         db_nsr = None
7875         db_nslcmop_update = {}
7876         db_nsr_update = {}
7877         db_vnfrs = {}  # vnf's info indexed by _id
7878         exc = None
7879         old_operational_status = ""
7880         old_config_status = ""
7881         nsi_id = None
7882         try:
7883             # wait for any previous tasks in process
7884             step = "Waiting for previous operations to terminate"
7885             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7886             self._write_ns_status(
7887                 nsr_id=nsr_id,
7888                 ns_state=None,
7889                 current_operation="HEALING",
7890                 current_operation_id=nslcmop_id,
7891             )
7892
7893             step = "Getting nslcmop from database"
7894             self.logger.debug(
7895                 step + " after having waited for previous tasks to be completed"
7896             )
7897             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7898
7899             step = "Getting nsr from database"
7900             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
7901             old_operational_status = db_nsr["operational-status"]
7902             old_config_status = db_nsr["config-status"]
7903
7904             db_nsr_update = {
7905                 "_admin.deployed.RO.operational-status": "healing",
7906             }
7907             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7908
7909             step = "Sending heal order to VIM"
7910             await self.heal_RO(
7911                 logging_text=logging_text,
7912                 nsr_id=nsr_id,
7913                 db_nslcmop=db_nslcmop,
7914                 stage=stage,
7915             )
7916             # VCA tasks
7917             # read from db: nsd
7918             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
7919             self.logger.debug(logging_text + stage[1])
7920             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7921             self.fs.sync(db_nsr["nsd-id"])
7922             db_nsr["nsd"] = nsd
7923             # read from db: vnfr's of this ns
7924             step = "Getting vnfrs from db"
7925             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
7926             for vnfr in db_vnfrs_list:
7927                 db_vnfrs[vnfr["_id"]] = vnfr
7928             self.logger.debug("ns.heal db_vnfrs={}".format(db_vnfrs))
7929
7930             # Check for each target VNF
7931             target_list = db_nslcmop.get("operationParams", {}).get("healVnfData", {})
7932             for target_vnf in target_list:
7933                 # Find this VNF in the list from DB
7934                 vnfr_id = target_vnf.get("vnfInstanceId", None)
7935                 if vnfr_id:
7936                     db_vnfr = db_vnfrs[vnfr_id]
7937                     vnfd_id = db_vnfr.get("vnfd-id")
7938                     vnfd_ref = db_vnfr.get("vnfd-ref")
7939                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7940                     base_folder = vnfd["_admin"]["storage"]
7941                     vdu_id = None
7942                     vdu_index = 0
7943                     vdu_name = None
7944                     kdu_name = None
7945                     nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
7946                     member_vnf_index = db_vnfr.get("member-vnf-index-ref")
7947
7948                     # Check each target VDU and deploy N2VC
7949                     target_vdu_list = target_vnf.get("additionalParams", {}).get(
7950                         "vdu", []
7951                     )
7952                     if not target_vdu_list:
7953                         # Codigo nuevo para crear diccionario
7954                         target_vdu_list = []
7955                         for existing_vdu in db_vnfr.get("vdur"):
7956                             vdu_name = existing_vdu.get("vdu-name", None)
7957                             vdu_index = existing_vdu.get("count-index", 0)
7958                             vdu_run_day1 = target_vnf.get("additionalParams", {}).get(
7959                                 "run-day1", False
7960                             )
7961                             vdu_to_be_healed = {
7962                                 "vdu-id": vdu_name,
7963                                 "count-index": vdu_index,
7964                                 "run-day1": vdu_run_day1,
7965                             }
7966                             target_vdu_list.append(vdu_to_be_healed)
7967                     for target_vdu in target_vdu_list:
7968                         deploy_params_vdu = target_vdu
7969                         # Set run-day1 vnf level value if not vdu level value exists
7970                         if not deploy_params_vdu.get("run-day1") and target_vnf.get(
7971                             "additionalParams", {}
7972                         ).get("run-day1"):
7973                             deploy_params_vdu["run-day1"] = target_vnf[
7974                                 "additionalParams"
7975                             ].get("run-day1")
7976                         vdu_name = target_vdu.get("vdu-id", None)
7977                         # TODO: Get vdu_id from vdud.
7978                         vdu_id = vdu_name
7979                         # For multi instance VDU count-index is mandatory
7980                         # For single session VDU count-indes is 0
7981                         vdu_index = target_vdu.get("count-index", 0)
7982
7983                         # n2vc_redesign STEP 3 to 6 Deploy N2VC
7984                         stage[1] = "Deploying Execution Environments."
7985                         self.logger.debug(logging_text + stage[1])
7986
7987                         # VNF Level charm. Normal case when proxy charms.
7988                         # If target instance is management machine continue with actions: recreate EE for native charms or reinject juju key for proxy charms.
7989                         descriptor_config = get_configuration(vnfd, vnfd_ref)
7990                         if descriptor_config:
7991                             # Continue if healed machine is management machine
7992                             vnf_ip_address = db_vnfr.get("ip-address")
7993                             target_instance = None
7994                             for instance in db_vnfr.get("vdur", None):
7995                                 if (
7996                                     instance["vdu-name"] == vdu_name
7997                                     and instance["count-index"] == vdu_index
7998                                 ):
7999                                     target_instance = instance
8000                                     break
8001                             if vnf_ip_address == target_instance.get("ip-address"):
8002                                 self._heal_n2vc(
8003                                     logging_text=logging_text
8004                                     + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
8005                                         member_vnf_index, vdu_name, vdu_index
8006                                     ),
8007                                     db_nsr=db_nsr,
8008                                     db_vnfr=db_vnfr,
8009                                     nslcmop_id=nslcmop_id,
8010                                     nsr_id=nsr_id,
8011                                     nsi_id=nsi_id,
8012                                     vnfd_id=vnfd_ref,
8013                                     vdu_id=None,
8014                                     kdu_name=None,
8015                                     member_vnf_index=member_vnf_index,
8016                                     vdu_index=0,
8017                                     vdu_name=None,
8018                                     deploy_params=deploy_params_vdu,
8019                                     descriptor_config=descriptor_config,
8020                                     base_folder=base_folder,
8021                                     task_instantiation_info=tasks_dict_info,
8022                                     stage=stage,
8023                                 )
8024
8025                         # VDU Level charm. Normal case with native charms.
8026                         descriptor_config = get_configuration(vnfd, vdu_name)
8027                         if descriptor_config:
8028                             self._heal_n2vc(
8029                                 logging_text=logging_text
8030                                 + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
8031                                     member_vnf_index, vdu_name, vdu_index
8032                                 ),
8033                                 db_nsr=db_nsr,
8034                                 db_vnfr=db_vnfr,
8035                                 nslcmop_id=nslcmop_id,
8036                                 nsr_id=nsr_id,
8037                                 nsi_id=nsi_id,
8038                                 vnfd_id=vnfd_ref,
8039                                 vdu_id=vdu_id,
8040                                 kdu_name=kdu_name,
8041                                 member_vnf_index=member_vnf_index,
8042                                 vdu_index=vdu_index,
8043                                 vdu_name=vdu_name,
8044                                 deploy_params=deploy_params_vdu,
8045                                 descriptor_config=descriptor_config,
8046                                 base_folder=base_folder,
8047                                 task_instantiation_info=tasks_dict_info,
8048                                 stage=stage,
8049                             )
8050
8051         except (
8052             ROclient.ROClientException,
8053             DbException,
8054             LcmException,
8055             NgRoException,
8056         ) as e:
8057             self.logger.error(logging_text + "Exit Exception {}".format(e))
8058             exc = e
8059         except asyncio.CancelledError:
8060             self.logger.error(
8061                 logging_text + "Cancelled Exception while '{}'".format(step)
8062             )
8063             exc = "Operation was cancelled"
8064         except Exception as e:
8065             exc = traceback.format_exc()
8066             self.logger.critical(
8067                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
8068                 exc_info=True,
8069             )
8070         finally:
8071             error_list = list()
8072             if exc:
8073                 error_list.append(str(exc))
8074             try:
8075                 if tasks_dict_info:
8076                     stage[1] = "Waiting for healing pending tasks."
8077                     self.logger.debug(logging_text + stage[1])
8078                     exc = await self._wait_for_tasks(
8079                         logging_text,
8080                         tasks_dict_info,
8081                         self.timeout.ns_deploy,
8082                         stage,
8083                         nslcmop_id,
8084                         nsr_id=nsr_id,
8085                     )
8086             except asyncio.CancelledError:
8087                 error_list.append("Cancelled")
8088                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
8089                 await self._wait_for_tasks(
8090                     logging_text,
8091                     tasks_dict_info,
8092                     self.timeout.ns_deploy,
8093                     stage,
8094                     nslcmop_id,
8095                     nsr_id=nsr_id,
8096                 )
8097             if error_list:
8098                 error_detail = "; ".join(error_list)
8099                 db_nslcmop_update[
8100                     "detailed-status"
8101                 ] = error_description_nslcmop = "FAILED {}: {}".format(
8102                     step, error_detail
8103                 )
8104                 nslcmop_operation_state = "FAILED"
8105                 if db_nsr:
8106                     db_nsr_update["operational-status"] = old_operational_status
8107                     db_nsr_update["config-status"] = old_config_status
8108                     db_nsr_update[
8109                         "detailed-status"
8110                     ] = "FAILED healing nslcmop={} {}: {}".format(
8111                         nslcmop_id, step, error_detail
8112                     )
8113                     for task, task_name in tasks_dict_info.items():
8114                         if not task.done() or task.cancelled() or task.exception():
8115                             if task_name.startswith(self.task_name_deploy_vca):
8116                                 # A N2VC task is pending
8117                                 db_nsr_update["config-status"] = "failed"
8118                             else:
8119                                 # RO task is pending
8120                                 db_nsr_update["operational-status"] = "failed"
8121             else:
8122                 error_description_nslcmop = None
8123                 nslcmop_operation_state = "COMPLETED"
8124                 db_nslcmop_update["detailed-status"] = "Done"
8125                 db_nsr_update["detailed-status"] = "Done"
8126                 db_nsr_update["operational-status"] = "running"
8127                 db_nsr_update["config-status"] = "configured"
8128
8129             self._write_op_status(
8130                 op_id=nslcmop_id,
8131                 stage="",
8132                 error_message=error_description_nslcmop,
8133                 operation_state=nslcmop_operation_state,
8134                 other_update=db_nslcmop_update,
8135             )
8136             if db_nsr:
8137                 self._write_ns_status(
8138                     nsr_id=nsr_id,
8139                     ns_state=None,
8140                     current_operation="IDLE",
8141                     current_operation_id=None,
8142                     other_update=db_nsr_update,
8143                 )
8144
8145             if nslcmop_operation_state:
8146                 try:
8147                     msg = {
8148                         "nsr_id": nsr_id,
8149                         "nslcmop_id": nslcmop_id,
8150                         "operationState": nslcmop_operation_state,
8151                     }
8152                     await self.msg.aiowrite("ns", "healed", msg)
8153                 except Exception as e:
8154                     self.logger.error(
8155                         logging_text + "kafka_write notification Exception {}".format(e)
8156                     )
8157             self.logger.debug(logging_text + "Exit")
8158             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_heal")
8159
8160     async def heal_RO(
8161         self,
8162         logging_text,
8163         nsr_id,
8164         db_nslcmop,
8165         stage,
8166     ):
8167         """
8168         Heal at RO
8169         :param logging_text: preffix text to use at logging
8170         :param nsr_id: nsr identity
8171         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
8172         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
8173         :return: None or exception
8174         """
8175
8176         def get_vim_account(vim_account_id):
8177             nonlocal db_vims
8178             if vim_account_id in db_vims:
8179                 return db_vims[vim_account_id]
8180             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
8181             db_vims[vim_account_id] = db_vim
8182             return db_vim
8183
8184         try:
8185             start_heal = time()
8186             ns_params = db_nslcmop.get("operationParams")
8187             if ns_params and ns_params.get("timeout_ns_heal"):
8188                 timeout_ns_heal = ns_params["timeout_ns_heal"]
8189             else:
8190                 timeout_ns_heal = self.timeout.ns_heal
8191
8192             db_vims = {}
8193
8194             nslcmop_id = db_nslcmop["_id"]
8195             target = {
8196                 "action_id": nslcmop_id,
8197             }
8198             self.logger.warning(
8199                 "db_nslcmop={} and timeout_ns_heal={}".format(
8200                     db_nslcmop, timeout_ns_heal
8201                 )
8202             )
8203             target.update(db_nslcmop.get("operationParams", {}))
8204
8205             self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
8206             desc = await self.RO.recreate(nsr_id, target)
8207             self.logger.debug("RO return > {}".format(desc))
8208             action_id = desc["action_id"]
8209             # waits for RO to complete because Reinjecting juju key at ro can find VM in state Deleted
8210             await self._wait_ng_ro(
8211                 nsr_id,
8212                 action_id,
8213                 nslcmop_id,
8214                 start_heal,
8215                 timeout_ns_heal,
8216                 stage,
8217                 operation="healing",
8218             )
8219
8220             # Updating NSR
8221             db_nsr_update = {
8222                 "_admin.deployed.RO.operational-status": "running",
8223                 "detailed-status": " ".join(stage),
8224             }
8225             self.update_db_2("nsrs", nsr_id, db_nsr_update)
8226             self._write_op_status(nslcmop_id, stage)
8227             self.logger.debug(
8228                 logging_text + "ns healed at RO. RO_id={}".format(action_id)
8229             )
8230
8231         except Exception as e:
8232             stage[2] = "ERROR healing at VIM"
8233             # self.set_vnfr_at_error(db_vnfrs, str(e))
8234             self.logger.error(
8235                 "Error healing at VIM {}".format(e),
8236                 exc_info=not isinstance(
8237                     e,
8238                     (
8239                         ROclient.ROClientException,
8240                         LcmException,
8241                         DbException,
8242                         NgRoException,
8243                     ),
8244                 ),
8245             )
8246             raise
8247
8248     def _heal_n2vc(
8249         self,
8250         logging_text,
8251         db_nsr,
8252         db_vnfr,
8253         nslcmop_id,
8254         nsr_id,
8255         nsi_id,
8256         vnfd_id,
8257         vdu_id,
8258         kdu_name,
8259         member_vnf_index,
8260         vdu_index,
8261         vdu_name,
8262         deploy_params,
8263         descriptor_config,
8264         base_folder,
8265         task_instantiation_info,
8266         stage,
8267     ):
8268         # launch instantiate_N2VC in a asyncio task and register task object
8269         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
8270         # if not found, create one entry and update database
8271         # fill db_nsr._admin.deployed.VCA.<index>
8272
8273         self.logger.debug(
8274             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
8275         )
8276
8277         charm_name = ""
8278         get_charm_name = False
8279         if "execution-environment-list" in descriptor_config:
8280             ee_list = descriptor_config.get("execution-environment-list", [])
8281         elif "juju" in descriptor_config:
8282             ee_list = [descriptor_config]  # ns charms
8283             if "execution-environment-list" not in descriptor_config:
8284                 # charm name is only required for ns charms
8285                 get_charm_name = True
8286         else:  # other types as script are not supported
8287             ee_list = []
8288
8289         for ee_item in ee_list:
8290             self.logger.debug(
8291                 logging_text
8292                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
8293                     ee_item.get("juju"), ee_item.get("helm-chart")
8294                 )
8295             )
8296             ee_descriptor_id = ee_item.get("id")
8297             vca_name, charm_name, vca_type = self.get_vca_info(
8298                 ee_item, db_nsr, get_charm_name
8299             )
8300             if not vca_type:
8301                 self.logger.debug(
8302                     logging_text + "skipping, non juju/charm/helm configuration"
8303                 )
8304                 continue
8305
8306             vca_index = -1
8307             for vca_index, vca_deployed in enumerate(
8308                 db_nsr["_admin"]["deployed"]["VCA"]
8309             ):
8310                 if not vca_deployed:
8311                     continue
8312                 if (
8313                     vca_deployed.get("member-vnf-index") == member_vnf_index
8314                     and vca_deployed.get("vdu_id") == vdu_id
8315                     and vca_deployed.get("kdu_name") == kdu_name
8316                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
8317                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
8318                 ):
8319                     break
8320             else:
8321                 # not found, create one.
8322                 target = (
8323                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
8324                 )
8325                 if vdu_id:
8326                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
8327                 elif kdu_name:
8328                     target += "/kdu/{}".format(kdu_name)
8329                 vca_deployed = {
8330                     "target_element": target,
8331                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
8332                     "member-vnf-index": member_vnf_index,
8333                     "vdu_id": vdu_id,
8334                     "kdu_name": kdu_name,
8335                     "vdu_count_index": vdu_index,
8336                     "operational-status": "init",  # TODO revise
8337                     "detailed-status": "",  # TODO revise
8338                     "step": "initial-deploy",  # TODO revise
8339                     "vnfd_id": vnfd_id,
8340                     "vdu_name": vdu_name,
8341                     "type": vca_type,
8342                     "ee_descriptor_id": ee_descriptor_id,
8343                     "charm_name": charm_name,
8344                 }
8345                 vca_index += 1
8346
8347                 # create VCA and configurationStatus in db
8348                 db_dict = {
8349                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
8350                     "configurationStatus.{}".format(vca_index): dict(),
8351                 }
8352                 self.update_db_2("nsrs", nsr_id, db_dict)
8353
8354                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
8355
8356             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
8357             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
8358             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
8359
8360             # Launch task
8361             task_n2vc = asyncio.ensure_future(
8362                 self.heal_N2VC(
8363                     logging_text=logging_text,
8364                     vca_index=vca_index,
8365                     nsi_id=nsi_id,
8366                     db_nsr=db_nsr,
8367                     db_vnfr=db_vnfr,
8368                     vdu_id=vdu_id,
8369                     kdu_name=kdu_name,
8370                     vdu_index=vdu_index,
8371                     deploy_params=deploy_params,
8372                     config_descriptor=descriptor_config,
8373                     base_folder=base_folder,
8374                     nslcmop_id=nslcmop_id,
8375                     stage=stage,
8376                     vca_type=vca_type,
8377                     vca_name=vca_name,
8378                     ee_config_descriptor=ee_item,
8379                 )
8380             )
8381             self.lcm_tasks.register(
8382                 "ns",
8383                 nsr_id,
8384                 nslcmop_id,
8385                 "instantiate_N2VC-{}".format(vca_index),
8386                 task_n2vc,
8387             )
8388             task_instantiation_info[
8389                 task_n2vc
8390             ] = self.task_name_deploy_vca + " {}.{}".format(
8391                 member_vnf_index or "", vdu_id or ""
8392             )
8393
8394     async def heal_N2VC(
8395         self,
8396         logging_text,
8397         vca_index,
8398         nsi_id,
8399         db_nsr,
8400         db_vnfr,
8401         vdu_id,
8402         kdu_name,
8403         vdu_index,
8404         config_descriptor,
8405         deploy_params,
8406         base_folder,
8407         nslcmop_id,
8408         stage,
8409         vca_type,
8410         vca_name,
8411         ee_config_descriptor,
8412     ):
8413         nsr_id = db_nsr["_id"]
8414         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
8415         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
8416         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
8417         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
8418         db_dict = {
8419             "collection": "nsrs",
8420             "filter": {"_id": nsr_id},
8421             "path": db_update_entry,
8422         }
8423         step = ""
8424         try:
8425             element_type = "NS"
8426             element_under_configuration = nsr_id
8427
8428             vnfr_id = None
8429             if db_vnfr:
8430                 vnfr_id = db_vnfr["_id"]
8431                 osm_config["osm"]["vnf_id"] = vnfr_id
8432
8433             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
8434
8435             if vca_type == "native_charm":
8436                 index_number = 0
8437             else:
8438                 index_number = vdu_index or 0
8439
8440             if vnfr_id:
8441                 element_type = "VNF"
8442                 element_under_configuration = vnfr_id
8443                 namespace += ".{}-{}".format(vnfr_id, index_number)
8444                 if vdu_id:
8445                     namespace += ".{}-{}".format(vdu_id, index_number)
8446                     element_type = "VDU"
8447                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
8448                     osm_config["osm"]["vdu_id"] = vdu_id
8449                 elif kdu_name:
8450                     namespace += ".{}".format(kdu_name)
8451                     element_type = "KDU"
8452                     element_under_configuration = kdu_name
8453                     osm_config["osm"]["kdu_name"] = kdu_name
8454
8455             # Get artifact path
8456             if base_folder["pkg-dir"]:
8457                 artifact_path = "{}/{}/{}/{}".format(
8458                     base_folder["folder"],
8459                     base_folder["pkg-dir"],
8460                     "charms"
8461                     if vca_type
8462                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
8463                     else "helm-charts",
8464                     vca_name,
8465                 )
8466             else:
8467                 artifact_path = "{}/Scripts/{}/{}/".format(
8468                     base_folder["folder"],
8469                     "charms"
8470                     if vca_type
8471                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
8472                     else "helm-charts",
8473                     vca_name,
8474                 )
8475
8476             self.logger.debug("Artifact path > {}".format(artifact_path))
8477
8478             # get initial_config_primitive_list that applies to this element
8479             initial_config_primitive_list = config_descriptor.get(
8480                 "initial-config-primitive"
8481             )
8482
8483             self.logger.debug(
8484                 "Initial config primitive list > {}".format(
8485                     initial_config_primitive_list
8486                 )
8487             )
8488
8489             # add config if not present for NS charm
8490             ee_descriptor_id = ee_config_descriptor.get("id")
8491             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
8492             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
8493                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
8494             )
8495
8496             self.logger.debug(
8497                 "Initial config primitive list #2 > {}".format(
8498                     initial_config_primitive_list
8499                 )
8500             )
8501             # n2vc_redesign STEP 3.1
8502             # find old ee_id if exists
8503             ee_id = vca_deployed.get("ee_id")
8504
8505             vca_id = self.get_vca_id(db_vnfr, db_nsr)
8506             # create or register execution environment in VCA. Only for native charms when healing
8507             if vca_type == "native_charm":
8508                 step = "Waiting to VM being up and getting IP address"
8509                 self.logger.debug(logging_text + step)
8510                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
8511                     logging_text,
8512                     nsr_id,
8513                     vnfr_id,
8514                     vdu_id,
8515                     vdu_index,
8516                     user=None,
8517                     pub_key=None,
8518                 )
8519                 credentials = {"hostname": rw_mgmt_ip}
8520                 # get username
8521                 username = deep_get(
8522                     config_descriptor, ("config-access", "ssh-access", "default-user")
8523                 )
8524                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
8525                 #  merged. Meanwhile let's get username from initial-config-primitive
8526                 if not username and initial_config_primitive_list:
8527                     for config_primitive in initial_config_primitive_list:
8528                         for param in config_primitive.get("parameter", ()):
8529                             if param["name"] == "ssh-username":
8530                                 username = param["value"]
8531                                 break
8532                 if not username:
8533                     raise LcmException(
8534                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
8535                         "'config-access.ssh-access.default-user'"
8536                     )
8537                 credentials["username"] = username
8538
8539                 # n2vc_redesign STEP 3.2
8540                 # TODO: Before healing at RO it is needed to destroy native charm units to be deleted.
8541                 self._write_configuration_status(
8542                     nsr_id=nsr_id,
8543                     vca_index=vca_index,
8544                     status="REGISTERING",
8545                     element_under_configuration=element_under_configuration,
8546                     element_type=element_type,
8547                 )
8548
8549                 step = "register execution environment {}".format(credentials)
8550                 self.logger.debug(logging_text + step)
8551                 ee_id = await self.vca_map[vca_type].register_execution_environment(
8552                     credentials=credentials,
8553                     namespace=namespace,
8554                     db_dict=db_dict,
8555                     vca_id=vca_id,
8556                 )
8557
8558                 # update ee_id en db
8559                 db_dict_ee_id = {
8560                     "_admin.deployed.VCA.{}.ee_id".format(vca_index): ee_id,
8561                 }
8562                 self.update_db_2("nsrs", nsr_id, db_dict_ee_id)
8563
8564             # for compatibility with MON/POL modules, the need model and application name at database
8565             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
8566             # Not sure if this need to be done when healing
8567             """
8568             ee_id_parts = ee_id.split(".")
8569             db_nsr_update = {db_update_entry + "ee_id": ee_id}
8570             if len(ee_id_parts) >= 2:
8571                 model_name = ee_id_parts[0]
8572                 application_name = ee_id_parts[1]
8573                 db_nsr_update[db_update_entry + "model"] = model_name
8574                 db_nsr_update[db_update_entry + "application"] = application_name
8575             """
8576
8577             # n2vc_redesign STEP 3.3
8578             # Install configuration software. Only for native charms.
8579             step = "Install configuration Software"
8580
8581             self._write_configuration_status(
8582                 nsr_id=nsr_id,
8583                 vca_index=vca_index,
8584                 status="INSTALLING SW",
8585                 element_under_configuration=element_under_configuration,
8586                 element_type=element_type,
8587                 # other_update=db_nsr_update,
8588                 other_update=None,
8589             )
8590
8591             # TODO check if already done
8592             self.logger.debug(logging_text + step)
8593             config = None
8594             if vca_type == "native_charm":
8595                 config_primitive = next(
8596                     (p for p in initial_config_primitive_list if p["name"] == "config"),
8597                     None,
8598                 )
8599                 if config_primitive:
8600                     config = self._map_primitive_params(
8601                         config_primitive, {}, deploy_params
8602                     )
8603                 await self.vca_map[vca_type].install_configuration_sw(
8604                     ee_id=ee_id,
8605                     artifact_path=artifact_path,
8606                     db_dict=db_dict,
8607                     config=config,
8608                     num_units=1,
8609                     vca_id=vca_id,
8610                     vca_type=vca_type,
8611                 )
8612
8613             # write in db flag of configuration_sw already installed
8614             self.update_db_2(
8615                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
8616             )
8617
8618             # Not sure if this need to be done when healing
8619             """
8620             # add relations for this VCA (wait for other peers related with this VCA)
8621             await self._add_vca_relations(
8622                 logging_text=logging_text,
8623                 nsr_id=nsr_id,
8624                 vca_type=vca_type,
8625                 vca_index=vca_index,
8626             )
8627             """
8628
8629             # if SSH access is required, then get execution environment SSH public
8630             # if native charm we have waited already to VM be UP
8631             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm-v3"):
8632                 pub_key = None
8633                 user = None
8634                 # self.logger.debug("get ssh key block")
8635                 if deep_get(
8636                     config_descriptor, ("config-access", "ssh-access", "required")
8637                 ):
8638                     # self.logger.debug("ssh key needed")
8639                     # Needed to inject a ssh key
8640                     user = deep_get(
8641                         config_descriptor,
8642                         ("config-access", "ssh-access", "default-user"),
8643                     )
8644                     step = "Install configuration Software, getting public ssh key"
8645                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
8646                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
8647                     )
8648
8649                     step = "Insert public key into VM user={} ssh_key={}".format(
8650                         user, pub_key
8651                     )
8652                 else:
8653                     # self.logger.debug("no need to get ssh key")
8654                     step = "Waiting to VM being up and getting IP address"
8655                 self.logger.debug(logging_text + step)
8656
8657                 # n2vc_redesign STEP 5.1
8658                 # wait for RO (ip-address) Insert pub_key into VM
8659                 # IMPORTANT: We need do wait for RO to complete healing operation.
8660                 await self._wait_heal_ro(nsr_id, self.timeout.ns_heal)
8661                 if vnfr_id:
8662                     if kdu_name:
8663                         rw_mgmt_ip = await self.wait_kdu_up(
8664                             logging_text, nsr_id, vnfr_id, kdu_name
8665                         )
8666                     else:
8667                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
8668                             logging_text,
8669                             nsr_id,
8670                             vnfr_id,
8671                             vdu_id,
8672                             vdu_index,
8673                             user=user,
8674                             pub_key=pub_key,
8675                         )
8676                 else:
8677                     rw_mgmt_ip = None  # This is for a NS configuration
8678
8679                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
8680
8681             # store rw_mgmt_ip in deploy params for later replacement
8682             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
8683
8684             # Day1 operations.
8685             # get run-day1 operation parameter
8686             runDay1 = deploy_params.get("run-day1", False)
8687             self.logger.debug(
8688                 "Healing vnf={}, vdu={}, runDay1 ={}".format(vnfr_id, vdu_id, runDay1)
8689             )
8690             if runDay1:
8691                 # n2vc_redesign STEP 6  Execute initial config primitive
8692                 step = "execute initial config primitive"
8693
8694                 # wait for dependent primitives execution (NS -> VNF -> VDU)
8695                 if initial_config_primitive_list:
8696                     await self._wait_dependent_n2vc(
8697                         nsr_id, vca_deployed_list, vca_index
8698                     )
8699
8700                 # stage, in function of element type: vdu, kdu, vnf or ns
8701                 my_vca = vca_deployed_list[vca_index]
8702                 if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
8703                     # VDU or KDU
8704                     stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
8705                 elif my_vca.get("member-vnf-index"):
8706                     # VNF
8707                     stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
8708                 else:
8709                     # NS
8710                     stage[0] = "Stage 5/5: running Day-1 primitives for NS."
8711
8712                 self._write_configuration_status(
8713                     nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
8714                 )
8715
8716                 self._write_op_status(op_id=nslcmop_id, stage=stage)
8717
8718                 check_if_terminated_needed = True
8719                 for initial_config_primitive in initial_config_primitive_list:
8720                     # adding information on the vca_deployed if it is a NS execution environment
8721                     if not vca_deployed["member-vnf-index"]:
8722                         deploy_params["ns_config_info"] = json.dumps(
8723                             self._get_ns_config_info(nsr_id)
8724                         )
8725                     # TODO check if already done
8726                     primitive_params_ = self._map_primitive_params(
8727                         initial_config_primitive, {}, deploy_params
8728                     )
8729
8730                     step = "execute primitive '{}' params '{}'".format(
8731                         initial_config_primitive["name"], primitive_params_
8732                     )
8733                     self.logger.debug(logging_text + step)
8734                     await self.vca_map[vca_type].exec_primitive(
8735                         ee_id=ee_id,
8736                         primitive_name=initial_config_primitive["name"],
8737                         params_dict=primitive_params_,
8738                         db_dict=db_dict,
8739                         vca_id=vca_id,
8740                         vca_type=vca_type,
8741                     )
8742                     # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
8743                     if check_if_terminated_needed:
8744                         if config_descriptor.get("terminate-config-primitive"):
8745                             self.update_db_2(
8746                                 "nsrs",
8747                                 nsr_id,
8748                                 {db_update_entry + "needed_terminate": True},
8749                             )
8750                         check_if_terminated_needed = False
8751
8752                     # TODO register in database that primitive is done
8753
8754             # STEP 7 Configure metrics
8755             # Not sure if this need to be done when healing
8756             """
8757             if vca_type == "helm" or vca_type == "helm-v3":
8758                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
8759                     ee_id=ee_id,
8760                     artifact_path=artifact_path,
8761                     ee_config_descriptor=ee_config_descriptor,
8762                     vnfr_id=vnfr_id,
8763                     nsr_id=nsr_id,
8764                     target_ip=rw_mgmt_ip,
8765                 )
8766                 if prometheus_jobs:
8767                     self.update_db_2(
8768                         "nsrs",
8769                         nsr_id,
8770                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
8771                     )
8772
8773                     for job in prometheus_jobs:
8774                         self.db.set_one(
8775                             "prometheus_jobs",
8776                             {"job_name": job["job_name"]},
8777                             job,
8778                             upsert=True,
8779                             fail_on_empty=False,
8780                         )
8781
8782             """
8783             step = "instantiated at VCA"
8784             self.logger.debug(logging_text + step)
8785
8786             self._write_configuration_status(
8787                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
8788             )
8789
8790         except Exception as e:  # TODO not use Exception but N2VC exception
8791             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
8792             if not isinstance(
8793                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
8794             ):
8795                 self.logger.error(
8796                     "Exception while {} : {}".format(step, e), exc_info=True
8797                 )
8798             self._write_configuration_status(
8799                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
8800             )
8801             raise LcmException("{} {}".format(step, e)) from e
8802
8803     async def _wait_heal_ro(
8804         self,
8805         nsr_id,
8806         timeout=600,
8807     ):
8808         start_time = time()
8809         while time() <= start_time + timeout:
8810             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
8811             operational_status_ro = db_nsr["_admin"]["deployed"]["RO"][
8812                 "operational-status"
8813             ]
8814             self.logger.debug("Wait Heal RO > {}".format(operational_status_ro))
8815             if operational_status_ro != "healing":
8816                 break
8817             await asyncio.sleep(15)
8818         else:  # timeout_ns_deploy
8819             raise NgRoException("Timeout waiting ns to deploy")
8820
8821     async def vertical_scale(self, nsr_id, nslcmop_id):
8822         """
8823         Vertical Scale the VDUs in a NS
8824
8825         :param: nsr_id: NS Instance ID
8826         :param: nslcmop_id: nslcmop ID of migrate
8827
8828         """
8829         # Try to lock HA task here
8830         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
8831         if not task_is_locked_by_me:
8832             return
8833         logging_text = "Task ns={} vertical scale ".format(nsr_id)
8834         self.logger.debug(logging_text + "Enter")
8835         # get all needed from database
8836         db_nslcmop = None
8837         db_nslcmop_update = {}
8838         nslcmop_operation_state = None
8839         old_db_update = {}
8840         q_filter = {}
8841         old_vdu_index = None
8842         old_flavor_id = None
8843         db_nsr_update = {}
8844         target = {}
8845         exc = None
8846         # in case of error, indicates what part of scale was failed to put nsr at error status
8847         start_deploy = time()
8848
8849         try:
8850             # wait for any previous tasks in process
8851             step = "Waiting for previous operations to terminate"
8852             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
8853
8854             self._write_ns_status(
8855                 nsr_id=nsr_id,
8856                 ns_state=None,
8857                 current_operation="VerticalScale",
8858                 current_operation_id=nslcmop_id,
8859             )
8860             step = "Getting nslcmop from database"
8861             self.logger.debug(
8862                 step + " after having waited for previous tasks to be completed"
8863             )
8864             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
8865             operationParams = db_nslcmop.get("operationParams")
8866             # Update the VNFRS and NSRS with the requested flavour detail, So that ro tasks can function properly
8867             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
8868             db_flavor = db_nsr.get("flavor")
8869             db_flavor_index = str(len(db_flavor))
8870             change_vnf_flavor_data = operationParams["changeVnfFlavorData"]
8871             flavor_dict = change_vnf_flavor_data["additionalParams"]
8872             count_index = flavor_dict["vduCountIndex"]
8873             vdu_id_ref = flavor_dict["vduid"]
8874             flavor_dict_update = {
8875                 "id": db_flavor_index,
8876                 "memory-mb": flavor_dict["virtualMemory"],
8877                 "name": f"{vdu_id_ref}-{count_index}-flv",
8878                 "storage-gb": flavor_dict["sizeOfStorage"],
8879                 "vcpu-count": flavor_dict["numVirtualCpu"],
8880             }
8881             db_flavor.append(flavor_dict_update)
8882             db_update = {}
8883             db_update["flavor"] = db_flavor
8884             ns_q_filter = {
8885                 "_id": nsr_id,
8886             }
8887             self.db.set_one(
8888                 "nsrs",
8889                 q_filter=ns_q_filter,
8890                 update_dict=db_update,
8891                 fail_on_empty=True,
8892             )
8893             db_vnfr = self.db.get_one(
8894                 "vnfrs", {"_id": change_vnf_flavor_data["vnfInstanceId"]}
8895             )
8896             for vdu_index, vdur in enumerate(db_vnfr.get("vdur", ())):
8897                 if (
8898                     vdur.get("count-index") == count_index
8899                     and vdur.get("vdu-id-ref") == vdu_id_ref
8900                 ):
8901                     old_flavor_id = vdur.get("ns-flavor-id", 0)
8902                     old_vdu_index = vdu_index
8903                     filter_text = {
8904                         "_id": change_vnf_flavor_data["vnfInstanceId"],
8905                         "vdur.count-index": count_index,
8906                         "vdur.vdu-id-ref": vdu_id_ref,
8907                     }
8908                     q_filter.update(filter_text)
8909                     db_update = {}
8910                     db_update[
8911                         "vdur.{}.ns-flavor-id".format(vdu_index)
8912                     ] = db_flavor_index
8913                     self.db.set_one(
8914                         "vnfrs",
8915                         q_filter=q_filter,
8916                         update_dict=db_update,
8917                         fail_on_empty=True,
8918                     )
8919             target = {}
8920             target.update(operationParams)
8921             desc = await self.RO.vertical_scale(nsr_id, target)
8922             self.logger.debug("RO return > {}".format(desc))
8923             action_id = desc["action_id"]
8924             await self._wait_ng_ro(
8925                 nsr_id,
8926                 action_id,
8927                 nslcmop_id,
8928                 start_deploy,
8929                 self.timeout.verticalscale,
8930                 operation="verticalscale",
8931             )
8932         except (
8933             NgRoException,
8934             ROclient.ROClientException,
8935             DbException,
8936             LcmException,
8937         ) as e:
8938             self.logger.error("Exit Exception {}".format(e))
8939             exc = e
8940         except asyncio.CancelledError:
8941             self.logger.error("Cancelled Exception while '{}'".format(step))
8942             exc = "Operation was cancelled"
8943         except Exception as e:
8944             exc = traceback.format_exc()
8945             self.logger.critical(
8946                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
8947             )
8948         finally:
8949             self._write_ns_status(
8950                 nsr_id=nsr_id,
8951                 ns_state=None,
8952                 current_operation="IDLE",
8953                 current_operation_id=None,
8954             )
8955             if exc:
8956                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
8957                 nslcmop_operation_state = "FAILED"
8958                 old_db_update[
8959                     "vdur.{}.ns-flavor-id".format(old_vdu_index)
8960                 ] = old_flavor_id
8961             else:
8962                 nslcmop_operation_state = "COMPLETED"
8963                 db_nslcmop_update["detailed-status"] = "Done"
8964                 db_nsr_update["detailed-status"] = "Done"
8965
8966             self._write_op_status(
8967                 op_id=nslcmop_id,
8968                 stage="",
8969                 error_message="",
8970                 operation_state=nslcmop_operation_state,
8971                 other_update=db_nslcmop_update,
8972             )
8973             if old_vdu_index and old_db_update != {}:
8974                 self.logger.critical(
8975                     "Reverting Old Flavor -- : {}".format(old_db_update)
8976                 )
8977                 self.db.set_one(
8978                     "vnfrs",
8979                     q_filter=q_filter,
8980                     update_dict=old_db_update,
8981                     fail_on_empty=True,
8982                 )
8983             if nslcmop_operation_state:
8984                 try:
8985                     msg = {
8986                         "nsr_id": nsr_id,
8987                         "nslcmop_id": nslcmop_id,
8988                         "operationState": nslcmop_operation_state,
8989                     }
8990                     await self.msg.aiowrite("ns", "verticalscaled", msg)
8991                 except Exception as e:
8992                     self.logger.error(
8993                         logging_text + "kafka_write notification Exception {}".format(e)
8994                     )
8995             self.logger.debug(logging_text + "Exit")
8996             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_verticalscale")