osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import shutil
  21 from typing import Any, Dict, List
  22 import yaml
  23 import logging
  24 import logging.handlers
  25 import traceback
  26 import ipaddress
  27 import json
  28 from jinja2 import (
  29     Environment,
  30     TemplateError,
  31     TemplateNotFound,
  32     StrictUndefined,
  33     UndefinedError,
  34     select_autoescape,
  35 )
  36
  37 from osm_lcm import ROclient
  38 from osm_lcm.data_utils.lcm_config import LcmCfg
  39 from osm_lcm.data_utils.nsr import (
  40     get_deployed_kdu,
  41     get_deployed_vca,
  42     get_deployed_vca_list,
  43     get_nsd,
  44 )
  45 from osm_lcm.data_utils.vca import (
  46     DeployedComponent,
  47     DeployedK8sResource,
  48     DeployedVCA,
  49     EELevel,
  50     Relation,
  51     EERelation,
  52     safe_get_ee_relation,
  53 )
  54 from osm_lcm.ng_ro import NgRoClient, NgRoException
  55 from osm_lcm.lcm_utils import (
  56     LcmException,
  57     LcmBase,
  58     deep_get,
  59     get_iterable,
  60     populate_dict,
  61     check_juju_bundle_existence,
  62     get_charm_artifact_path,
  63     get_ee_id_parts,
  64     vld_to_ro_ip_profile,
  65 )
  66 from osm_lcm.data_utils.nsd import (
  67     get_ns_configuration_relation_list,
  68     get_vnf_profile,
  69     get_vnf_profiles,
  70 )
  71 from osm_lcm.data_utils.vnfd import (
  72     get_kdu,
  73     get_kdu_services,
  74     get_relation_list,
  75     get_vdu_list,
  76     get_vdu_profile,
  77     get_ee_sorted_initial_config_primitive_list,
  78     get_ee_sorted_terminate_config_primitive_list,
  79     get_kdu_list,
  80     get_virtual_link_profiles,
  81     get_vdu,
  82     get_configuration,
  83     get_vdu_index,
  84     get_scaling_aspect,
  85     get_number_of_instances,
  86     get_juju_ee_ref,
  87     get_kdu_resource_profile,
  88     find_software_version,
  89     check_helm_ee_in_ns,
  90 )
  91 from osm_lcm.data_utils.list_utils import find_in_list
  92 from osm_lcm.data_utils.vnfr import (
  93     get_osm_params,
  94     get_vdur_index,
  95     get_kdur,
  96     get_volumes_from_instantiation_params,
  97 )
  98 from osm_lcm.data_utils.dict_utils import parse_yaml_strings
  99 from osm_lcm.data_utils.database.vim_account import VimAccountDB
 100 from n2vc.definitions import RelationEndpoint
 101 from n2vc.k8s_helm3_conn import K8sHelm3Connector
 102 from n2vc.k8s_juju_conn import K8sJujuConnector
 103
 104 from osm_common.dbbase import DbException
 105 from osm_common.fsbase import FsException
 106
 107 from osm_lcm.data_utils.database.database import Database
 108 from osm_lcm.data_utils.filesystem.filesystem import Filesystem
 109 from osm_lcm.data_utils.wim import (
 110     get_sdn_ports,
 111     get_target_wim_attrs,
 112     select_feasible_wim_account,
 113 )
 114
 115 from n2vc.n2vc_juju_conn import N2VCJujuConnector
 116 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
 117
 118 from osm_lcm.lcm_helm_conn import LCMHelmConn
 119 from osm_lcm.osm_config import OsmConfigBuilder
 120 from osm_lcm.prometheus import parse_job
 121
 122 from copy import copy, deepcopy
 123 from time import time
 124 from uuid import uuid4
 125
 126 from random import SystemRandom
 127
 128 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 129
 130
 131 class NsLcm(LcmBase):
 132     SUBOPERATION_STATUS_NOT_FOUND = -1
 133     SUBOPERATION_STATUS_NEW = -2
 134     SUBOPERATION_STATUS_SKIP = -3
 135     EE_TLS_NAME = "ee-tls"
 136     task_name_deploy_vca = "Deploying VCA"
 137     rel_operation_types = {
 138         "GE": ">=",
 139         "LE": "<=",
 140         "GT": ">",
 141         "LT": "<",
 142         "EQ": "==",
 143         "NE": "!=",
 144     }
 145
 146     def __init__(self, msg, lcm_tasks, config: LcmCfg):
 147         """
 148         Init, Connect to database, filesystem storage, and messaging
 149         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 150         :return: None
 151         """
 152         super().__init__(msg=msg, logger=logging.getLogger("lcm.ns"))
 153
 154         self.db = Database().instance.db
 155         self.fs = Filesystem().instance.fs
 156         self.lcm_tasks = lcm_tasks
 157         self.timeout = config.timeout
 158         self.ro_config = config.RO
 159         self.vca_config = config.VCA
 160
 161         # create N2VC connector
 162         self.n2vc = N2VCJujuConnector(
 163             log=self.logger,
 164             on_update_db=self._on_update_n2vc_db,
 165             fs=self.fs,
 166             db=self.db,
 167         )
 168
 169         self.conn_helm_ee = LCMHelmConn(
 170             log=self.logger,
 171             vca_config=self.vca_config,
 172             on_update_db=self._on_update_n2vc_db,
 173         )
 174
 175         self.k8sclusterhelm3 = K8sHelm3Connector(
 176             kubectl_command=self.vca_config.kubectlpath,
 177             helm_command=self.vca_config.helm3path,
 178             fs=self.fs,
 179             log=self.logger,
 180             db=self.db,
 181             on_update_db=None,
 182         )
 183
 184         self.k8sclusterjuju = K8sJujuConnector(
 185             kubectl_command=self.vca_config.kubectlpath,
 186             juju_command=self.vca_config.jujupath,
 187             log=self.logger,
 188             on_update_db=self._on_update_k8s_db,
 189             fs=self.fs,
 190             db=self.db,
 191         )
 192
 193         self.k8scluster_map = {
 194             "helm-chart-v3": self.k8sclusterhelm3,
 195             "chart": self.k8sclusterhelm3,
 196             "juju-bundle": self.k8sclusterjuju,
 197             "juju": self.k8sclusterjuju,
 198         }
 199
 200         self.vca_map = {
 201             "lxc_proxy_charm": self.n2vc,
 202             "native_charm": self.n2vc,
 203             "k8s_proxy_charm": self.n2vc,
 204             "helm": self.conn_helm_ee,
 205             "helm-v3": self.conn_helm_ee,
 206         }
 207
 208         # create RO client
 209         self.RO = NgRoClient(**self.ro_config.to_dict())
 210
 211         self.op_status_map = {
 212             "instantiation": self.RO.status,
 213             "termination": self.RO.status,
 214             "migrate": self.RO.status,
 215             "healing": self.RO.recreate_status,
 216             "verticalscale": self.RO.status,
 217             "start_stop_rebuild": self.RO.status,
 218         }
 219
 220     @staticmethod
 221     def increment_ip_mac(ip_mac, vm_index=1):
 222         if not isinstance(ip_mac, str):
 223             return ip_mac
 224         try:
 225             next_ipv6 = None
 226             next_ipv4 = None
 227             dual_ip = ip_mac.split(";")
 228             if len(dual_ip) == 2:
 229                 for ip in dual_ip:
 230                     if ipaddress.ip_address(ip).version == 6:
 231                         ipv6 = ipaddress.IPv6Address(ip)
 232                         next_ipv6 = str(ipaddress.IPv6Address(int(ipv6) + 1))
 233                     elif ipaddress.ip_address(ip).version == 4:
 234                         ipv4 = ipaddress.IPv4Address(ip)
 235                         next_ipv4 = str(ipaddress.IPv4Address(int(ipv4) + 1))
 236                 return [next_ipv4, next_ipv6]
 237             # try with ipv4 look for last dot
 238             i = ip_mac.rfind(".")
 239             if i > 0:
 240                 i += 1
 241                 return "{}{}".format(ip_mac[:i], int(ip_mac[i:]) + vm_index)
 242             # try with ipv6 or mac look for last colon. Operate in hex
 243             i = ip_mac.rfind(":")
 244             if i > 0:
 245                 i += 1
 246                 # format in hex, len can be 2 for mac or 4 for ipv6
 247                 return ("{}{:0" + str(len(ip_mac) - i) + "x}").format(
 248                     ip_mac[:i], int(ip_mac[i:], 16) + vm_index
 249                 )
 250         except Exception:
 251             pass
 252         return None
 253
 254     async def _on_update_n2vc_db(self, table, filter, path, updated_data, vca_id=None):
 255         # remove last dot from path (if exists)
 256         if path.endswith("."):
 257             path = path[:-1]
 258
 259         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 260         #                   .format(table, filter, path, updated_data))
 261         try:
 262             nsr_id = filter.get("_id")
 263
 264             # read ns record from database
 265             nsr = self.db.get_one(table="nsrs", q_filter=filter)
 266             current_ns_status = nsr.get("nsState")
 267
 268             # get vca status for NS
 269             status_dict = await self.n2vc.get_status(
 270                 namespace="." + nsr_id, yaml_format=False, vca_id=vca_id
 271             )
 272
 273             # vcaStatus
 274             db_dict = dict()
 275             db_dict["vcaStatus"] = status_dict
 276
 277             # update configurationStatus for this VCA
 278             try:
 279                 vca_index = int(path[path.rfind(".") + 1 :])
 280
 281                 vca_list = deep_get(
 282                     target_dict=nsr, key_list=("_admin", "deployed", "VCA")
 283                 )
 284                 vca_status = vca_list[vca_index].get("status")
 285
 286                 configuration_status_list = nsr.get("configurationStatus")
 287                 config_status = configuration_status_list[vca_index].get("status")
 288
 289                 if config_status == "BROKEN" and vca_status != "failed":
 290                     db_dict["configurationStatus"][vca_index] = "READY"
 291                 elif config_status != "BROKEN" and vca_status == "failed":
 292                     db_dict["configurationStatus"][vca_index] = "BROKEN"
 293             except Exception as e:
 294                 # not update configurationStatus
 295                 self.logger.debug("Error updating vca_index (ignore): {}".format(e))
 296
 297             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 298             # if nsState = 'DEGRADED' check if all is OK
 299             is_degraded = False
 300             if current_ns_status in ("READY", "DEGRADED"):
 301                 error_description = ""
 302                 # check machines
 303                 if status_dict.get("machines"):
 304                     for machine_id in status_dict.get("machines"):
 305                         machine = status_dict.get("machines").get(machine_id)
 306                         # check machine agent-status
 307                         if machine.get("agent-status"):
 308                             s = machine.get("agent-status").get("status")
 309                             if s != "started":
 310                                 is_degraded = True
 311                                 error_description += (
 312                                     "machine {} agent-status={} ; ".format(
 313                                         machine_id, s
 314                                     )
 315                                 )
 316                         # check machine instance status
 317                         if machine.get("instance-status"):
 318                             s = machine.get("instance-status").get("status")
 319                             if s != "running":
 320                                 is_degraded = True
 321                                 error_description += (
 322                                     "machine {} instance-status={} ; ".format(
 323                                         machine_id, s
 324                                     )
 325                                 )
 326                 # check applications
 327                 if status_dict.get("applications"):
 328                     for app_id in status_dict.get("applications"):
 329                         app = status_dict.get("applications").get(app_id)
 330                         # check application status
 331                         if app.get("status"):
 332                             s = app.get("status").get("status")
 333                             if s != "active":
 334                                 is_degraded = True
 335                                 error_description += (
 336                                     "application {} status={} ; ".format(app_id, s)
 337                                 )
 338
 339                 if error_description:
 340                     db_dict["errorDescription"] = error_description
 341                 if current_ns_status == "READY" and is_degraded:
 342                     db_dict["nsState"] = "DEGRADED"
 343                 if current_ns_status == "DEGRADED" and not is_degraded:
 344                     db_dict["nsState"] = "READY"
 345
 346             # write to database
 347             self.update_db_2("nsrs", nsr_id, db_dict)
 348
 349         except (asyncio.CancelledError, asyncio.TimeoutError):
 350             raise
 351         except Exception as e:
 352             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 353
 354     async def _on_update_k8s_db(
 355         self, cluster_uuid, kdu_instance, filter=None, vca_id=None, cluster_type="juju"
 356     ):
 357         """
 358         Updating vca status in NSR record
 359         :param cluster_uuid: UUID of a k8s cluster
 360         :param kdu_instance: The unique name of the KDU instance
 361         :param filter: To get nsr_id
 362         :cluster_type: The cluster type (juju, k8s)
 363         :return: none
 364         """
 365
 366         # self.logger.debug("_on_update_k8s_db(cluster_uuid={}, kdu_instance={}, filter={}"
 367         #                   .format(cluster_uuid, kdu_instance, filter))
 368
 369         nsr_id = filter.get("_id")
 370         try:
 371             vca_status = await self.k8scluster_map[cluster_type].status_kdu(
 372                 cluster_uuid=cluster_uuid,
 373                 kdu_instance=kdu_instance,
 374                 yaml_format=False,
 375                 complete_status=True,
 376                 vca_id=vca_id,
 377             )
 378
 379             # vcaStatus
 380             db_dict = dict()
 381             db_dict["vcaStatus"] = {nsr_id: vca_status}
 382
 383             self.logger.debug(
 384                 f"Obtained VCA status for cluster type '{cluster_type}': {vca_status}"
 385             )
 386
 387             # write to database
 388             self.update_db_2("nsrs", nsr_id, db_dict)
 389         except (asyncio.CancelledError, asyncio.TimeoutError):
 390             raise
 391         except Exception as e:
 392             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 393
 394     @staticmethod
 395     def _parse_cloud_init(cloud_init_text, additional_params, vnfd_id, vdu_id):
 396         try:
 397             env = Environment(
 398                 undefined=StrictUndefined,
 399                 autoescape=select_autoescape(default_for_string=True, default=True),
 400             )
 401             template = env.from_string(cloud_init_text)
 402             return template.render(additional_params or {})
 403         except UndefinedError as e:
 404             raise LcmException(
 405                 "Variable {} at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 406                 "file, must be provided in the instantiation parameters inside the "
 407                 "'additionalParamsForVnf/Vdu' block".format(e, vnfd_id, vdu_id)
 408             )
 409         except (TemplateError, TemplateNotFound) as e:
 410             raise LcmException(
 411                 "Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".format(
 412                     vnfd_id, vdu_id, e
 413                 )
 414             )
 415
 416     def _get_vdu_cloud_init_content(self, vdu, vnfd):
 417         cloud_init_content = cloud_init_file = None
 418         try:
 419             if vdu.get("cloud-init-file"):
 420                 base_folder = vnfd["_admin"]["storage"]
 421                 if base_folder["pkg-dir"]:
 422                     cloud_init_file = "{}/{}/cloud_init/{}".format(
 423                         base_folder["folder"],
 424                         base_folder["pkg-dir"],
 425                         vdu["cloud-init-file"],
 426                     )
 427                 else:
 428                     cloud_init_file = "{}/Scripts/cloud_init/{}".format(
 429                         base_folder["folder"],
 430                         vdu["cloud-init-file"],
 431                     )
 432                 with self.fs.file_open(cloud_init_file, "r") as ci_file:
 433                     cloud_init_content = ci_file.read()
 434             elif vdu.get("cloud-init"):
 435                 cloud_init_content = vdu["cloud-init"]
 436
 437             return cloud_init_content
 438         except FsException as e:
 439             raise LcmException(
 440                 "Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".format(
 441                     vnfd["id"], vdu["id"], cloud_init_file, e
 442                 )
 443             )
 444
 445     def _get_vdu_additional_params(self, db_vnfr, vdu_id):
 446         vdur = next(
 447             (vdur for vdur in db_vnfr.get("vdur") if vdu_id == vdur["vdu-id-ref"]), {}
 448         )
 449         additional_params = vdur.get("additionalParams")
 450         return parse_yaml_strings(additional_params)
 451
 452     @staticmethod
 453     def ip_profile_2_RO(ip_profile):
 454         RO_ip_profile = deepcopy(ip_profile)
 455         if "dns-server" in RO_ip_profile:
 456             if isinstance(RO_ip_profile["dns-server"], list):
 457                 RO_ip_profile["dns-address"] = []
 458                 for ds in RO_ip_profile.pop("dns-server"):
 459                     RO_ip_profile["dns-address"].append(ds["address"])
 460             else:
 461                 RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 462         if RO_ip_profile.get("ip-version") == "ipv4":
 463             RO_ip_profile["ip-version"] = "IPv4"
 464         if RO_ip_profile.get("ip-version") == "ipv6":
 465             RO_ip_profile["ip-version"] = "IPv6"
 466         if "dhcp-params" in RO_ip_profile:
 467             RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 468         return RO_ip_profile
 469
 470     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None, mark_delete=False):
 471         db_vdu_push_list = []
 472         template_vdur = []
 473         db_update = {"_admin.modified": time()}
 474         if vdu_create:
 475             for vdu_id, vdu_count in vdu_create.items():
 476                 vdur = next(
 477                     (
 478                         vdur
 479                         for vdur in reversed(db_vnfr["vdur"])
 480                         if vdur["vdu-id-ref"] == vdu_id
 481                     ),
 482                     None,
 483                 )
 484                 if not vdur:
 485                     # Read the template saved in the db:
 486                     self.logger.debug(
 487                         "No vdur in the database. Using the vdur-template to scale"
 488                     )
 489                     vdur_template = db_vnfr.get("vdur-template")
 490                     if not vdur_template:
 491                         raise LcmException(
 492                             "Error scaling OUT VNFR for {}. No vnfr or template exists".format(
 493                                 vdu_id
 494                             )
 495                         )
 496                     vdur = vdur_template[0]
 497                     # Delete a template from the database after using it
 498                     self.db.set_one(
 499                         "vnfrs",
 500                         {"_id": db_vnfr["_id"]},
 501                         None,
 502                         pull={"vdur-template": {"_id": vdur["_id"]}},
 503                     )
 504                 for count in range(vdu_count):
 505                     vdur_copy = deepcopy(vdur)
 506                     vdur_copy["status"] = "BUILD"
 507                     vdur_copy["status-detailed"] = None
 508                     vdur_copy["ip-address"] = None
 509                     vdur_copy["_id"] = str(uuid4())
 510                     vdur_copy["count-index"] += count + 1
 511                     vdur_copy["id"] = "{}-{}".format(
 512                         vdur_copy["vdu-id-ref"], vdur_copy["count-index"]
 513                     )
 514                     vdur_copy.pop("vim_info", None)
 515                     for iface in vdur_copy["interfaces"]:
 516                         if iface.get("fixed-ip"):
 517                             iface["ip-address"] = self.increment_ip_mac(
 518                                 iface["ip-address"], count + 1
 519                             )
 520                         else:
 521                             iface.pop("ip-address", None)
 522                         if iface.get("fixed-mac"):
 523                             iface["mac-address"] = self.increment_ip_mac(
 524                                 iface["mac-address"], count + 1
 525                             )
 526                         else:
 527                             iface.pop("mac-address", None)
 528                         if db_vnfr["vdur"]:
 529                             iface.pop(
 530                                 "mgmt_vnf", None
 531                             )  # only first vdu can be managment of vnf
 532                     db_vdu_push_list.append(vdur_copy)
 533                     # self.logger.debug("scale out, adding vdu={}".format(vdur_copy))
 534         if vdu_delete:
 535             if len(db_vnfr["vdur"]) == 1:
 536                 # The scale will move to 0 instances
 537                 self.logger.debug(
 538                     "Scaling to 0 !, creating the template with the last vdur"
 539                 )
 540                 template_vdur = [db_vnfr["vdur"][0]]
 541             for vdu_id, vdu_count in vdu_delete.items():
 542                 if mark_delete:
 543                     indexes_to_delete = [
 544                         iv[0]
 545                         for iv in enumerate(db_vnfr["vdur"])
 546                         if iv[1]["vdu-id-ref"] == vdu_id
 547                     ]
 548                     db_update.update(
 549                         {
 550                             "vdur.{}.status".format(i): "DELETING"
 551                             for i in indexes_to_delete[-vdu_count:]
 552                         }
 553                     )
 554                 else:
 555                     # it must be deleted one by one because common.db does not allow otherwise
 556                     vdus_to_delete = [
 557                         v
 558                         for v in reversed(db_vnfr["vdur"])
 559                         if v["vdu-id-ref"] == vdu_id
 560                     ]
 561                     for vdu in vdus_to_delete[:vdu_count]:
 562                         self.db.set_one(
 563                             "vnfrs",
 564                             {"_id": db_vnfr["_id"]},
 565                             None,
 566                             pull={"vdur": {"_id": vdu["_id"]}},
 567                         )
 568         db_push = {}
 569         if db_vdu_push_list:
 570             db_push["vdur"] = db_vdu_push_list
 571         if template_vdur:
 572             db_push["vdur-template"] = template_vdur
 573         if not db_push:
 574             db_push = None
 575         db_vnfr["vdur-template"] = template_vdur
 576         self.db.set_one("vnfrs", {"_id": db_vnfr["_id"]}, db_update, push_list=db_push)
 577         # modify passed dictionary db_vnfr
 578         db_vnfr_ = self.db.get_one("vnfrs", {"_id": db_vnfr["_id"]})
 579         db_vnfr["vdur"] = db_vnfr_["vdur"]
 580
 581     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 582         """
 583         Updates database nsr with the RO info for the created vld
 584         :param ns_update_nsr: dictionary to be filled with the updated info
 585         :param db_nsr: content of db_nsr. This is also modified
 586         :param nsr_desc_RO: nsr descriptor from RO
 587         :return: Nothing, LcmException is raised on errors
 588         """
 589
 590         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 591             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 592                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 593                     continue
 594                 vld["vim-id"] = net_RO.get("vim_net_id")
 595                 vld["name"] = net_RO.get("vim_name")
 596                 vld["status"] = net_RO.get("status")
 597                 vld["status-detailed"] = net_RO.get("error_msg")
 598                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 599                 break
 600             else:
 601                 raise LcmException(
 602                     "ns_update_nsr: Not found vld={} at RO info".format(vld["id"])
 603                 )
 604
 605     def set_vnfr_at_error(self, db_vnfrs, error_text):
 606         try:
 607             for db_vnfr in db_vnfrs.values():
 608                 vnfr_update = {"status": "ERROR"}
 609                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 610                     if "status" not in vdur:
 611                         vdur["status"] = "ERROR"
 612                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 613                         if error_text:
 614                             vdur["status-detailed"] = str(error_text)
 615                             vnfr_update[
 616                                 "vdur.{}.status-detailed".format(vdu_index)
 617                             ] = "ERROR"
 618                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 619         except DbException as e:
 620             self.logger.error("Cannot update vnf. {}".format(e))
 621
 622     def _get_ns_config_info(self, nsr_id):
 623         """
 624         Generates a mapping between vnf,vdu elements and the N2VC id
 625         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 626         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 627             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 628             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 629         """
 630         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 631         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 632         mapping = {}
 633         ns_config_info = {"osm-config-mapping": mapping}
 634         for vca in vca_deployed_list:
 635             if not vca["member-vnf-index"]:
 636                 continue
 637             if not vca["vdu_id"]:
 638                 mapping[vca["member-vnf-index"]] = vca["application"]
 639             else:
 640                 mapping[
 641                     "{}.{}.{}".format(
 642                         vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"]
 643                     )
 644                 ] = vca["application"]
 645         return ns_config_info
 646
 647     async def _instantiate_ng_ro(
 648         self,
 649         logging_text,
 650         nsr_id,
 651         nsd,
 652         db_nsr,
 653         db_nslcmop,
 654         db_vnfrs,
 655         db_vnfds,
 656         n2vc_key_list,
 657         stage,
 658         start_deploy,
 659         timeout_ns_deploy,
 660     ):
 661         db_vims = {}
 662
 663         def get_vim_account(vim_account_id):
 664             nonlocal db_vims
 665             if vim_account_id in db_vims:
 666                 return db_vims[vim_account_id]
 667             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
 668             db_vims[vim_account_id] = db_vim
 669             return db_vim
 670
 671         # modify target_vld info with instantiation parameters
 672         def parse_vld_instantiation_params(
 673             target_vim, target_vld, vld_params, target_sdn
 674         ):
 675             if vld_params.get("ip-profile"):
 676                 target_vld["vim_info"][target_vim]["ip_profile"] = vld_to_ro_ip_profile(
 677                     vld_params["ip-profile"]
 678                 )
 679             if vld_params.get("provider-network"):
 680                 target_vld["vim_info"][target_vim]["provider_network"] = vld_params[
 681                     "provider-network"
 682                 ]
 683                 if "sdn-ports" in vld_params["provider-network"] and target_sdn:
 684                     target_vld["vim_info"][target_sdn]["sdn-ports"] = vld_params[
 685                         "provider-network"
 686                     ]["sdn-ports"]
 687
 688             # check if WIM is needed; if needed, choose a feasible WIM able to connect VIMs
 689             # if wim_account_id is specified in vld_params, validate if it is feasible.
 690             wim_account_id, db_wim = select_feasible_wim_account(
 691                 db_nsr, db_vnfrs, target_vld, vld_params, self.logger
 692             )
 693
 694             if wim_account_id:
 695                 # WIM is needed and a feasible one was found, populate WIM target and SDN ports
 696                 self.logger.info("WIM selected: {:s}".format(str(wim_account_id)))
 697                 # update vld_params with correct WIM account Id
 698                 vld_params["wimAccountId"] = wim_account_id
 699
 700                 target_wim = "wim:{}".format(wim_account_id)
 701                 target_wim_attrs = get_target_wim_attrs(nsr_id, target_vld, vld_params)
 702                 sdn_ports = get_sdn_ports(vld_params, db_wim)
 703                 if len(sdn_ports) > 0:
 704                     target_vld["vim_info"][target_wim] = target_wim_attrs
 705                     target_vld["vim_info"][target_wim]["sdn-ports"] = sdn_ports
 706
 707                 self.logger.debug(
 708                     "Target VLD with WIM data: {:s}".format(str(target_vld))
 709                 )
 710
 711             for param in ("vim-network-name", "vim-network-id"):
 712                 if vld_params.get(param):
 713                     if isinstance(vld_params[param], dict):
 714                         for vim, vim_net in vld_params[param].items():
 715                             other_target_vim = "vim:" + vim
 716                             populate_dict(
 717                                 target_vld["vim_info"],
 718                                 (other_target_vim, param.replace("-", "_")),
 719                                 vim_net,
 720                             )
 721                     else:  # isinstance str
 722                         target_vld["vim_info"][target_vim][
 723                             param.replace("-", "_")
 724                         ] = vld_params[param]
 725             if vld_params.get("common_id"):
 726                 target_vld["common_id"] = vld_params.get("common_id")
 727
 728         # modify target["ns"]["vld"] with instantiation parameters to override vnf vim-account
 729         def update_ns_vld_target(target, ns_params):
 730             for vnf_params in ns_params.get("vnf", ()):
 731                 if vnf_params.get("vimAccountId"):
 732                     target_vnf = next(
 733                         (
 734                             vnfr
 735                             for vnfr in db_vnfrs.values()
 736                             if vnf_params["member-vnf-index"]
 737                             == vnfr["member-vnf-index-ref"]
 738                         ),
 739                         None,
 740                     )
 741                     vdur = next((vdur for vdur in target_vnf.get("vdur", ())), None)
 742                     if not vdur:
 743                         continue
 744                     for a_index, a_vld in enumerate(target["ns"]["vld"]):
 745                         target_vld = find_in_list(
 746                             get_iterable(vdur, "interfaces"),
 747                             lambda iface: iface.get("ns-vld-id") == a_vld["name"],
 748                         )
 749
 750                         vld_params = find_in_list(
 751                             get_iterable(ns_params, "vld"),
 752                             lambda v_vld: v_vld["name"] in (a_vld["name"], a_vld["id"]),
 753                         )
 754                         if target_vld:
 755                             if vnf_params.get("vimAccountId") not in a_vld.get(
 756                                 "vim_info", {}
 757                             ):
 758                                 target_vim_network_list = [
 759                                     v for _, v in a_vld.get("vim_info").items()
 760                                 ]
 761                                 target_vim_network_name = next(
 762                                     (
 763                                         item.get("vim_network_name", "")
 764                                         for item in target_vim_network_list
 765                                     ),
 766                                     "",
 767                                 )
 768
 769                                 target["ns"]["vld"][a_index].get("vim_info").update(
 770                                     {
 771                                         "vim:{}".format(vnf_params["vimAccountId"]): {
 772                                             "vim_network_name": target_vim_network_name,
 773                                         }
 774                                     }
 775                                 )
 776
 777                                 if vld_params:
 778                                     for param in ("vim-network-name", "vim-network-id"):
 779                                         if vld_params.get(param) and isinstance(
 780                                             vld_params[param], dict
 781                                         ):
 782                                             for vim, vim_net in vld_params[
 783                                                 param
 784                                             ].items():
 785                                                 other_target_vim = "vim:" + vim
 786                                                 populate_dict(
 787                                                     target["ns"]["vld"][a_index].get(
 788                                                         "vim_info"
 789                                                     ),
 790                                                     (
 791                                                         other_target_vim,
 792                                                         param.replace("-", "_"),
 793                                                     ),
 794                                                     vim_net,
 795                                                 )
 796
 797         nslcmop_id = db_nslcmop["_id"]
 798         target = {
 799             "name": db_nsr["name"],
 800             "ns": {"vld": []},
 801             "vnf": [],
 802             "image": deepcopy(db_nsr["image"]),
 803             "flavor": deepcopy(db_nsr["flavor"]),
 804             "action_id": nslcmop_id,
 805             "cloud_init_content": {},
 806         }
 807         for image in target["image"]:
 808             image["vim_info"] = {}
 809         for flavor in target["flavor"]:
 810             flavor["vim_info"] = {}
 811         if db_nsr.get("shared-volumes"):
 812             target["shared-volumes"] = deepcopy(db_nsr["shared-volumes"])
 813             for shared_volumes in target["shared-volumes"]:
 814                 shared_volumes["vim_info"] = {}
 815         if db_nsr.get("affinity-or-anti-affinity-group"):
 816             target["affinity-or-anti-affinity-group"] = deepcopy(
 817                 db_nsr["affinity-or-anti-affinity-group"]
 818             )
 819             for affinity_or_anti_affinity_group in target[
 820                 "affinity-or-anti-affinity-group"
 821             ]:
 822                 affinity_or_anti_affinity_group["vim_info"] = {}
 823
 824         if db_nslcmop.get("lcmOperationType") != "instantiate":
 825             # get parameters of instantiation:
 826             db_nslcmop_instantiate = self.db.get_list(
 827                 "nslcmops",
 828                 {
 829                     "nsInstanceId": db_nslcmop["nsInstanceId"],
 830                     "lcmOperationType": "instantiate",
 831                 },
 832             )[-1]
 833             ns_params = db_nslcmop_instantiate.get("operationParams")
 834         else:
 835             ns_params = db_nslcmop.get("operationParams")
 836         ssh_keys_instantiation = ns_params.get("ssh_keys") or []
 837         ssh_keys_all = ssh_keys_instantiation + (n2vc_key_list or [])
 838
 839         cp2target = {}
 840         for vld_index, vld in enumerate(db_nsr.get("vld")):
 841             target_vim = "vim:{}".format(ns_params["vimAccountId"])
 842             target_vld = {
 843                 "id": vld["id"],
 844                 "name": vld["name"],
 845                 "mgmt-network": vld.get("mgmt-network", False),
 846                 "type": vld.get("type"),
 847                 "vim_info": {
 848                     target_vim: {
 849                         "vim_network_name": vld.get("vim-network-name"),
 850                         "vim_account_id": ns_params["vimAccountId"],
 851                     }
 852                 },
 853             }
 854             # check if this network needs SDN assist
 855             if vld.get("pci-interfaces"):
 856                 db_vim = get_vim_account(ns_params["vimAccountId"])
 857                 if vim_config := db_vim.get("config"):
 858                     if sdnc_id := vim_config.get("sdn-controller"):
 859                         sdn_vld = "nsrs:{}:vld.{}".format(nsr_id, vld["id"])
 860                         target_sdn = "sdn:{}".format(sdnc_id)
 861                         target_vld["vim_info"][target_sdn] = {
 862                             "sdn": True,
 863                             "target_vim": target_vim,
 864                             "vlds": [sdn_vld],
 865                             "type": vld.get("type"),
 866                         }
 867
 868             nsd_vnf_profiles = get_vnf_profiles(nsd)
 869             for nsd_vnf_profile in nsd_vnf_profiles:
 870                 for cp in nsd_vnf_profile["virtual-link-connectivity"]:
 871                     if cp["virtual-link-profile-id"] == vld["id"]:
 872                         cp2target[
 873                             "member_vnf:{}.{}".format(
 874                                 cp["constituent-cpd-id"][0][
 875                                     "constituent-base-element-id"
 876                                 ],
 877                                 cp["constituent-cpd-id"][0]["constituent-cpd-id"],
 878                             )
 879                         ] = "nsrs:{}:vld.{}".format(nsr_id, vld_index)
 880
 881             # check at nsd descriptor, if there is an ip-profile
 882             vld_params = {}
 883             nsd_vlp = find_in_list(
 884                 get_virtual_link_profiles(nsd),
 885                 lambda a_link_profile: a_link_profile["virtual-link-desc-id"]
 886                 == vld["id"],
 887             )
 888             if (
 889                 nsd_vlp
 890                 and nsd_vlp.get("virtual-link-protocol-data")
 891                 and nsd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
 892             ):
 893                 vld_params["ip-profile"] = nsd_vlp["virtual-link-protocol-data"][
 894                     "l3-protocol-data"
 895                 ]
 896
 897             # update vld_params with instantiation params
 898             vld_instantiation_params = find_in_list(
 899                 get_iterable(ns_params, "vld"),
 900                 lambda a_vld: a_vld["name"] in (vld["name"], vld["id"]),
 901             )
 902             if vld_instantiation_params:
 903                 vld_params.update(vld_instantiation_params)
 904             parse_vld_instantiation_params(target_vim, target_vld, vld_params, None)
 905             target["ns"]["vld"].append(target_vld)
 906         # Update the target ns_vld if vnf vim_account is overriden by instantiation params
 907         update_ns_vld_target(target, ns_params)
 908
 909         for vnfr in db_vnfrs.values():
 910             vnfd = find_in_list(
 911                 db_vnfds, lambda db_vnf: db_vnf["id"] == vnfr["vnfd-ref"]
 912             )
 913             vnf_params = find_in_list(
 914                 get_iterable(ns_params, "vnf"),
 915                 lambda a_vnf: a_vnf["member-vnf-index"] == vnfr["member-vnf-index-ref"],
 916             )
 917             target_vnf = deepcopy(vnfr)
 918             target_vim = "vim:{}".format(vnfr["vim-account-id"])
 919             for vld in target_vnf.get("vld", ()):
 920                 # check if connected to a ns.vld, to fill target'
 921                 vnf_cp = find_in_list(
 922                     vnfd.get("int-virtual-link-desc", ()),
 923                     lambda cpd: cpd.get("id") == vld["id"],
 924                 )
 925                 if vnf_cp:
 926                     ns_cp = "member_vnf:{}.{}".format(
 927                         vnfr["member-vnf-index-ref"], vnf_cp["id"]
 928                     )
 929                     if cp2target.get(ns_cp):
 930                         vld["target"] = cp2target[ns_cp]
 931
 932                 vld["vim_info"] = {
 933                     target_vim: {"vim_network_name": vld.get("vim-network-name")}
 934                 }
 935                 # check if this network needs SDN assist
 936                 target_sdn = None
 937                 if vld.get("pci-interfaces"):
 938                     db_vim = get_vim_account(vnfr["vim-account-id"])
 939                     sdnc_id = db_vim["config"].get("sdn-controller")
 940                     if sdnc_id:
 941                         sdn_vld = "vnfrs:{}:vld.{}".format(target_vnf["_id"], vld["id"])
 942                         target_sdn = "sdn:{}".format(sdnc_id)
 943                         vld["vim_info"][target_sdn] = {
 944                             "sdn": True,
 945                             "target_vim": target_vim,
 946                             "vlds": [sdn_vld],
 947                             "type": vld.get("type"),
 948                         }
 949
 950                 # check at vnfd descriptor, if there is an ip-profile
 951                 vld_params = {}
 952                 vnfd_vlp = find_in_list(
 953                     get_virtual_link_profiles(vnfd),
 954                     lambda a_link_profile: a_link_profile["id"] == vld["id"],
 955                 )
 956                 if (
 957                     vnfd_vlp
 958                     and vnfd_vlp.get("virtual-link-protocol-data")
 959                     and vnfd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
 960                 ):
 961                     vld_params["ip-profile"] = vnfd_vlp["virtual-link-protocol-data"][
 962                         "l3-protocol-data"
 963                     ]
 964                 # update vld_params with instantiation params
 965                 if vnf_params:
 966                     vld_instantiation_params = find_in_list(
 967                         get_iterable(vnf_params, "internal-vld"),
 968                         lambda i_vld: i_vld["name"] == vld["id"],
 969                     )
 970                     if vld_instantiation_params:
 971                         vld_params.update(vld_instantiation_params)
 972                 parse_vld_instantiation_params(target_vim, vld, vld_params, target_sdn)
 973
 974             vdur_list = []
 975             for vdur in target_vnf.get("vdur", ()):
 976                 if vdur.get("status") == "DELETING" or vdur.get("pdu-type"):
 977                     continue  # This vdu must not be created
 978                 vdur["vim_info"] = {"vim_account_id": vnfr["vim-account-id"]}
 979
 980                 self.logger.debug("NS > ssh_keys > {}".format(ssh_keys_all))
 981
 982                 if ssh_keys_all:
 983                     vdu_configuration = get_configuration(vnfd, vdur["vdu-id-ref"])
 984                     vnf_configuration = get_configuration(vnfd, vnfd["id"])
 985                     if (
 986                         vdu_configuration
 987                         and vdu_configuration.get("config-access")
 988                         and vdu_configuration.get("config-access").get("ssh-access")
 989                     ):
 990                         vdur["ssh-keys"] = ssh_keys_all
 991                         vdur["ssh-access-required"] = vdu_configuration[
 992                             "config-access"
 993                         ]["ssh-access"]["required"]
 994                     elif (
 995                         vnf_configuration
 996                         and vnf_configuration.get("config-access")
 997                         and vnf_configuration.get("config-access").get("ssh-access")
 998                         and any(iface.get("mgmt-vnf") for iface in vdur["interfaces"])
 999                     ):
1000                         vdur["ssh-keys"] = ssh_keys_all
1001                         vdur["ssh-access-required"] = vnf_configuration[
1002                             "config-access"
1003                         ]["ssh-access"]["required"]
1004                     elif ssh_keys_instantiation and find_in_list(
1005                         vdur["interfaces"], lambda iface: iface.get("mgmt-vnf")
1006                     ):
1007                         vdur["ssh-keys"] = ssh_keys_instantiation
1008
1009                 self.logger.debug("NS > vdur > {}".format(vdur))
1010
1011                 vdud = get_vdu(vnfd, vdur["vdu-id-ref"])
1012                 # cloud-init
1013                 if vdud.get("cloud-init-file"):
1014                     vdur["cloud-init"] = "{}:file:{}".format(
1015                         vnfd["_id"], vdud.get("cloud-init-file")
1016                     )
1017                     # read file and put content at target.cloul_init_content. Avoid ng_ro to use shared package system
1018                     if vdur["cloud-init"] not in target["cloud_init_content"]:
1019                         base_folder = vnfd["_admin"]["storage"]
1020                         if base_folder["pkg-dir"]:
1021                             cloud_init_file = "{}/{}/cloud_init/{}".format(
1022                                 base_folder["folder"],
1023                                 base_folder["pkg-dir"],
1024                                 vdud.get("cloud-init-file"),
1025                             )
1026                         else:
1027                             cloud_init_file = "{}/Scripts/cloud_init/{}".format(
1028                                 base_folder["folder"],
1029                                 vdud.get("cloud-init-file"),
1030                             )
1031                         with self.fs.file_open(cloud_init_file, "r") as ci_file:
1032                             target["cloud_init_content"][
1033                                 vdur["cloud-init"]
1034                             ] = ci_file.read()
1035                 elif vdud.get("cloud-init"):
1036                     vdur["cloud-init"] = "{}:vdu:{}".format(
1037                         vnfd["_id"], get_vdu_index(vnfd, vdur["vdu-id-ref"])
1038                     )
1039                     # put content at target.cloul_init_content. Avoid ng_ro read vnfd descriptor
1040                     target["cloud_init_content"][vdur["cloud-init"]] = vdud[
1041                         "cloud-init"
1042                     ]
1043                 vdur["additionalParams"] = vdur.get("additionalParams") or {}
1044                 deploy_params_vdu = self._format_additional_params(
1045                     vdur.get("additionalParams") or {}
1046                 )
1047                 deploy_params_vdu["OSM"] = get_osm_params(
1048                     vnfr, vdur["vdu-id-ref"], vdur["count-index"]
1049                 )
1050                 vdur["additionalParams"] = deploy_params_vdu
1051
1052                 # flavor
1053                 ns_flavor = target["flavor"][int(vdur["ns-flavor-id"])]
1054                 if target_vim not in ns_flavor["vim_info"]:
1055                     ns_flavor["vim_info"][target_vim] = {}
1056
1057                 # deal with images
1058                 # in case alternative images are provided we must check if they should be applied
1059                 # for the vim_type, modify the vim_type taking into account
1060                 ns_image_id = int(vdur["ns-image-id"])
1061                 if vdur.get("alt-image-ids"):
1062                     db_vim = get_vim_account(vnfr["vim-account-id"])
1063                     vim_type = db_vim["vim_type"]
1064                     for alt_image_id in vdur.get("alt-image-ids"):
1065                         ns_alt_image = target["image"][int(alt_image_id)]
1066                         if vim_type == ns_alt_image.get("vim-type"):
1067                             # must use alternative image
1068                             self.logger.debug(
1069                                 "use alternative image id: {}".format(alt_image_id)
1070                             )
1071                             ns_image_id = alt_image_id
1072                             vdur["ns-image-id"] = ns_image_id
1073                             break
1074                 ns_image = target["image"][int(ns_image_id)]
1075                 if target_vim not in ns_image["vim_info"]:
1076                     ns_image["vim_info"][target_vim] = {}
1077
1078                 # Affinity groups
1079                 if vdur.get("affinity-or-anti-affinity-group-id"):
1080                     for ags_id in vdur["affinity-or-anti-affinity-group-id"]:
1081                         ns_ags = target["affinity-or-anti-affinity-group"][int(ags_id)]
1082                         if target_vim not in ns_ags["vim_info"]:
1083                             ns_ags["vim_info"][target_vim] = {}
1084
1085                 # shared-volumes
1086                 if vdur.get("shared-volumes-id"):
1087                     for sv_id in vdur["shared-volumes-id"]:
1088                         ns_sv = find_in_list(
1089                             target["shared-volumes"], lambda sv: sv_id in sv["id"]
1090                         )
1091                         if ns_sv:
1092                             ns_sv["vim_info"][target_vim] = {}
1093
1094                 vdur["vim_info"] = {target_vim: {}}
1095                 # instantiation parameters
1096                 if vnf_params:
1097                     vdu_instantiation_params = find_in_list(
1098                         get_iterable(vnf_params, "vdu"),
1099                         lambda i_vdu: i_vdu["id"] == vdud["id"],
1100                     )
1101                     if vdu_instantiation_params:
1102                         # Parse the vdu_volumes from the instantiation params
1103                         vdu_volumes = get_volumes_from_instantiation_params(
1104                             vdu_instantiation_params, vdud
1105                         )
1106                         vdur["additionalParams"]["OSM"]["vdu_volumes"] = vdu_volumes
1107                         vdur["additionalParams"]["OSM"][
1108                             "vim_flavor_id"
1109                         ] = vdu_instantiation_params.get("vim-flavor-id")
1110                 vdur_list.append(vdur)
1111             target_vnf["vdur"] = vdur_list
1112             target["vnf"].append(target_vnf)
1113
1114         self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
1115         desc = await self.RO.deploy(nsr_id, target)
1116         self.logger.debug("RO return > {}".format(desc))
1117         action_id = desc["action_id"]
1118         await self._wait_ng_ro(
1119             nsr_id,
1120             action_id,
1121             nslcmop_id,
1122             start_deploy,
1123             timeout_ns_deploy,
1124             stage,
1125             operation="instantiation",
1126         )
1127
1128         # Updating NSR
1129         db_nsr_update = {
1130             "_admin.deployed.RO.operational-status": "running",
1131             "detailed-status": " ".join(stage),
1132         }
1133         # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
1134         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1135         self._write_op_status(nslcmop_id, stage)
1136         self.logger.debug(
1137             logging_text + "ns deployed at RO. RO_id={}".format(action_id)
1138         )
1139         return
1140
1141     async def _wait_ng_ro(
1142         self,
1143         nsr_id,
1144         action_id,
1145         nslcmop_id=None,
1146         start_time=None,
1147         timeout=600,
1148         stage=None,
1149         operation=None,
1150     ):
1151         detailed_status_old = None
1152         db_nsr_update = {}
1153         start_time = start_time or time()
1154         while time() <= start_time + timeout:
1155             desc_status = await self.op_status_map[operation](nsr_id, action_id)
1156             self.logger.debug("Wait NG RO > {}".format(desc_status))
1157             if desc_status["status"] == "FAILED":
1158                 raise NgRoException(desc_status["details"])
1159             elif desc_status["status"] == "BUILD":
1160                 if stage:
1161                     stage[2] = "VIM: ({})".format(desc_status["details"])
1162             elif desc_status["status"] == "DONE":
1163                 if stage:
1164                     stage[2] = "Deployed at VIM"
1165                 break
1166             else:
1167                 assert False, "ROclient.check_ns_status returns unknown {}".format(
1168                     desc_status["status"]
1169                 )
1170             if stage and nslcmop_id and stage[2] != detailed_status_old:
1171                 detailed_status_old = stage[2]
1172                 db_nsr_update["detailed-status"] = " ".join(stage)
1173                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1174                 self._write_op_status(nslcmop_id, stage)
1175             await asyncio.sleep(15)
1176         else:  # timeout_ns_deploy
1177             raise NgRoException("Timeout waiting ns to deploy")
1178
1179     async def _terminate_ng_ro(
1180         self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
1181     ):
1182         db_nsr_update = {}
1183         failed_detail = []
1184         action_id = None
1185         start_deploy = time()
1186         try:
1187             target = {
1188                 "ns": {"vld": []},
1189                 "vnf": [],
1190                 "image": [],
1191                 "flavor": [],
1192                 "action_id": nslcmop_id,
1193             }
1194             desc = await self.RO.deploy(nsr_id, target)
1195             action_id = desc["action_id"]
1196             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETING"
1197             self.logger.debug(
1198                 logging_text
1199                 + "ns terminate action at RO. action_id={}".format(action_id)
1200             )
1201
1202             # wait until done
1203             delete_timeout = 20 * 60  # 20 minutes
1204             await self._wait_ng_ro(
1205                 nsr_id,
1206                 action_id,
1207                 nslcmop_id,
1208                 start_deploy,
1209                 delete_timeout,
1210                 stage,
1211                 operation="termination",
1212             )
1213             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1214             # delete all nsr
1215             await self.RO.delete(nsr_id)
1216         except NgRoException as e:
1217             if e.http_code == 404:  # not found
1218                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1219                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1220                 self.logger.debug(
1221                     logging_text + "RO_action_id={} already deleted".format(action_id)
1222                 )
1223             elif e.http_code == 409:  # conflict
1224                 failed_detail.append("delete conflict: {}".format(e))
1225                 self.logger.debug(
1226                     logging_text
1227                     + "RO_action_id={} delete conflict: {}".format(action_id, e)
1228                 )
1229             else:
1230                 failed_detail.append("delete error: {}".format(e))
1231                 self.logger.error(
1232                     logging_text
1233                     + "RO_action_id={} delete error: {}".format(action_id, e)
1234                 )
1235         except Exception as e:
1236             failed_detail.append("delete error: {}".format(e))
1237             self.logger.error(
1238                 logging_text + "RO_action_id={} delete error: {}".format(action_id, e)
1239             )
1240
1241         if failed_detail:
1242             stage[2] = "Error deleting from VIM"
1243         else:
1244             stage[2] = "Deleted from VIM"
1245         db_nsr_update["detailed-status"] = " ".join(stage)
1246         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1247         self._write_op_status(nslcmop_id, stage)
1248
1249         if failed_detail:
1250             raise LcmException("; ".join(failed_detail))
1251         return
1252
1253     async def instantiate_RO(
1254         self,
1255         logging_text,
1256         nsr_id,
1257         nsd,
1258         db_nsr,
1259         db_nslcmop,
1260         db_vnfrs,
1261         db_vnfds,
1262         n2vc_key_list,
1263         stage,
1264     ):
1265         """
1266         Instantiate at RO
1267         :param logging_text: preffix text to use at logging
1268         :param nsr_id: nsr identity
1269         :param nsd: database content of ns descriptor
1270         :param db_nsr: database content of ns record
1271         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
1272         :param db_vnfrs:
1273         :param db_vnfds: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
1274         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
1275         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
1276         :return: None or exception
1277         """
1278         try:
1279             start_deploy = time()
1280             ns_params = db_nslcmop.get("operationParams")
1281             if ns_params and ns_params.get("timeout_ns_deploy"):
1282                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1283             else:
1284                 timeout_ns_deploy = self.timeout.ns_deploy
1285
1286             # Check for and optionally request placement optimization. Database will be updated if placement activated
1287             stage[2] = "Waiting for Placement."
1288             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
1289                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
1290                 for vnfr in db_vnfrs.values():
1291                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
1292                         break
1293                 else:
1294                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
1295
1296             return await self._instantiate_ng_ro(
1297                 logging_text,
1298                 nsr_id,
1299                 nsd,
1300                 db_nsr,
1301                 db_nslcmop,
1302                 db_vnfrs,
1303                 db_vnfds,
1304                 n2vc_key_list,
1305                 stage,
1306                 start_deploy,
1307                 timeout_ns_deploy,
1308             )
1309         except Exception as e:
1310             stage[2] = "ERROR deploying at VIM"
1311             self.set_vnfr_at_error(db_vnfrs, str(e))
1312             self.logger.error(
1313                 "Error deploying at VIM {}".format(e),
1314                 exc_info=not isinstance(
1315                     e,
1316                     (
1317                         ROclient.ROClientException,
1318                         LcmException,
1319                         DbException,
1320                         NgRoException,
1321                     ),
1322                 ),
1323             )
1324             raise
1325
1326     async def wait_kdu_up(self, logging_text, nsr_id, vnfr_id, kdu_name):
1327         """
1328         Wait for kdu to be up, get ip address
1329         :param logging_text: prefix use for logging
1330         :param nsr_id:
1331         :param vnfr_id:
1332         :param kdu_name:
1333         :return: IP address, K8s services
1334         """
1335
1336         # self.logger.debug(logging_text + "Starting wait_kdu_up")
1337         nb_tries = 0
1338
1339         while nb_tries < 360:
1340             db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1341             kdur = next(
1342                 (
1343                     x
1344                     for x in get_iterable(db_vnfr, "kdur")
1345                     if x.get("kdu-name") == kdu_name
1346                 ),
1347                 None,
1348             )
1349             if not kdur:
1350                 raise LcmException(
1351                     "Not found vnfr_id={}, kdu_name={}".format(vnfr_id, kdu_name)
1352                 )
1353             if kdur.get("status"):
1354                 if kdur["status"] in ("READY", "ENABLED"):
1355                     return kdur.get("ip-address"), kdur.get("services")
1356                 else:
1357                     raise LcmException(
1358                         "target KDU={} is in error state".format(kdu_name)
1359                     )
1360
1361             await asyncio.sleep(10)
1362             nb_tries += 1
1363         raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
1364
1365     async def wait_vm_up_insert_key_ro(
1366         self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None
1367     ):
1368         """
1369         Wait for ip addres at RO, and optionally, insert public key in virtual machine
1370         :param logging_text: prefix use for logging
1371         :param nsr_id:
1372         :param vnfr_id:
1373         :param vdu_id:
1374         :param vdu_index:
1375         :param pub_key: public ssh key to inject, None to skip
1376         :param user: user to apply the public ssh key
1377         :return: IP address
1378         """
1379
1380         self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1381         ip_address = None
1382         target_vdu_id = None
1383         ro_retries = 0
1384
1385         while True:
1386             ro_retries += 1
1387             if ro_retries >= 360:  # 1 hour
1388                 raise LcmException(
1389                     "Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id)
1390                 )
1391
1392             await asyncio.sleep(10)
1393
1394             # get ip address
1395             if not target_vdu_id:
1396                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1397
1398                 if not vdu_id:  # for the VNF case
1399                     if db_vnfr.get("status") == "ERROR":
1400                         raise LcmException(
1401                             "Cannot inject ssh-key because target VNF is in error state"
1402                         )
1403                     ip_address = db_vnfr.get("ip-address")
1404                     if not ip_address:
1405                         continue
1406                     vdur = next(
1407                         (
1408                             x
1409                             for x in get_iterable(db_vnfr, "vdur")
1410                             if x.get("ip-address") == ip_address
1411                         ),
1412                         None,
1413                     )
1414                 else:  # VDU case
1415                     vdur = next(
1416                         (
1417                             x
1418                             for x in get_iterable(db_vnfr, "vdur")
1419                             if x.get("vdu-id-ref") == vdu_id
1420                             and x.get("count-index") == vdu_index
1421                         ),
1422                         None,
1423                     )
1424
1425                 if (
1426                     not vdur and len(db_vnfr.get("vdur", ())) == 1
1427                 ):  # If only one, this should be the target vdu
1428                     vdur = db_vnfr["vdur"][0]
1429                 if not vdur:
1430                     raise LcmException(
1431                         "Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(
1432                             vnfr_id, vdu_id, vdu_index
1433                         )
1434                     )
1435                 # New generation RO stores information at "vim_info"
1436                 ng_ro_status = None
1437                 target_vim = None
1438                 if vdur.get("vim_info"):
1439                     target_vim = next(
1440                         t for t in vdur["vim_info"]
1441                     )  # there should be only one key
1442                     ng_ro_status = vdur["vim_info"][target_vim].get("vim_status")
1443                 if (
1444                     vdur.get("pdu-type")
1445                     or vdur.get("status") == "ACTIVE"
1446                     or ng_ro_status == "ACTIVE"
1447                 ):
1448                     ip_address = vdur.get("ip-address")
1449                     if not ip_address:
1450                         continue
1451                     target_vdu_id = vdur["vdu-id-ref"]
1452                 elif vdur.get("status") == "ERROR" or ng_ro_status == "ERROR":
1453                     raise LcmException(
1454                         "Cannot inject ssh-key because target VM is in error state"
1455                     )
1456
1457             if not target_vdu_id:
1458                 continue
1459
1460             # inject public key into machine
1461             if pub_key and user:
1462                 self.logger.debug(logging_text + "Inserting RO key")
1463                 self.logger.debug("SSH > PubKey > {}".format(pub_key))
1464                 if vdur.get("pdu-type"):
1465                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1466                     return ip_address
1467                 try:
1468                     target = {
1469                         "action": {
1470                             "action": "inject_ssh_key",
1471                             "key": pub_key,
1472                             "user": user,
1473                         },
1474                         "vnf": [{"_id": vnfr_id, "vdur": [{"id": vdur["id"]}]}],
1475                     }
1476                     desc = await self.RO.deploy(nsr_id, target)
1477                     action_id = desc["action_id"]
1478                     await self._wait_ng_ro(
1479                         nsr_id, action_id, timeout=600, operation="instantiation"
1480                     )
1481                     break
1482                 except NgRoException as e:
1483                     raise LcmException(
1484                         "Reaching max tries injecting key. Error: {}".format(e)
1485                     )
1486             else:
1487                 break
1488
1489         return ip_address
1490
1491     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1492         """
1493         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1494         """
1495         my_vca = vca_deployed_list[vca_index]
1496         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1497             # vdu or kdu: no dependencies
1498             return
1499         timeout = 300
1500         while timeout >= 0:
1501             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1502             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1503             configuration_status_list = db_nsr["configurationStatus"]
1504             for index, vca_deployed in enumerate(configuration_status_list):
1505                 if index == vca_index:
1506                     # myself
1507                     continue
1508                 if not my_vca.get("member-vnf-index") or (
1509                     vca_deployed.get("member-vnf-index")
1510                     == my_vca.get("member-vnf-index")
1511                 ):
1512                     internal_status = configuration_status_list[index].get("status")
1513                     if internal_status == "READY":
1514                         continue
1515                     elif internal_status == "BROKEN":
1516                         raise LcmException(
1517                             "Configuration aborted because dependent charm/s has failed"
1518                         )
1519                     else:
1520                         break
1521             else:
1522                 # no dependencies, return
1523                 return
1524             await asyncio.sleep(10)
1525             timeout -= 1
1526
1527         raise LcmException("Configuration aborted because dependent charm/s timeout")
1528
1529     def get_vca_id(self, db_vnfr: dict, db_nsr: dict):
1530         vca_id = None
1531         if db_vnfr:
1532             vca_id = deep_get(db_vnfr, ("vca-id",))
1533         elif db_nsr:
1534             vim_account_id = deep_get(db_nsr, ("instantiate_params", "vimAccountId"))
1535             vca_id = VimAccountDB.get_vim_account_with_id(vim_account_id).get("vca")
1536         return vca_id
1537
1538     async def instantiate_N2VC(
1539         self,
1540         logging_text,
1541         vca_index,
1542         nsi_id,
1543         db_nsr,
1544         db_vnfr,
1545         vdu_id,
1546         kdu_name,
1547         vdu_index,
1548         kdu_index,
1549         config_descriptor,
1550         deploy_params,
1551         base_folder,
1552         nslcmop_id,
1553         stage,
1554         vca_type,
1555         vca_name,
1556         ee_config_descriptor,
1557     ):
1558         nsr_id = db_nsr["_id"]
1559         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1560         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1561         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1562         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
1563         db_dict = {
1564             "collection": "nsrs",
1565             "filter": {"_id": nsr_id},
1566             "path": db_update_entry,
1567         }
1568         step = ""
1569         try:
1570             element_type = "NS"
1571             element_under_configuration = nsr_id
1572
1573             vnfr_id = None
1574             if db_vnfr:
1575                 vnfr_id = db_vnfr["_id"]
1576                 osm_config["osm"]["vnf_id"] = vnfr_id
1577
1578             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
1579
1580             if vca_type == "native_charm":
1581                 index_number = 0
1582             else:
1583                 index_number = vdu_index or 0
1584
1585             if vnfr_id:
1586                 element_type = "VNF"
1587                 element_under_configuration = vnfr_id
1588                 namespace += ".{}-{}".format(vnfr_id, index_number)
1589                 if vdu_id:
1590                     namespace += ".{}-{}".format(vdu_id, index_number)
1591                     element_type = "VDU"
1592                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
1593                     osm_config["osm"]["vdu_id"] = vdu_id
1594                 elif kdu_name:
1595                     namespace += ".{}".format(kdu_name)
1596                     element_type = "KDU"
1597                     element_under_configuration = kdu_name
1598                     osm_config["osm"]["kdu_name"] = kdu_name
1599
1600             # Get artifact path
1601             if base_folder["pkg-dir"]:
1602                 artifact_path = "{}/{}/{}/{}".format(
1603                     base_folder["folder"],
1604                     base_folder["pkg-dir"],
1605                     "charms"
1606                     if vca_type
1607                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1608                     else "helm-charts",
1609                     vca_name,
1610                 )
1611             else:
1612                 artifact_path = "{}/Scripts/{}/{}/".format(
1613                     base_folder["folder"],
1614                     "charms"
1615                     if vca_type
1616                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1617                     else "helm-charts",
1618                     vca_name,
1619                 )
1620
1621             self.logger.debug("Artifact path > {}".format(artifact_path))
1622
1623             # get initial_config_primitive_list that applies to this element
1624             initial_config_primitive_list = config_descriptor.get(
1625                 "initial-config-primitive"
1626             )
1627
1628             self.logger.debug(
1629                 "Initial config primitive list > {}".format(
1630                     initial_config_primitive_list
1631                 )
1632             )
1633
1634             # add config if not present for NS charm
1635             ee_descriptor_id = ee_config_descriptor.get("id")
1636             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
1637             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
1638                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
1639             )
1640
1641             self.logger.debug(
1642                 "Initial config primitive list #2 > {}".format(
1643                     initial_config_primitive_list
1644                 )
1645             )
1646             # n2vc_redesign STEP 3.1
1647             # find old ee_id if exists
1648             ee_id = vca_deployed.get("ee_id")
1649
1650             vca_id = self.get_vca_id(db_vnfr, db_nsr)
1651             # create or register execution environment in VCA
1652             if vca_type in ("lxc_proxy_charm", "k8s_proxy_charm", "helm-v3"):
1653                 self._write_configuration_status(
1654                     nsr_id=nsr_id,
1655                     vca_index=vca_index,
1656                     status="CREATING",
1657                     element_under_configuration=element_under_configuration,
1658                     element_type=element_type,
1659                 )
1660
1661                 step = "create execution environment"
1662                 self.logger.debug(logging_text + step)
1663
1664                 ee_id = None
1665                 credentials = None
1666                 if vca_type == "k8s_proxy_charm":
1667                     ee_id = await self.vca_map[vca_type].install_k8s_proxy_charm(
1668                         charm_name=artifact_path[artifact_path.rfind("/") + 1 :],
1669                         namespace=namespace,
1670                         artifact_path=artifact_path,
1671                         db_dict=db_dict,
1672                         vca_id=vca_id,
1673                     )
1674                 elif vca_type == "helm-v3":
1675                     ee_id, credentials = await self.vca_map[
1676                         vca_type
1677                     ].create_execution_environment(
1678                         namespace=nsr_id,
1679                         reuse_ee_id=ee_id,
1680                         db_dict=db_dict,
1681                         config=osm_config,
1682                         artifact_path=artifact_path,
1683                         chart_model=vca_name,
1684                         vca_type=vca_type,
1685                     )
1686                 else:
1687                     ee_id, credentials = await self.vca_map[
1688                         vca_type
1689                     ].create_execution_environment(
1690                         namespace=namespace,
1691                         reuse_ee_id=ee_id,
1692                         db_dict=db_dict,
1693                         vca_id=vca_id,
1694                     )
1695
1696             elif vca_type == "native_charm":
1697                 step = "Waiting to VM being up and getting IP address"
1698                 self.logger.debug(logging_text + step)
1699                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
1700                     logging_text,
1701                     nsr_id,
1702                     vnfr_id,
1703                     vdu_id,
1704                     vdu_index,
1705                     user=None,
1706                     pub_key=None,
1707                 )
1708                 credentials = {"hostname": rw_mgmt_ip}
1709                 # get username
1710                 username = deep_get(
1711                     config_descriptor, ("config-access", "ssh-access", "default-user")
1712                 )
1713                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1714                 #  merged. Meanwhile let's get username from initial-config-primitive
1715                 if not username and initial_config_primitive_list:
1716                     for config_primitive in initial_config_primitive_list:
1717                         for param in config_primitive.get("parameter", ()):
1718                             if param["name"] == "ssh-username":
1719                                 username = param["value"]
1720                                 break
1721                 if not username:
1722                     raise LcmException(
1723                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
1724                         "'config-access.ssh-access.default-user'"
1725                     )
1726                 credentials["username"] = username
1727                 # n2vc_redesign STEP 3.2
1728
1729                 self._write_configuration_status(
1730                     nsr_id=nsr_id,
1731                     vca_index=vca_index,
1732                     status="REGISTERING",
1733                     element_under_configuration=element_under_configuration,
1734                     element_type=element_type,
1735                 )
1736
1737                 step = "register execution environment {}".format(credentials)
1738                 self.logger.debug(logging_text + step)
1739                 ee_id = await self.vca_map[vca_type].register_execution_environment(
1740                     credentials=credentials,
1741                     namespace=namespace,
1742                     db_dict=db_dict,
1743                     vca_id=vca_id,
1744                 )
1745
1746             # for compatibility with MON/POL modules, the need model and application name at database
1747             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
1748             ee_id_parts = ee_id.split(".")
1749             db_nsr_update = {db_update_entry + "ee_id": ee_id}
1750             if len(ee_id_parts) >= 2:
1751                 model_name = ee_id_parts[0]
1752                 application_name = ee_id_parts[1]
1753                 db_nsr_update[db_update_entry + "model"] = model_name
1754                 db_nsr_update[db_update_entry + "application"] = application_name
1755
1756             # n2vc_redesign STEP 3.3
1757             step = "Install configuration Software"
1758
1759             self._write_configuration_status(
1760                 nsr_id=nsr_id,
1761                 vca_index=vca_index,
1762                 status="INSTALLING SW",
1763                 element_under_configuration=element_under_configuration,
1764                 element_type=element_type,
1765                 other_update=db_nsr_update,
1766             )
1767
1768             # TODO check if already done
1769             self.logger.debug(logging_text + step)
1770             config = None
1771             if vca_type == "native_charm":
1772                 config_primitive = next(
1773                     (p for p in initial_config_primitive_list if p["name"] == "config"),
1774                     None,
1775                 )
1776                 if config_primitive:
1777                     config = self._map_primitive_params(
1778                         config_primitive, {}, deploy_params
1779                     )
1780             num_units = 1
1781             if vca_type == "lxc_proxy_charm":
1782                 if element_type == "NS":
1783                     num_units = db_nsr.get("config-units") or 1
1784                 elif element_type == "VNF":
1785                     num_units = db_vnfr.get("config-units") or 1
1786                 elif element_type == "VDU":
1787                     for v in db_vnfr["vdur"]:
1788                         if vdu_id == v["vdu-id-ref"]:
1789                             num_units = v.get("config-units") or 1
1790                             break
1791             if vca_type != "k8s_proxy_charm":
1792                 await self.vca_map[vca_type].install_configuration_sw(
1793                     ee_id=ee_id,
1794                     artifact_path=artifact_path,
1795                     db_dict=db_dict,
1796                     config=config,
1797                     num_units=num_units,
1798                     vca_id=vca_id,
1799                     vca_type=vca_type,
1800                 )
1801
1802             # write in db flag of configuration_sw already installed
1803             self.update_db_2(
1804                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
1805             )
1806
1807             # add relations for this VCA (wait for other peers related with this VCA)
1808             is_relation_added = await self._add_vca_relations(
1809                 logging_text=logging_text,
1810                 nsr_id=nsr_id,
1811                 vca_type=vca_type,
1812                 vca_index=vca_index,
1813             )
1814
1815             if not is_relation_added:
1816                 raise LcmException("Relations could not be added to VCA.")
1817
1818             # if SSH access is required, then get execution environment SSH public
1819             # if native charm we have waited already to VM be UP
1820             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm-v3"):
1821                 pub_key = None
1822                 user = None
1823                 # self.logger.debug("get ssh key block")
1824                 if deep_get(
1825                     config_descriptor, ("config-access", "ssh-access", "required")
1826                 ):
1827                     # self.logger.debug("ssh key needed")
1828                     # Needed to inject a ssh key
1829                     user = deep_get(
1830                         config_descriptor,
1831                         ("config-access", "ssh-access", "default-user"),
1832                     )
1833                     step = "Install configuration Software, getting public ssh key"
1834                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
1835                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
1836                     )
1837
1838                     step = "Insert public key into VM user={} ssh_key={}".format(
1839                         user, pub_key
1840                     )
1841                 else:
1842                     # self.logger.debug("no need to get ssh key")
1843                     step = "Waiting to VM being up and getting IP address"
1844                 self.logger.debug(logging_text + step)
1845
1846                 # default rw_mgmt_ip to None, avoiding the non definition of the variable
1847                 rw_mgmt_ip = None
1848
1849                 # n2vc_redesign STEP 5.1
1850                 # wait for RO (ip-address) Insert pub_key into VM
1851                 if vnfr_id:
1852                     if kdu_name:
1853                         rw_mgmt_ip, services = await self.wait_kdu_up(
1854                             logging_text, nsr_id, vnfr_id, kdu_name
1855                         )
1856                         vnfd = self.db.get_one(
1857                             "vnfds_revisions",
1858                             {"_id": f'{db_vnfr["vnfd-id"]}:{db_vnfr["revision"]}'},
1859                         )
1860                         kdu = get_kdu(vnfd, kdu_name)
1861                         kdu_services = [
1862                             service["name"] for service in get_kdu_services(kdu)
1863                         ]
1864                         exposed_services = []
1865                         for service in services:
1866                             if any(s in service["name"] for s in kdu_services):
1867                                 exposed_services.append(service)
1868                         await self.vca_map[vca_type].exec_primitive(
1869                             ee_id=ee_id,
1870                             primitive_name="config",
1871                             params_dict={
1872                                 "osm-config": json.dumps(
1873                                     OsmConfigBuilder(
1874                                         k8s={"services": exposed_services}
1875                                     ).build()
1876                                 )
1877                             },
1878                             vca_id=vca_id,
1879                         )
1880
1881                     # This verification is needed in order to avoid trying to add a public key
1882                     # to a VM, when the VNF is a KNF (in the edge case where the user creates a VCA
1883                     # for a KNF and not for its KDUs, the previous verification gives False, and the code
1884                     # jumps to this block, meaning that there is the need to verify if the VNF is actually a VNF
1885                     # or it is a KNF)
1886                     elif db_vnfr.get("vdur"):
1887                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
1888                             logging_text,
1889                             nsr_id,
1890                             vnfr_id,
1891                             vdu_id,
1892                             vdu_index,
1893                             user=user,
1894                             pub_key=pub_key,
1895                         )
1896
1897                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
1898
1899             # store rw_mgmt_ip in deploy params for later replacement
1900             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
1901
1902             # n2vc_redesign STEP 6  Execute initial config primitive
1903             step = "execute initial config primitive"
1904
1905             # wait for dependent primitives execution (NS -> VNF -> VDU)
1906             if initial_config_primitive_list:
1907                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
1908
1909             # stage, in function of element type: vdu, kdu, vnf or ns
1910             my_vca = vca_deployed_list[vca_index]
1911             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1912                 # VDU or KDU
1913                 stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
1914             elif my_vca.get("member-vnf-index"):
1915                 # VNF
1916                 stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
1917             else:
1918                 # NS
1919                 stage[0] = "Stage 5/5: running Day-1 primitives for NS."
1920
1921             self._write_configuration_status(
1922                 nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
1923             )
1924
1925             self._write_op_status(op_id=nslcmop_id, stage=stage)
1926
1927             check_if_terminated_needed = True
1928             for initial_config_primitive in initial_config_primitive_list:
1929                 # adding information on the vca_deployed if it is a NS execution environment
1930                 if not vca_deployed["member-vnf-index"]:
1931                     deploy_params["ns_config_info"] = json.dumps(
1932                         self._get_ns_config_info(nsr_id)
1933                     )
1934                 # TODO check if already done
1935                 primitive_params_ = self._map_primitive_params(
1936                     initial_config_primitive, {}, deploy_params
1937                 )
1938
1939                 step = "execute primitive '{}' params '{}'".format(
1940                     initial_config_primitive["name"], primitive_params_
1941                 )
1942                 self.logger.debug(logging_text + step)
1943                 await self.vca_map[vca_type].exec_primitive(
1944                     ee_id=ee_id,
1945                     primitive_name=initial_config_primitive["name"],
1946                     params_dict=primitive_params_,
1947                     db_dict=db_dict,
1948                     vca_id=vca_id,
1949                     vca_type=vca_type,
1950                 )
1951                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
1952                 if check_if_terminated_needed:
1953                     if config_descriptor.get("terminate-config-primitive"):
1954                         self.update_db_2(
1955                             "nsrs", nsr_id, {db_update_entry + "needed_terminate": True}
1956                         )
1957                     check_if_terminated_needed = False
1958
1959                 # TODO register in database that primitive is done
1960
1961             # STEP 7 Configure metrics
1962             if vca_type == "helm-v3":
1963                 # TODO: review for those cases where the helm chart is a reference and
1964                 # is not part of the NF package
1965                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
1966                     ee_id=ee_id,
1967                     artifact_path=artifact_path,
1968                     ee_config_descriptor=ee_config_descriptor,
1969                     vnfr_id=vnfr_id,
1970                     nsr_id=nsr_id,
1971                     target_ip=rw_mgmt_ip,
1972                     element_type=element_type,
1973                     vnf_member_index=db_vnfr.get("member-vnf-index-ref", ""),
1974                     vdu_id=vdu_id,
1975                     vdu_index=vdu_index,
1976                     kdu_name=kdu_name,
1977                     kdu_index=kdu_index,
1978                 )
1979                 if prometheus_jobs:
1980                     self.update_db_2(
1981                         "nsrs",
1982                         nsr_id,
1983                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
1984                     )
1985
1986                     for job in prometheus_jobs:
1987                         self.db.set_one(
1988                             "prometheus_jobs",
1989                             {"job_name": job["job_name"]},
1990                             job,
1991                             upsert=True,
1992                             fail_on_empty=False,
1993                         )
1994
1995             step = "instantiated at VCA"
1996             self.logger.debug(logging_text + step)
1997
1998             self._write_configuration_status(
1999                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
2000             )
2001
2002         except Exception as e:  # TODO not use Exception but N2VC exception
2003             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
2004             if not isinstance(
2005                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
2006             ):
2007                 self.logger.error(
2008                     "Exception while {} : {}".format(step, e), exc_info=True
2009                 )
2010             self._write_configuration_status(
2011                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
2012             )
2013             raise LcmException("{}. {}".format(step, e)) from e
2014
2015     def _write_ns_status(
2016         self,
2017         nsr_id: str,
2018         ns_state: str,
2019         current_operation: str,
2020         current_operation_id: str,
2021         error_description: str = None,
2022         error_detail: str = None,
2023         other_update: dict = None,
2024     ):
2025         """
2026         Update db_nsr fields.
2027         :param nsr_id:
2028         :param ns_state:
2029         :param current_operation:
2030         :param current_operation_id:
2031         :param error_description:
2032         :param error_detail:
2033         :param other_update: Other required changes at database if provided, will be cleared
2034         :return:
2035         """
2036         try:
2037             db_dict = other_update or {}
2038             db_dict[
2039                 "_admin.nslcmop"
2040             ] = current_operation_id  # for backward compatibility
2041             db_dict["_admin.current-operation"] = current_operation_id
2042             db_dict["_admin.operation-type"] = (
2043                 current_operation if current_operation != "IDLE" else None
2044             )
2045             db_dict["currentOperation"] = current_operation
2046             db_dict["currentOperationID"] = current_operation_id
2047             db_dict["errorDescription"] = error_description
2048             db_dict["errorDetail"] = error_detail
2049
2050             if ns_state:
2051                 db_dict["nsState"] = ns_state
2052             self.update_db_2("nsrs", nsr_id, db_dict)
2053         except DbException as e:
2054             self.logger.warn("Error writing NS status, ns={}: {}".format(nsr_id, e))
2055
2056     def _write_op_status(
2057         self,
2058         op_id: str,
2059         stage: list = None,
2060         error_message: str = None,
2061         queuePosition: int = 0,
2062         operation_state: str = None,
2063         other_update: dict = None,
2064     ):
2065         try:
2066             db_dict = other_update or {}
2067             db_dict["queuePosition"] = queuePosition
2068             if isinstance(stage, list):
2069                 db_dict["stage"] = stage[0]
2070                 db_dict["detailed-status"] = " ".join(stage)
2071             elif stage is not None:
2072                 db_dict["stage"] = str(stage)
2073
2074             if error_message is not None:
2075                 db_dict["errorMessage"] = error_message
2076             if operation_state is not None:
2077                 db_dict["operationState"] = operation_state
2078                 db_dict["statusEnteredTime"] = time()
2079             self.update_db_2("nslcmops", op_id, db_dict)
2080         except DbException as e:
2081             self.logger.warn(
2082                 "Error writing OPERATION status for op_id: {} -> {}".format(op_id, e)
2083             )
2084
2085     def _write_all_config_status(self, db_nsr: dict, status: str):
2086         try:
2087             nsr_id = db_nsr["_id"]
2088             # configurationStatus
2089             config_status = db_nsr.get("configurationStatus")
2090             if config_status:
2091                 db_nsr_update = {
2092                     "configurationStatus.{}.status".format(index): status
2093                     for index, v in enumerate(config_status)
2094                     if v
2095                 }
2096                 # update status
2097                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2098
2099         except DbException as e:
2100             self.logger.warn(
2101                 "Error writing all configuration status, ns={}: {}".format(nsr_id, e)
2102             )
2103
2104     def _write_configuration_status(
2105         self,
2106         nsr_id: str,
2107         vca_index: int,
2108         status: str = None,
2109         element_under_configuration: str = None,
2110         element_type: str = None,
2111         other_update: dict = None,
2112     ):
2113         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
2114         #                   .format(vca_index, status))
2115
2116         try:
2117             db_path = "configurationStatus.{}.".format(vca_index)
2118             db_dict = other_update or {}
2119             if status:
2120                 db_dict[db_path + "status"] = status
2121             if element_under_configuration:
2122                 db_dict[
2123                     db_path + "elementUnderConfiguration"
2124                 ] = element_under_configuration
2125             if element_type:
2126                 db_dict[db_path + "elementType"] = element_type
2127             self.update_db_2("nsrs", nsr_id, db_dict)
2128         except DbException as e:
2129             self.logger.warn(
2130                 "Error writing configuration status={}, ns={}, vca_index={}: {}".format(
2131                     status, nsr_id, vca_index, e
2132                 )
2133             )
2134
2135     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
2136         """
2137         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
2138         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
2139         Database is used because the result can be obtained from a different LCM worker in case of HA.
2140         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
2141         :param db_nslcmop: database content of nslcmop
2142         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
2143         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
2144             computed 'vim-account-id'
2145         """
2146         modified = False
2147         nslcmop_id = db_nslcmop["_id"]
2148         placement_engine = deep_get(db_nslcmop, ("operationParams", "placement-engine"))
2149         if placement_engine == "PLA":
2150             self.logger.debug(
2151                 logging_text + "Invoke and wait for placement optimization"
2152             )
2153             await self.msg.aiowrite("pla", "get_placement", {"nslcmopId": nslcmop_id})
2154             db_poll_interval = 5
2155             wait = db_poll_interval * 10
2156             pla_result = None
2157             while not pla_result and wait >= 0:
2158                 await asyncio.sleep(db_poll_interval)
2159                 wait -= db_poll_interval
2160                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2161                 pla_result = deep_get(db_nslcmop, ("_admin", "pla"))
2162
2163             if not pla_result:
2164                 raise LcmException(
2165                     "Placement timeout for nslcmopId={}".format(nslcmop_id)
2166                 )
2167
2168             for pla_vnf in pla_result["vnf"]:
2169                 vnfr = db_vnfrs.get(pla_vnf["member-vnf-index"])
2170                 if not pla_vnf.get("vimAccountId") or not vnfr:
2171                     continue
2172                 modified = True
2173                 self.db.set_one(
2174                     "vnfrs",
2175                     {"_id": vnfr["_id"]},
2176                     {"vim-account-id": pla_vnf["vimAccountId"]},
2177                 )
2178                 # Modifies db_vnfrs
2179                 vnfr["vim-account-id"] = pla_vnf["vimAccountId"]
2180         return modified
2181
2182     def _gather_vnfr_healing_alerts(self, vnfr, vnfd):
2183         alerts = []
2184         nsr_id = vnfr["nsr-id-ref"]
2185         df = vnfd.get("df", [{}])[0]
2186         # Checking for auto-healing configuration
2187         if "healing-aspect" in df:
2188             healing_aspects = df["healing-aspect"]
2189             for healing in healing_aspects:
2190                 for healing_policy in healing.get("healing-policy", ()):
2191                     vdu_id = healing_policy["vdu-id"]
2192                     vdur = next(
2193                         (vdur for vdur in vnfr["vdur"] if vdu_id == vdur["vdu-id-ref"]),
2194                         {},
2195                     )
2196                     if not vdur:
2197                         continue
2198                     metric_name = "vm_status"
2199                     vdu_name = vdur.get("name")
2200                     vnf_member_index = vnfr["member-vnf-index-ref"]
2201                     uuid = str(uuid4())
2202                     name = f"healing_{uuid}"
2203                     action = healing_policy
2204                     # action_on_recovery = healing.get("action-on-recovery")
2205                     # cooldown_time = healing.get("cooldown-time")
2206                     # day1 = healing.get("day1")
2207                     alert = {
2208                         "uuid": uuid,
2209                         "name": name,
2210                         "metric": metric_name,
2211                         "tags": {
2212                             "ns_id": nsr_id,
2213                             "vnf_member_index": vnf_member_index,
2214                             "vdu_name": vdu_name,
2215                         },
2216                         "alarm_status": "ok",
2217                         "action_type": "healing",
2218                         "action": action,
2219                     }
2220                     alerts.append(alert)
2221         return alerts
2222
2223     def _gather_vnfr_scaling_alerts(self, vnfr, vnfd):
2224         alerts = []
2225         nsr_id = vnfr["nsr-id-ref"]
2226         df = vnfd.get("df", [{}])[0]
2227         # Checking for auto-scaling configuration
2228         if "scaling-aspect" in df:
2229             scaling_aspects = df["scaling-aspect"]
2230             all_vnfd_monitoring_params = {}
2231             for ivld in vnfd.get("int-virtual-link-desc", ()):
2232                 for mp in ivld.get("monitoring-parameters", ()):
2233                     all_vnfd_monitoring_params[mp.get("id")] = mp
2234             for vdu in vnfd.get("vdu", ()):
2235                 for mp in vdu.get("monitoring-parameter", ()):
2236                     all_vnfd_monitoring_params[mp.get("id")] = mp
2237             for df in vnfd.get("df", ()):
2238                 for mp in df.get("monitoring-parameter", ()):
2239                     all_vnfd_monitoring_params[mp.get("id")] = mp
2240             for scaling_aspect in scaling_aspects:
2241                 scaling_group_name = scaling_aspect.get("name", "")
2242                 # Get monitored VDUs
2243                 all_monitored_vdus = set()
2244                 for delta in scaling_aspect.get("aspect-delta-details", {}).get(
2245                     "deltas", ()
2246                 ):
2247                     for vdu_delta in delta.get("vdu-delta", ()):
2248                         all_monitored_vdus.add(vdu_delta.get("id"))
2249                 monitored_vdurs = list(
2250                     filter(
2251                         lambda vdur: vdur["vdu-id-ref"] in all_monitored_vdus,
2252                         vnfr["vdur"],
2253                     )
2254                 )
2255                 if not monitored_vdurs:
2256                     self.logger.error(
2257                         "Scaling criteria is referring to a vnf-monitoring-param that does not contain a reference to a vdu or vnf metric"
2258                     )
2259                     continue
2260                 for scaling_policy in scaling_aspect.get("scaling-policy", ()):
2261                     if scaling_policy["scaling-type"] != "automatic":
2262                         continue
2263                     threshold_time = scaling_policy.get("threshold-time", "1")
2264                     cooldown_time = scaling_policy.get("cooldown-time", "0")
2265                     for scaling_criteria in scaling_policy["scaling-criteria"]:
2266                         monitoring_param_ref = scaling_criteria.get(
2267                             "vnf-monitoring-param-ref"
2268                         )
2269                         vnf_monitoring_param = all_vnfd_monitoring_params[
2270                             monitoring_param_ref
2271                         ]
2272                         for vdur in monitored_vdurs:
2273                             vdu_id = vdur["vdu-id-ref"]
2274                             metric_name = vnf_monitoring_param.get("performance-metric")
2275                             metric_name = f"osm_{metric_name}"
2276                             vnf_member_index = vnfr["member-vnf-index-ref"]
2277                             scalein_threshold = scaling_criteria.get(
2278                                 "scale-in-threshold"
2279                             )
2280                             scaleout_threshold = scaling_criteria.get(
2281                                 "scale-out-threshold"
2282                             )
2283                             # Looking for min/max-number-of-instances
2284                             instances_min_number = 1
2285                             instances_max_number = 1
2286                             vdu_profile = df["vdu-profile"]
2287                             if vdu_profile:
2288                                 profile = next(
2289                                     item for item in vdu_profile if item["id"] == vdu_id
2290                                 )
2291                                 instances_min_number = profile.get(
2292                                     "min-number-of-instances", 1
2293                                 )
2294                                 instances_max_number = profile.get(
2295                                     "max-number-of-instances", 1
2296                                 )
2297
2298                             if scalein_threshold:
2299                                 uuid = str(uuid4())
2300                                 name = f"scalein_{uuid}"
2301                                 operation = scaling_criteria[
2302                                     "scale-in-relational-operation"
2303                                 ]
2304                                 rel_operator = self.rel_operation_types.get(
2305                                     operation, "<="
2306                                 )
2307                                 metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2308                                 expression = f"(count ({metric_selector}) > {instances_min_number}) and (avg({metric_selector}) {rel_operator} {scalein_threshold})"
2309                                 labels = {
2310                                     "ns_id": nsr_id,
2311                                     "vnf_member_index": vnf_member_index,
2312                                     "vdu_id": vdu_id,
2313                                 }
2314                                 prom_cfg = {
2315                                     "alert": name,
2316                                     "expr": expression,
2317                                     "for": str(threshold_time) + "m",
2318                                     "labels": labels,
2319                                 }
2320                                 action = scaling_policy
2321                                 action = {
2322                                     "scaling-group": scaling_group_name,
2323                                     "cooldown-time": cooldown_time,
2324                                 }
2325                                 alert = {
2326                                     "uuid": uuid,
2327                                     "name": name,
2328                                     "metric": metric_name,
2329                                     "tags": {
2330                                         "ns_id": nsr_id,
2331                                         "vnf_member_index": vnf_member_index,
2332                                         "vdu_id": vdu_id,
2333                                     },
2334                                     "alarm_status": "ok",
2335                                     "action_type": "scale_in",
2336                                     "action": action,
2337                                     "prometheus_config": prom_cfg,
2338                                 }
2339                                 alerts.append(alert)
2340
2341                             if scaleout_threshold:
2342                                 uuid = str(uuid4())
2343                                 name = f"scaleout_{uuid}"
2344                                 operation = scaling_criteria[
2345                                     "scale-out-relational-operation"
2346                                 ]
2347                                 rel_operator = self.rel_operation_types.get(
2348                                     operation, "<="
2349                                 )
2350                                 metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2351                                 expression = f"(count ({metric_selector}) < {instances_max_number}) and (avg({metric_selector}) {rel_operator} {scaleout_threshold})"
2352                                 labels = {
2353                                     "ns_id": nsr_id,
2354                                     "vnf_member_index": vnf_member_index,
2355                                     "vdu_id": vdu_id,
2356                                 }
2357                                 prom_cfg = {
2358                                     "alert": name,
2359                                     "expr": expression,
2360                                     "for": str(threshold_time) + "m",
2361                                     "labels": labels,
2362                                 }
2363                                 action = scaling_policy
2364                                 action = {
2365                                     "scaling-group": scaling_group_name,
2366                                     "cooldown-time": cooldown_time,
2367                                 }
2368                                 alert = {
2369                                     "uuid": uuid,
2370                                     "name": name,
2371                                     "metric": metric_name,
2372                                     "tags": {
2373                                         "ns_id": nsr_id,
2374                                         "vnf_member_index": vnf_member_index,
2375                                         "vdu_id": vdu_id,
2376                                     },
2377                                     "alarm_status": "ok",
2378                                     "action_type": "scale_out",
2379                                     "action": action,
2380                                     "prometheus_config": prom_cfg,
2381                                 }
2382                                 alerts.append(alert)
2383         return alerts
2384
2385     def _gather_vnfr_alarm_alerts(self, vnfr, vnfd):
2386         alerts = []
2387         nsr_id = vnfr["nsr-id-ref"]
2388         vnf_member_index = vnfr["member-vnf-index-ref"]
2389
2390         # Checking for VNF alarm configuration
2391         for vdur in vnfr["vdur"]:
2392             vdu_id = vdur["vdu-id-ref"]
2393             vdu = next(filter(lambda vdu: vdu["id"] == vdu_id, vnfd["vdu"]))
2394             if "alarm" in vdu:
2395                 # Get VDU monitoring params, since alerts are based on them
2396                 vdu_monitoring_params = {}
2397                 for mp in vdu.get("monitoring-parameter", []):
2398                     vdu_monitoring_params[mp.get("id")] = mp
2399                 if not vdu_monitoring_params:
2400                     self.logger.error(
2401                         "VDU alarm refers to a VDU monitoring param, but there are no VDU monitoring params in the VDU"
2402                     )
2403                     continue
2404                 # Get alarms in the VDU
2405                 alarm_descriptors = vdu["alarm"]
2406                 # Create VDU alarms for each alarm in the VDU
2407                 for alarm_descriptor in alarm_descriptors:
2408                     # Check that the VDU alarm refers to a proper monitoring param
2409                     alarm_monitoring_param = alarm_descriptor.get(
2410                         "vnf-monitoring-param-ref", ""
2411                     )
2412                     vdu_specific_monitoring_param = vdu_monitoring_params.get(
2413                         alarm_monitoring_param, {}
2414                     )
2415                     if not vdu_specific_monitoring_param:
2416                         self.logger.error(
2417                             "VDU alarm refers to a VDU monitoring param not present in the VDU"
2418                         )
2419                         continue
2420                     metric_name = vdu_specific_monitoring_param.get(
2421                         "performance-metric"
2422                     )
2423                     if not metric_name:
2424                         self.logger.error(
2425                             "VDU alarm refers to a VDU monitoring param that has no associated performance-metric"
2426                         )
2427                         continue
2428                     # Set params of the alarm to be created in Prometheus
2429                     metric_name = f"osm_{metric_name}"
2430                     metric_threshold = alarm_descriptor.get("value")
2431                     uuid = str(uuid4())
2432                     alert_name = f"vdu_alarm_{uuid}"
2433                     operation = alarm_descriptor["operation"]
2434                     rel_operator = self.rel_operation_types.get(operation, "<=")
2435                     metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
2436                     expression = f"{metric_selector} {rel_operator} {metric_threshold}"
2437                     labels = {
2438                         "ns_id": nsr_id,
2439                         "vnf_member_index": vnf_member_index,
2440                         "vdu_id": vdu_id,
2441                         "vdu_name": "{{ $labels.vdu_name }}",
2442                     }
2443                     prom_cfg = {
2444                         "alert": alert_name,
2445                         "expr": expression,
2446                         "for": "1m",  # default value. Ideally, this should be related to an IM param, but there is not such param
2447                         "labels": labels,
2448                     }
2449                     alarm_action = dict()
2450                     for action_type in ["ok", "insufficient-data", "alarm"]:
2451                         if (
2452                             "actions" in alarm_descriptor
2453                             and action_type in alarm_descriptor["actions"]
2454                         ):
2455                             alarm_action[action_type] = alarm_descriptor["actions"][
2456                                 action_type
2457                             ]
2458                     alert = {
2459                         "uuid": uuid,
2460                         "name": alert_name,
2461                         "metric": metric_name,
2462                         "tags": {
2463                             "ns_id": nsr_id,
2464                             "vnf_member_index": vnf_member_index,
2465                             "vdu_id": vdu_id,
2466                         },
2467                         "alarm_status": "ok",
2468                         "action_type": "vdu_alarm",
2469                         "action": alarm_action,
2470                         "prometheus_config": prom_cfg,
2471                     }
2472                     alerts.append(alert)
2473         return alerts
2474
2475     def update_nsrs_with_pla_result(self, params):
2476         try:
2477             nslcmop_id = deep_get(params, ("placement", "nslcmopId"))
2478             self.update_db_2(
2479                 "nslcmops", nslcmop_id, {"_admin.pla": params.get("placement")}
2480             )
2481         except Exception as e:
2482             self.logger.warn("Update failed for nslcmop_id={}:{}".format(nslcmop_id, e))
2483
2484     async def instantiate(self, nsr_id, nslcmop_id):
2485         """
2486
2487         :param nsr_id: ns instance to deploy
2488         :param nslcmop_id: operation to run
2489         :return:
2490         """
2491
2492         # Try to lock HA task here
2493         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
2494         if not task_is_locked_by_me:
2495             self.logger.debug(
2496                 "instantiate() task is not locked by me, ns={}".format(nsr_id)
2497             )
2498             return
2499
2500         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
2501         self.logger.debug(logging_text + "Enter")
2502
2503         # get all needed from database
2504
2505         # database nsrs record
2506         db_nsr = None
2507
2508         # database nslcmops record
2509         db_nslcmop = None
2510
2511         # update operation on nsrs
2512         db_nsr_update = {}
2513         # update operation on nslcmops
2514         db_nslcmop_update = {}
2515
2516         timeout_ns_deploy = self.timeout.ns_deploy
2517
2518         nslcmop_operation_state = None
2519         db_vnfrs = {}  # vnf's info indexed by member-index
2520         # n2vc_info = {}
2521         tasks_dict_info = {}  # from task to info text
2522         exc = None
2523         error_list = []
2524         stage = [
2525             "Stage 1/5: preparation of the environment.",
2526             "Waiting for previous operations to terminate.",
2527             "",
2528         ]
2529         # ^ stage, step, VIM progress
2530         try:
2531             # wait for any previous tasks in process
2532             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
2533
2534             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
2535             stage[1] = "Reading from database."
2536             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
2537             db_nsr_update["detailed-status"] = "creating"
2538             db_nsr_update["operational-status"] = "init"
2539             self._write_ns_status(
2540                 nsr_id=nsr_id,
2541                 ns_state="BUILDING",
2542                 current_operation="INSTANTIATING",
2543                 current_operation_id=nslcmop_id,
2544                 other_update=db_nsr_update,
2545             )
2546             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
2547
2548             # read from db: operation
2549             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
2550             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2551             if db_nslcmop["operationParams"].get("additionalParamsForVnf"):
2552                 db_nslcmop["operationParams"]["additionalParamsForVnf"] = json.loads(
2553                     db_nslcmop["operationParams"]["additionalParamsForVnf"]
2554                 )
2555             ns_params = db_nslcmop.get("operationParams")
2556             if ns_params and ns_params.get("timeout_ns_deploy"):
2557                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
2558
2559             # read from db: ns
2560             stage[1] = "Getting nsr={} from db.".format(nsr_id)
2561             self.logger.debug(logging_text + stage[1])
2562             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2563             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
2564             self.logger.debug(logging_text + stage[1])
2565             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2566             self.fs.sync(db_nsr["nsd-id"])
2567             db_nsr["nsd"] = nsd
2568             # nsr_name = db_nsr["name"]   # TODO short-name??
2569
2570             # read from db: vnf's of this ns
2571             stage[1] = "Getting vnfrs from db."
2572             self.logger.debug(logging_text + stage[1])
2573             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
2574
2575             # read from db: vnfd's for every vnf
2576             db_vnfds = []  # every vnfd data
2577
2578             # for each vnf in ns, read vnfd
2579             for vnfr in db_vnfrs_list:
2580                 if vnfr.get("kdur"):
2581                     kdur_list = []
2582                     for kdur in vnfr["kdur"]:
2583                         if kdur.get("additionalParams"):
2584                             kdur["additionalParams"] = json.loads(
2585                                 kdur["additionalParams"]
2586                             )
2587                         kdur_list.append(kdur)
2588                     vnfr["kdur"] = kdur_list
2589
2590                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
2591                 vnfd_id = vnfr["vnfd-id"]
2592                 vnfd_ref = vnfr["vnfd-ref"]
2593                 self.fs.sync(vnfd_id)
2594
2595                 # if we haven't this vnfd, read it from db
2596                 if vnfd_id not in db_vnfds:
2597                     # read from db
2598                     stage[1] = "Getting vnfd={} id='{}' from db.".format(
2599                         vnfd_id, vnfd_ref
2600                     )
2601                     self.logger.debug(logging_text + stage[1])
2602                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
2603
2604                     # store vnfd
2605                     db_vnfds.append(vnfd)
2606
2607             # Get or generates the _admin.deployed.VCA list
2608             vca_deployed_list = None
2609             if db_nsr["_admin"].get("deployed"):
2610                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
2611             if vca_deployed_list is None:
2612                 vca_deployed_list = []
2613                 configuration_status_list = []
2614                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2615                 db_nsr_update["configurationStatus"] = configuration_status_list
2616                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
2617                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2618             elif isinstance(vca_deployed_list, dict):
2619                 # maintain backward compatibility. Change a dict to list at database
2620                 vca_deployed_list = list(vca_deployed_list.values())
2621                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2622                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2623
2624             if not isinstance(
2625                 deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list
2626             ):
2627                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
2628                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
2629
2630             # set state to INSTANTIATED. When instantiated NBI will not delete directly
2631             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
2632             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2633             self.db.set_list(
2634                 "vnfrs", {"nsr-id-ref": nsr_id}, {"_admin.nsState": "INSTANTIATED"}
2635             )
2636
2637             # n2vc_redesign STEP 2 Deploy Network Scenario
2638             stage[0] = "Stage 2/5: deployment of KDUs, VMs and execution environments."
2639             self._write_op_status(op_id=nslcmop_id, stage=stage)
2640
2641             stage[1] = "Deploying KDUs."
2642             # self.logger.debug(logging_text + "Before deploy_kdus")
2643             # Call to deploy_kdus in case exists the "vdu:kdu" param
2644             await self.deploy_kdus(
2645                 logging_text=logging_text,
2646                 nsr_id=nsr_id,
2647                 nslcmop_id=nslcmop_id,
2648                 db_vnfrs=db_vnfrs,
2649                 db_vnfds=db_vnfds,
2650                 task_instantiation_info=tasks_dict_info,
2651             )
2652
2653             stage[1] = "Getting VCA public key."
2654             # n2vc_redesign STEP 1 Get VCA public ssh-key
2655             # feature 1429. Add n2vc public key to needed VMs
2656             n2vc_key = self.n2vc.get_public_key()
2657             n2vc_key_list = [n2vc_key]
2658             if self.vca_config.public_key:
2659                 n2vc_key_list.append(self.vca_config.public_key)
2660
2661             stage[1] = "Deploying NS at VIM."
2662             task_ro = asyncio.ensure_future(
2663                 self.instantiate_RO(
2664                     logging_text=logging_text,
2665                     nsr_id=nsr_id,
2666                     nsd=nsd,
2667                     db_nsr=db_nsr,
2668                     db_nslcmop=db_nslcmop,
2669                     db_vnfrs=db_vnfrs,
2670                     db_vnfds=db_vnfds,
2671                     n2vc_key_list=n2vc_key_list,
2672                     stage=stage,
2673                 )
2674             )
2675             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
2676             tasks_dict_info[task_ro] = "Deploying at VIM"
2677
2678             # n2vc_redesign STEP 3 to 6 Deploy N2VC
2679             stage[1] = "Deploying Execution Environments."
2680             self.logger.debug(logging_text + stage[1])
2681
2682             # create namespace and certificate if any helm based EE is present in the NS
2683             if check_helm_ee_in_ns(db_vnfds):
2684                 await self.vca_map["helm-v3"].setup_ns_namespace(
2685                     name=nsr_id,
2686                 )
2687                 # create TLS certificates
2688                 await self.vca_map["helm-v3"].create_tls_certificate(
2689                     secret_name=self.EE_TLS_NAME,
2690                     dns_prefix="*",
2691                     nsr_id=nsr_id,
2692                     usage="server auth",
2693                     namespace=nsr_id,
2694                 )
2695
2696             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
2697             for vnf_profile in get_vnf_profiles(nsd):
2698                 vnfd_id = vnf_profile["vnfd-id"]
2699                 vnfd = find_in_list(db_vnfds, lambda a_vnf: a_vnf["id"] == vnfd_id)
2700                 member_vnf_index = str(vnf_profile["id"])
2701                 db_vnfr = db_vnfrs[member_vnf_index]
2702                 base_folder = vnfd["_admin"]["storage"]
2703                 vdu_id = None
2704                 vdu_index = 0
2705                 vdu_name = None
2706                 kdu_name = None
2707                 kdu_index = None
2708
2709                 # Get additional parameters
2710                 deploy_params = {"OSM": get_osm_params(db_vnfr)}
2711                 if db_vnfr.get("additionalParamsForVnf"):
2712                     deploy_params.update(
2713                         parse_yaml_strings(db_vnfr["additionalParamsForVnf"].copy())
2714                     )
2715
2716                 descriptor_config = get_configuration(vnfd, vnfd["id"])
2717                 if descriptor_config:
2718                     self._deploy_n2vc(
2719                         logging_text=logging_text
2720                         + "member_vnf_index={} ".format(member_vnf_index),
2721                         db_nsr=db_nsr,
2722                         db_vnfr=db_vnfr,
2723                         nslcmop_id=nslcmop_id,
2724                         nsr_id=nsr_id,
2725                         nsi_id=nsi_id,
2726                         vnfd_id=vnfd_id,
2727                         vdu_id=vdu_id,
2728                         kdu_name=kdu_name,
2729                         member_vnf_index=member_vnf_index,
2730                         vdu_index=vdu_index,
2731                         kdu_index=kdu_index,
2732                         vdu_name=vdu_name,
2733                         deploy_params=deploy_params,
2734                         descriptor_config=descriptor_config,
2735                         base_folder=base_folder,
2736                         task_instantiation_info=tasks_dict_info,
2737                         stage=stage,
2738                     )
2739
2740                 # Deploy charms for each VDU that supports one.
2741                 for vdud in get_vdu_list(vnfd):
2742                     vdu_id = vdud["id"]
2743                     descriptor_config = get_configuration(vnfd, vdu_id)
2744                     vdur = find_in_list(
2745                         db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
2746                     )
2747
2748                     if vdur.get("additionalParams"):
2749                         deploy_params_vdu = parse_yaml_strings(vdur["additionalParams"])
2750                     else:
2751                         deploy_params_vdu = deploy_params
2752                     deploy_params_vdu["OSM"] = get_osm_params(
2753                         db_vnfr, vdu_id, vdu_count_index=0
2754                     )
2755                     vdud_count = get_number_of_instances(vnfd, vdu_id)
2756
2757                     self.logger.debug("VDUD > {}".format(vdud))
2758                     self.logger.debug(
2759                         "Descriptor config > {}".format(descriptor_config)
2760                     )
2761                     if descriptor_config:
2762                         vdu_name = None
2763                         kdu_name = None
2764                         kdu_index = None
2765                         for vdu_index in range(vdud_count):
2766                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
2767                             self._deploy_n2vc(
2768                                 logging_text=logging_text
2769                                 + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
2770                                     member_vnf_index, vdu_id, vdu_index
2771                                 ),
2772                                 db_nsr=db_nsr,
2773                                 db_vnfr=db_vnfr,
2774                                 nslcmop_id=nslcmop_id,
2775                                 nsr_id=nsr_id,
2776                                 nsi_id=nsi_id,
2777                                 vnfd_id=vnfd_id,
2778                                 vdu_id=vdu_id,
2779                                 kdu_name=kdu_name,
2780                                 kdu_index=kdu_index,
2781                                 member_vnf_index=member_vnf_index,
2782                                 vdu_index=vdu_index,
2783                                 vdu_name=vdu_name,
2784                                 deploy_params=deploy_params_vdu,
2785                                 descriptor_config=descriptor_config,
2786                                 base_folder=base_folder,
2787                                 task_instantiation_info=tasks_dict_info,
2788                                 stage=stage,
2789                             )
2790                 for kdud in get_kdu_list(vnfd):
2791                     kdu_name = kdud["name"]
2792                     descriptor_config = get_configuration(vnfd, kdu_name)
2793                     if descriptor_config:
2794                         vdu_id = None
2795                         vdu_index = 0
2796                         vdu_name = None
2797                         kdu_index, kdur = next(
2798                             x
2799                             for x in enumerate(db_vnfr["kdur"])
2800                             if x[1]["kdu-name"] == kdu_name
2801                         )
2802                         deploy_params_kdu = {"OSM": get_osm_params(db_vnfr)}
2803                         if kdur.get("additionalParams"):
2804                             deploy_params_kdu.update(
2805                                 parse_yaml_strings(kdur["additionalParams"].copy())
2806                             )
2807
2808                         self._deploy_n2vc(
2809                             logging_text=logging_text,
2810                             db_nsr=db_nsr,
2811                             db_vnfr=db_vnfr,
2812                             nslcmop_id=nslcmop_id,
2813                             nsr_id=nsr_id,
2814                             nsi_id=nsi_id,
2815                             vnfd_id=vnfd_id,
2816                             vdu_id=vdu_id,
2817                             kdu_name=kdu_name,
2818                             member_vnf_index=member_vnf_index,
2819                             vdu_index=vdu_index,
2820                             kdu_index=kdu_index,
2821                             vdu_name=vdu_name,
2822                             deploy_params=deploy_params_kdu,
2823                             descriptor_config=descriptor_config,
2824                             base_folder=base_folder,
2825                             task_instantiation_info=tasks_dict_info,
2826                             stage=stage,
2827                         )
2828
2829             # Check if each vnf has exporter for metric collection if so update prometheus job records
2830             if "exporters-endpoints" in vnfd.get("df")[0]:
2831                 exporter_config = vnfd.get("df")[0].get("exporters-endpoints")
2832                 self.logger.debug("exporter config :{}".format(exporter_config))
2833                 artifact_path = "{}/{}/{}".format(
2834                     base_folder["folder"],
2835                     base_folder["pkg-dir"],
2836                     "exporter-endpoint",
2837                 )
2838                 ee_id = None
2839                 ee_config_descriptor = exporter_config
2840                 vnfr_id = db_vnfr["id"]
2841                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
2842                     logging_text,
2843                     nsr_id,
2844                     vnfr_id,
2845                     vdu_id=None,
2846                     vdu_index=None,
2847                     user=None,
2848                     pub_key=None,
2849                 )
2850                 self.logger.debug("rw_mgmt_ip:{}".format(rw_mgmt_ip))
2851                 self.logger.debug("Artifact_path:{}".format(artifact_path))
2852                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
2853                 vdu_id_for_prom = None
2854                 vdu_index_for_prom = None
2855                 for x in get_iterable(db_vnfr, "vdur"):
2856                     vdu_id_for_prom = x.get("vdu-id-ref")
2857                     vdu_index_for_prom = x.get("count-index")
2858                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
2859                     ee_id=ee_id,
2860                     artifact_path=artifact_path,
2861                     ee_config_descriptor=ee_config_descriptor,
2862                     vnfr_id=vnfr_id,
2863                     nsr_id=nsr_id,
2864                     target_ip=rw_mgmt_ip,
2865                     element_type="VDU",
2866                     vdu_id=vdu_id_for_prom,
2867                     vdu_index=vdu_index_for_prom,
2868                 )
2869
2870                 self.logger.debug("Prometheus job:{}".format(prometheus_jobs))
2871                 if prometheus_jobs:
2872                     db_nsr_update["_admin.deployed.prometheus_jobs"] = prometheus_jobs
2873                     self.update_db_2(
2874                         "nsrs",
2875                         nsr_id,
2876                         db_nsr_update,
2877                     )
2878
2879                     for job in prometheus_jobs:
2880                         self.db.set_one(
2881                             "prometheus_jobs",
2882                             {"job_name": job["job_name"]},
2883                             job,
2884                             upsert=True,
2885                             fail_on_empty=False,
2886                         )
2887
2888             # Check if this NS has a charm configuration
2889             descriptor_config = nsd.get("ns-configuration")
2890             if descriptor_config and descriptor_config.get("juju"):
2891                 vnfd_id = None
2892                 db_vnfr = None
2893                 member_vnf_index = None
2894                 vdu_id = None
2895                 kdu_name = None
2896                 kdu_index = None
2897                 vdu_index = 0
2898                 vdu_name = None
2899
2900                 # Get additional parameters
2901                 deploy_params = {"OSM": {"vim_account_id": ns_params["vimAccountId"]}}
2902                 if db_nsr.get("additionalParamsForNs"):
2903                     deploy_params.update(
2904                         parse_yaml_strings(db_nsr["additionalParamsForNs"].copy())
2905                     )
2906                 base_folder = nsd["_admin"]["storage"]
2907                 self._deploy_n2vc(
2908                     logging_text=logging_text,
2909                     db_nsr=db_nsr,
2910                     db_vnfr=db_vnfr,
2911                     nslcmop_id=nslcmop_id,
2912                     nsr_id=nsr_id,
2913                     nsi_id=nsi_id,
2914                     vnfd_id=vnfd_id,
2915                     vdu_id=vdu_id,
2916                     kdu_name=kdu_name,
2917                     member_vnf_index=member_vnf_index,
2918                     vdu_index=vdu_index,
2919                     kdu_index=kdu_index,
2920                     vdu_name=vdu_name,
2921                     deploy_params=deploy_params,
2922                     descriptor_config=descriptor_config,
2923                     base_folder=base_folder,
2924                     task_instantiation_info=tasks_dict_info,
2925                     stage=stage,
2926                 )
2927
2928             # rest of staff will be done at finally
2929
2930         except (
2931             ROclient.ROClientException,
2932             DbException,
2933             LcmException,
2934             N2VCException,
2935         ) as e:
2936             self.logger.error(
2937                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e)
2938             )
2939             exc = e
2940         except asyncio.CancelledError:
2941             self.logger.error(
2942                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
2943             )
2944             exc = "Operation was cancelled"
2945         except Exception as e:
2946             exc = traceback.format_exc()
2947             self.logger.critical(
2948                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
2949                 exc_info=True,
2950             )
2951         finally:
2952             if exc:
2953                 error_list.append(str(exc))
2954             try:
2955                 # wait for pending tasks
2956                 if tasks_dict_info:
2957                     stage[1] = "Waiting for instantiate pending tasks."
2958                     self.logger.debug(logging_text + stage[1])
2959                     error_list += await self._wait_for_tasks(
2960                         logging_text,
2961                         tasks_dict_info,
2962                         timeout_ns_deploy,
2963                         stage,
2964                         nslcmop_id,
2965                         nsr_id=nsr_id,
2966                     )
2967                 stage[1] = stage[2] = ""
2968             except asyncio.CancelledError:
2969                 error_list.append("Cancelled")
2970                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
2971                 await self._wait_for_tasks(
2972                     logging_text,
2973                     tasks_dict_info,
2974                     timeout_ns_deploy,
2975                     stage,
2976                     nslcmop_id,
2977                     nsr_id=nsr_id,
2978                 )
2979             except Exception as exc:
2980                 error_list.append(str(exc))
2981
2982             # update operation-status
2983             db_nsr_update["operational-status"] = "running"
2984             # let's begin with VCA 'configured' status (later we can change it)
2985             db_nsr_update["config-status"] = "configured"
2986             for task, task_name in tasks_dict_info.items():
2987                 if not task.done() or task.cancelled() or task.exception():
2988                     if task_name.startswith(self.task_name_deploy_vca):
2989                         # A N2VC task is pending
2990                         db_nsr_update["config-status"] = "failed"
2991                     else:
2992                         # RO or KDU task is pending
2993                         db_nsr_update["operational-status"] = "failed"
2994
2995             # update status at database
2996             if error_list:
2997                 error_detail = ". ".join(error_list)
2998                 self.logger.error(logging_text + error_detail)
2999                 error_description_nslcmop = "{} Detail: {}".format(
3000                     stage[0], error_detail
3001                 )
3002                 error_description_nsr = "Operation: INSTANTIATING.{}, {}".format(
3003                     nslcmop_id, stage[0]
3004                 )
3005
3006                 db_nsr_update["detailed-status"] = (
3007                     error_description_nsr + " Detail: " + error_detail
3008                 )
3009                 db_nslcmop_update["detailed-status"] = error_detail
3010                 nslcmop_operation_state = "FAILED"
3011                 ns_state = "BROKEN"
3012             else:
3013                 error_detail = None
3014                 error_description_nsr = error_description_nslcmop = None
3015                 ns_state = "READY"
3016                 db_nsr_update["detailed-status"] = "Done"
3017                 db_nslcmop_update["detailed-status"] = "Done"
3018                 nslcmop_operation_state = "COMPLETED"
3019                 # Gather auto-healing and auto-scaling alerts for each vnfr
3020                 healing_alerts = []
3021                 scaling_alerts = []
3022                 for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
3023                     vnfd = next(
3024                         (sub for sub in db_vnfds if sub["_id"] == vnfr["vnfd-id"]), None
3025                     )
3026                     healing_alerts = self._gather_vnfr_healing_alerts(vnfr, vnfd)
3027                     for alert in healing_alerts:
3028                         self.logger.info(f"Storing healing alert in MongoDB: {alert}")
3029                         self.db.create("alerts", alert)
3030
3031                     scaling_alerts = self._gather_vnfr_scaling_alerts(vnfr, vnfd)
3032                     for alert in scaling_alerts:
3033                         self.logger.info(f"Storing scaling alert in MongoDB: {alert}")
3034                         self.db.create("alerts", alert)
3035
3036                     alarm_alerts = self._gather_vnfr_alarm_alerts(vnfr, vnfd)
3037                     for alert in alarm_alerts:
3038                         self.logger.info(f"Storing VNF alarm alert in MongoDB: {alert}")
3039                         self.db.create("alerts", alert)
3040             if db_nsr:
3041                 self._write_ns_status(
3042                     nsr_id=nsr_id,
3043                     ns_state=ns_state,
3044                     current_operation="IDLE",
3045                     current_operation_id=None,
3046                     error_description=error_description_nsr,
3047                     error_detail=error_detail,
3048                     other_update=db_nsr_update,
3049                 )
3050             self._write_op_status(
3051                 op_id=nslcmop_id,
3052                 stage="",
3053                 error_message=error_description_nslcmop,
3054                 operation_state=nslcmop_operation_state,
3055                 other_update=db_nslcmop_update,
3056             )
3057
3058             if nslcmop_operation_state:
3059                 try:
3060                     await self.msg.aiowrite(
3061                         "ns",
3062                         "instantiated",
3063                         {
3064                             "nsr_id": nsr_id,
3065                             "nslcmop_id": nslcmop_id,
3066                             "operationState": nslcmop_operation_state,
3067                             "startTime": db_nslcmop["startTime"],
3068                             "links": db_nslcmop["links"],
3069                             "operationParams": {
3070                                 "nsInstanceId": nsr_id,
3071                                 "nsdId": db_nsr["nsd-id"],
3072                             },
3073                         },
3074                     )
3075                 except Exception as e:
3076                     self.logger.error(
3077                         logging_text + "kafka_write notification Exception {}".format(e)
3078                     )
3079
3080             self.logger.debug(logging_text + "Exit")
3081             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
3082
3083     def _get_vnfd(self, vnfd_id: str, projects_read: str, cached_vnfds: Dict[str, Any]):
3084         if vnfd_id not in cached_vnfds:
3085             cached_vnfds[vnfd_id] = self.db.get_one(
3086                 "vnfds", {"id": vnfd_id, "_admin.projects_read": projects_read}
3087             )
3088         return cached_vnfds[vnfd_id]
3089
3090     def _get_vnfr(self, nsr_id: str, vnf_profile_id: str, cached_vnfrs: Dict[str, Any]):
3091         if vnf_profile_id not in cached_vnfrs:
3092             cached_vnfrs[vnf_profile_id] = self.db.get_one(
3093                 "vnfrs",
3094                 {
3095                     "member-vnf-index-ref": vnf_profile_id,
3096                     "nsr-id-ref": nsr_id,
3097                 },
3098             )
3099         return cached_vnfrs[vnf_profile_id]
3100
3101     def _is_deployed_vca_in_relation(
3102         self, vca: DeployedVCA, relation: Relation
3103     ) -> bool:
3104         found = False
3105         for endpoint in (relation.provider, relation.requirer):
3106             if endpoint["kdu-resource-profile-id"]:
3107                 continue
3108             found = (
3109                 vca.vnf_profile_id == endpoint.vnf_profile_id
3110                 and vca.vdu_profile_id == endpoint.vdu_profile_id
3111                 and vca.execution_environment_ref == endpoint.execution_environment_ref
3112             )
3113             if found:
3114                 break
3115         return found
3116
3117     def _update_ee_relation_data_with_implicit_data(
3118         self, nsr_id, nsd, ee_relation_data, cached_vnfds, vnf_profile_id: str = None
3119     ):
3120         ee_relation_data = safe_get_ee_relation(
3121             nsr_id, ee_relation_data, vnf_profile_id=vnf_profile_id
3122         )
3123         ee_relation_level = EELevel.get_level(ee_relation_data)
3124         if (ee_relation_level in (EELevel.VNF, EELevel.VDU)) and not ee_relation_data[
3125             "execution-environment-ref"
3126         ]:
3127             vnf_profile = get_vnf_profile(nsd, ee_relation_data["vnf-profile-id"])
3128             vnfd_id = vnf_profile["vnfd-id"]
3129             project = nsd["_admin"]["projects_read"][0]
3130             db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3131             entity_id = (
3132                 vnfd_id
3133                 if ee_relation_level == EELevel.VNF
3134                 else ee_relation_data["vdu-profile-id"]
3135             )
3136             ee = get_juju_ee_ref(db_vnfd, entity_id)
3137             if not ee:
3138                 raise Exception(
3139                     f"not execution environments found for ee_relation {ee_relation_data}"
3140                 )
3141             ee_relation_data["execution-environment-ref"] = ee["id"]
3142         return ee_relation_data
3143
3144     def _get_ns_relations(
3145         self,
3146         nsr_id: str,
3147         nsd: Dict[str, Any],
3148         vca: DeployedVCA,
3149         cached_vnfds: Dict[str, Any],
3150     ) -> List[Relation]:
3151         relations = []
3152         db_ns_relations = get_ns_configuration_relation_list(nsd)
3153         for r in db_ns_relations:
3154             provider_dict = None
3155             requirer_dict = None
3156             if all(key in r for key in ("provider", "requirer")):
3157                 provider_dict = r["provider"]
3158                 requirer_dict = r["requirer"]
3159             elif "entities" in r:
3160                 provider_id = r["entities"][0]["id"]
3161                 provider_dict = {
3162                     "nsr-id": nsr_id,
3163                     "endpoint": r["entities"][0]["endpoint"],
3164                 }
3165                 if provider_id != nsd["id"]:
3166                     provider_dict["vnf-profile-id"] = provider_id
3167                 requirer_id = r["entities"][1]["id"]
3168                 requirer_dict = {
3169                     "nsr-id": nsr_id,
3170                     "endpoint": r["entities"][1]["endpoint"],
3171                 }
3172                 if requirer_id != nsd["id"]:
3173                     requirer_dict["vnf-profile-id"] = requirer_id
3174             else:
3175                 raise Exception(
3176                     "provider/requirer or entities must be included in the relation."
3177                 )
3178             relation_provider = self._update_ee_relation_data_with_implicit_data(
3179                 nsr_id, nsd, provider_dict, cached_vnfds
3180             )
3181             relation_requirer = self._update_ee_relation_data_with_implicit_data(
3182                 nsr_id, nsd, requirer_dict, cached_vnfds
3183             )
3184             provider = EERelation(relation_provider)
3185             requirer = EERelation(relation_requirer)
3186             relation = Relation(r["name"], provider, requirer)
3187             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
3188             if vca_in_relation:
3189                 relations.append(relation)
3190         return relations
3191
3192     def _get_vnf_relations(
3193         self,
3194         nsr_id: str,
3195         nsd: Dict[str, Any],
3196         vca: DeployedVCA,
3197         cached_vnfds: Dict[str, Any],
3198     ) -> List[Relation]:
3199         relations = []
3200         if vca.target_element == "ns":
3201             self.logger.debug("VCA is a NS charm, not a VNF.")
3202             return relations
3203         vnf_profile = get_vnf_profile(nsd, vca.vnf_profile_id)
3204         vnf_profile_id = vnf_profile["id"]
3205         vnfd_id = vnf_profile["vnfd-id"]
3206         project = nsd["_admin"]["projects_read"][0]
3207         db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3208         db_vnf_relations = get_relation_list(db_vnfd, vnfd_id)
3209         for r in db_vnf_relations:
3210             provider_dict = None
3211             requirer_dict = None
3212             if all(key in r for key in ("provider", "requirer")):
3213                 provider_dict = r["provider"]
3214                 requirer_dict = r["requirer"]
3215             elif "entities" in r:
3216                 provider_id = r["entities"][0]["id"]
3217                 provider_dict = {
3218                     "nsr-id": nsr_id,
3219                     "vnf-profile-id": vnf_profile_id,
3220                     "endpoint": r["entities"][0]["endpoint"],
3221                 }
3222                 if provider_id != vnfd_id:
3223                     provider_dict["vdu-profile-id"] = provider_id
3224                 requirer_id = r["entities"][1]["id"]
3225                 requirer_dict = {
3226                     "nsr-id": nsr_id,
3227                     "vnf-profile-id": vnf_profile_id,
3228                     "endpoint": r["entities"][1]["endpoint"],
3229                 }
3230                 if requirer_id != vnfd_id:
3231                     requirer_dict["vdu-profile-id"] = requirer_id
3232             else:
3233                 raise Exception(
3234                     "provider/requirer or entities must be included in the relation."
3235                 )
3236             relation_provider = self._update_ee_relation_data_with_implicit_data(
3237                 nsr_id, nsd, provider_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
3238             )
3239             relation_requirer = self._update_ee_relation_data_with_implicit_data(
3240                 nsr_id, nsd, requirer_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
3241             )
3242             provider = EERelation(relation_provider)
3243             requirer = EERelation(relation_requirer)
3244             relation = Relation(r["name"], provider, requirer)
3245             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
3246             if vca_in_relation:
3247                 relations.append(relation)
3248         return relations
3249
3250     def _get_kdu_resource_data(
3251         self,
3252         ee_relation: EERelation,
3253         db_nsr: Dict[str, Any],
3254         cached_vnfds: Dict[str, Any],
3255     ) -> DeployedK8sResource:
3256         nsd = get_nsd(db_nsr)
3257         vnf_profiles = get_vnf_profiles(nsd)
3258         vnfd_id = find_in_list(
3259             vnf_profiles,
3260             lambda vnf_profile: vnf_profile["id"] == ee_relation.vnf_profile_id,
3261         )["vnfd-id"]
3262         project = nsd["_admin"]["projects_read"][0]
3263         db_vnfd = self._get_vnfd(vnfd_id, project, cached_vnfds)
3264         kdu_resource_profile = get_kdu_resource_profile(
3265             db_vnfd, ee_relation.kdu_resource_profile_id
3266         )
3267         kdu_name = kdu_resource_profile["kdu-name"]
3268         deployed_kdu, _ = get_deployed_kdu(
3269             db_nsr.get("_admin", ()).get("deployed", ()),
3270             kdu_name,
3271             ee_relation.vnf_profile_id,
3272         )
3273         deployed_kdu.update({"resource-name": kdu_resource_profile["resource-name"]})
3274         return deployed_kdu
3275
3276     def _get_deployed_component(
3277         self,
3278         ee_relation: EERelation,
3279         db_nsr: Dict[str, Any],
3280         cached_vnfds: Dict[str, Any],
3281     ) -> DeployedComponent:
3282         nsr_id = db_nsr["_id"]
3283         deployed_component = None
3284         ee_level = EELevel.get_level(ee_relation)
3285         if ee_level == EELevel.NS:
3286             vca = get_deployed_vca(db_nsr, {"vdu_id": None, "member-vnf-index": None})
3287             if vca:
3288                 deployed_component = DeployedVCA(nsr_id, vca)
3289         elif ee_level == EELevel.VNF:
3290             vca = get_deployed_vca(
3291                 db_nsr,
3292                 {
3293                     "vdu_id": None,
3294                     "member-vnf-index": ee_relation.vnf_profile_id,
3295                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3296                 },
3297             )
3298             if vca:
3299                 deployed_component = DeployedVCA(nsr_id, vca)
3300         elif ee_level == EELevel.VDU:
3301             vca = get_deployed_vca(
3302                 db_nsr,
3303                 {
3304                     "vdu_id": ee_relation.vdu_profile_id,
3305                     "member-vnf-index": ee_relation.vnf_profile_id,
3306                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3307                 },
3308             )
3309             if vca:
3310                 deployed_component = DeployedVCA(nsr_id, vca)
3311         elif ee_level == EELevel.KDU:
3312             kdu_resource_data = self._get_kdu_resource_data(
3313                 ee_relation, db_nsr, cached_vnfds
3314             )
3315             if kdu_resource_data:
3316                 deployed_component = DeployedK8sResource(kdu_resource_data)
3317         return deployed_component
3318
3319     async def _add_relation(
3320         self,
3321         relation: Relation,
3322         vca_type: str,
3323         db_nsr: Dict[str, Any],
3324         cached_vnfds: Dict[str, Any],
3325         cached_vnfrs: Dict[str, Any],
3326     ) -> bool:
3327         deployed_provider = self._get_deployed_component(
3328             relation.provider, db_nsr, cached_vnfds
3329         )
3330         deployed_requirer = self._get_deployed_component(
3331             relation.requirer, db_nsr, cached_vnfds
3332         )
3333         if (
3334             deployed_provider
3335             and deployed_requirer
3336             and deployed_provider.config_sw_installed
3337             and deployed_requirer.config_sw_installed
3338         ):
3339             provider_db_vnfr = (
3340                 self._get_vnfr(
3341                     relation.provider.nsr_id,
3342                     relation.provider.vnf_profile_id,
3343                     cached_vnfrs,
3344                 )
3345                 if relation.provider.vnf_profile_id
3346                 else None
3347             )
3348             requirer_db_vnfr = (
3349                 self._get_vnfr(
3350                     relation.requirer.nsr_id,
3351                     relation.requirer.vnf_profile_id,
3352                     cached_vnfrs,
3353                 )
3354                 if relation.requirer.vnf_profile_id
3355                 else None
3356             )
3357             provider_vca_id = self.get_vca_id(provider_db_vnfr, db_nsr)
3358             requirer_vca_id = self.get_vca_id(requirer_db_vnfr, db_nsr)
3359             provider_relation_endpoint = RelationEndpoint(
3360                 deployed_provider.ee_id,
3361                 provider_vca_id,
3362                 relation.provider.endpoint,
3363             )
3364             requirer_relation_endpoint = RelationEndpoint(
3365                 deployed_requirer.ee_id,
3366                 requirer_vca_id,
3367                 relation.requirer.endpoint,
3368             )
3369             try:
3370                 await self.vca_map[vca_type].add_relation(
3371                     provider=provider_relation_endpoint,
3372                     requirer=requirer_relation_endpoint,
3373                 )
3374             except N2VCException as exception:
3375                 self.logger.error(exception)
3376                 raise LcmException(exception)
3377             return True
3378         return False
3379
3380     async def _add_vca_relations(
3381         self,
3382         logging_text,
3383         nsr_id,
3384         vca_type: str,
3385         vca_index: int,
3386         timeout: int = 3600,
3387     ) -> bool:
3388         # steps:
3389         # 1. find all relations for this VCA
3390         # 2. wait for other peers related
3391         # 3. add relations
3392
3393         try:
3394             # STEP 1: find all relations for this VCA
3395
3396             # read nsr record
3397             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3398             nsd = get_nsd(db_nsr)
3399
3400             # this VCA data
3401             deployed_vca_dict = get_deployed_vca_list(db_nsr)[vca_index]
3402             my_vca = DeployedVCA(nsr_id, deployed_vca_dict)
3403
3404             cached_vnfds = {}
3405             cached_vnfrs = {}
3406             relations = []
3407             relations.extend(self._get_ns_relations(nsr_id, nsd, my_vca, cached_vnfds))
3408             relations.extend(self._get_vnf_relations(nsr_id, nsd, my_vca, cached_vnfds))
3409
3410             # if no relations, terminate
3411             if not relations:
3412                 self.logger.debug(logging_text + " No relations")
3413                 return True
3414
3415             self.logger.debug(logging_text + " adding relations {}".format(relations))
3416
3417             # add all relations
3418             start = time()
3419             while True:
3420                 # check timeout
3421                 now = time()
3422                 if now - start >= timeout:
3423                     self.logger.error(logging_text + " : timeout adding relations")
3424                     return False
3425
3426                 # reload nsr from database (we need to update record: _admin.deployed.VCA)
3427                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3428
3429                 # for each relation, find the VCA's related
3430                 for relation in relations.copy():
3431                     added = await self._add_relation(
3432                         relation,
3433                         vca_type,
3434                         db_nsr,
3435                         cached_vnfds,
3436                         cached_vnfrs,
3437                     )
3438                     if added:
3439                         relations.remove(relation)
3440
3441                 if not relations:
3442                     self.logger.debug("Relations added")
3443                     break
3444                 await asyncio.sleep(5.0)
3445
3446             return True
3447
3448         except Exception as e:
3449             self.logger.warn(logging_text + " ERROR adding relations: {}".format(e))
3450             return False
3451
3452     async def _install_kdu(
3453         self,
3454         nsr_id: str,
3455         nsr_db_path: str,
3456         vnfr_data: dict,
3457         kdu_index: int,
3458         kdud: dict,
3459         vnfd: dict,
3460         k8s_instance_info: dict,
3461         k8params: dict = None,
3462         timeout: int = 600,
3463         vca_id: str = None,
3464     ):
3465         try:
3466             k8sclustertype = k8s_instance_info["k8scluster-type"]
3467             # Instantiate kdu
3468             db_dict_install = {
3469                 "collection": "nsrs",
3470                 "filter": {"_id": nsr_id},
3471                 "path": nsr_db_path,
3472             }
3473
3474             if k8s_instance_info.get("kdu-deployment-name"):
3475                 kdu_instance = k8s_instance_info.get("kdu-deployment-name")
3476             else:
3477                 kdu_instance = self.k8scluster_map[
3478                     k8sclustertype
3479                 ].generate_kdu_instance_name(
3480                     db_dict=db_dict_install,
3481                     kdu_model=k8s_instance_info["kdu-model"],
3482                     kdu_name=k8s_instance_info["kdu-name"],
3483                 )
3484
3485             # Update the nsrs table with the kdu-instance value
3486             self.update_db_2(
3487                 item="nsrs",
3488                 _id=nsr_id,
3489                 _desc={nsr_db_path + ".kdu-instance": kdu_instance},
3490             )
3491
3492             # Update the nsrs table with the actual namespace being used, if the k8scluster-type is `juju` or
3493             # `juju-bundle`. This verification is needed because there is not a standard/homogeneous namespace
3494             # between the Helm Charts and Juju Bundles-based KNFs. If we found a way of having an homogeneous
3495             # namespace, this first verification could be removed, and the next step would be done for any kind
3496             # of KNF.
3497             # TODO -> find a way to have an homogeneous namespace between the Helm Charts and Juju Bundles-based
3498             # KNFs (Bug 2027: https://osm.etsi.org/bugzilla/show_bug.cgi?id=2027)
3499             if k8sclustertype in ("juju", "juju-bundle"):
3500                 # First, verify if the current namespace is present in the `_admin.projects_read` (if not, it means
3501                 # that the user passed a namespace which he wants its KDU to be deployed in)
3502                 if (
3503                     self.db.count(
3504                         table="nsrs",
3505                         q_filter={
3506                             "_id": nsr_id,
3507                             "_admin.projects_write": k8s_instance_info["namespace"],
3508                             "_admin.projects_read": k8s_instance_info["namespace"],
3509                         },
3510                     )
3511                     > 0
3512                 ):
3513                     self.logger.debug(
3514                         f"Updating namespace/model for Juju Bundle from {k8s_instance_info['namespace']} to {kdu_instance}"
3515                     )
3516                     self.update_db_2(
3517                         item="nsrs",
3518                         _id=nsr_id,
3519                         _desc={f"{nsr_db_path}.namespace": kdu_instance},
3520                     )
3521                     k8s_instance_info["namespace"] = kdu_instance
3522
3523             await self.k8scluster_map[k8sclustertype].install(
3524                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3525                 kdu_model=k8s_instance_info["kdu-model"],
3526                 atomic=True,
3527                 params=k8params,
3528                 db_dict=db_dict_install,
3529                 timeout=timeout,
3530                 kdu_name=k8s_instance_info["kdu-name"],
3531                 namespace=k8s_instance_info["namespace"],
3532                 kdu_instance=kdu_instance,
3533                 vca_id=vca_id,
3534             )
3535
3536             # Obtain services to obtain management service ip
3537             services = await self.k8scluster_map[k8sclustertype].get_services(
3538                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3539                 kdu_instance=kdu_instance,
3540                 namespace=k8s_instance_info["namespace"],
3541             )
3542
3543             # Obtain management service info (if exists)
3544             vnfr_update_dict = {}
3545             kdu_config = get_configuration(vnfd, kdud["name"])
3546             if kdu_config:
3547                 target_ee_list = kdu_config.get("execution-environment-list", [])
3548             else:
3549                 target_ee_list = []
3550
3551             if services:
3552                 vnfr_update_dict["kdur.{}.services".format(kdu_index)] = services
3553                 mgmt_services = [
3554                     service
3555                     for service in kdud.get("service", [])
3556                     if service.get("mgmt-service")
3557                 ]
3558                 for mgmt_service in mgmt_services:
3559                     for service in services:
3560                         if service["name"].startswith(mgmt_service["name"]):
3561                             # Mgmt service found, Obtain service ip
3562                             ip = service.get("external_ip", service.get("cluster_ip"))
3563                             if isinstance(ip, list) and len(ip) == 1:
3564                                 ip = ip[0]
3565
3566                             vnfr_update_dict[
3567                                 "kdur.{}.ip-address".format(kdu_index)
3568                             ] = ip
3569
3570                             # Check if must update also mgmt ip at the vnf
3571                             service_external_cp = mgmt_service.get(
3572                                 "external-connection-point-ref"
3573                             )
3574                             if service_external_cp:
3575                                 if (
3576                                     deep_get(vnfd, ("mgmt-interface", "cp"))
3577                                     == service_external_cp
3578                                 ):
3579                                     vnfr_update_dict["ip-address"] = ip
3580
3581                                 if find_in_list(
3582                                     target_ee_list,
3583                                     lambda ee: ee.get(
3584                                         "external-connection-point-ref", ""
3585                                     )
3586                                     == service_external_cp,
3587                                 ):
3588                                     vnfr_update_dict[
3589                                         "kdur.{}.ip-address".format(kdu_index)
3590                                     ] = ip
3591                             break
3592                     else:
3593                         self.logger.warn(
3594                             "Mgmt service name: {} not found".format(
3595                                 mgmt_service["name"]
3596                             )
3597                         )
3598
3599             vnfr_update_dict["kdur.{}.status".format(kdu_index)] = "READY"
3600             self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
3601
3602             kdu_config = get_configuration(vnfd, k8s_instance_info["kdu-name"])
3603             if (
3604                 kdu_config
3605                 and kdu_config.get("initial-config-primitive")
3606                 and get_juju_ee_ref(vnfd, k8s_instance_info["kdu-name"]) is None
3607             ):
3608                 initial_config_primitive_list = kdu_config.get(
3609                     "initial-config-primitive"
3610                 )
3611                 initial_config_primitive_list.sort(key=lambda val: int(val["seq"]))
3612
3613                 for initial_config_primitive in initial_config_primitive_list:
3614                     primitive_params_ = self._map_primitive_params(
3615                         initial_config_primitive, {}, {}
3616                     )
3617
3618                     await asyncio.wait_for(
3619                         self.k8scluster_map[k8sclustertype].exec_primitive(
3620                             cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3621                             kdu_instance=kdu_instance,
3622                             primitive_name=initial_config_primitive["name"],
3623                             params=primitive_params_,
3624                             db_dict=db_dict_install,
3625                             vca_id=vca_id,
3626                         ),
3627                         timeout=timeout,
3628                     )
3629
3630         except Exception as e:
3631             # Prepare update db with error and raise exception
3632             try:
3633                 self.update_db_2(
3634                     "nsrs", nsr_id, {nsr_db_path + ".detailed-status": str(e)}
3635                 )
3636                 self.update_db_2(
3637                     "vnfrs",
3638                     vnfr_data.get("_id"),
3639                     {"kdur.{}.status".format(kdu_index): "ERROR"},
3640                 )
3641             except Exception as error:
3642                 # ignore to keep original exception
3643                 self.logger.warning(
3644                     f"An exception occurred while updating DB: {str(error)}"
3645                 )
3646             # reraise original error
3647             raise
3648
3649         return kdu_instance
3650
3651     async def deploy_kdus(
3652         self,
3653         logging_text,
3654         nsr_id,
3655         nslcmop_id,
3656         db_vnfrs,
3657         db_vnfds,
3658         task_instantiation_info,
3659     ):
3660         # Launch kdus if present in the descriptor
3661
3662         k8scluster_id_2_uuic = {
3663             "helm-chart-v3": {},
3664             "juju-bundle": {},
3665         }
3666
3667         async def _get_cluster_id(cluster_id, cluster_type):
3668             nonlocal k8scluster_id_2_uuic
3669             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
3670                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
3671
3672             # check if K8scluster is creating and wait look if previous tasks in process
3673             task_name, task_dependency = self.lcm_tasks.lookfor_related(
3674                 "k8scluster", cluster_id
3675             )
3676             if task_dependency:
3677                 text = "Waiting for related tasks '{}' on k8scluster {} to be completed".format(
3678                     task_name, cluster_id
3679                 )
3680                 self.logger.debug(logging_text + text)
3681                 await asyncio.wait(task_dependency, timeout=3600)
3682
3683             db_k8scluster = self.db.get_one(
3684                 "k8sclusters", {"_id": cluster_id}, fail_on_empty=False
3685             )
3686             if not db_k8scluster:
3687                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
3688
3689             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
3690             if not k8s_id:
3691                 if cluster_type == "helm-chart-v3":
3692                     try:
3693                         # backward compatibility for existing clusters that have not been initialized for helm v3
3694                         k8s_credentials = yaml.safe_dump(
3695                             db_k8scluster.get("credentials")
3696                         )
3697                         k8s_id, uninstall_sw = await self.k8sclusterhelm3.init_env(
3698                             k8s_credentials, reuse_cluster_uuid=cluster_id
3699                         )
3700                         db_k8scluster_update = {}
3701                         db_k8scluster_update["_admin.helm-chart-v3.error_msg"] = None
3702                         db_k8scluster_update["_admin.helm-chart-v3.id"] = k8s_id
3703                         db_k8scluster_update[
3704                             "_admin.helm-chart-v3.created"
3705                         ] = uninstall_sw
3706                         db_k8scluster_update[
3707                             "_admin.helm-chart-v3.operationalState"
3708                         ] = "ENABLED"
3709                         self.update_db_2(
3710                             "k8sclusters", cluster_id, db_k8scluster_update
3711                         )
3712                     except Exception as e:
3713                         self.logger.error(
3714                             logging_text
3715                             + "error initializing helm-v3 cluster: {}".format(str(e))
3716                         )
3717                         raise LcmException(
3718                             "K8s cluster '{}' has not been initialized for '{}'".format(
3719                                 cluster_id, cluster_type
3720                             )
3721                         )
3722                 else:
3723                     raise LcmException(
3724                         "K8s cluster '{}' has not been initialized for '{}'".format(
3725                             cluster_id, cluster_type
3726                         )
3727                     )
3728             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
3729             return k8s_id
3730
3731         logging_text += "Deploy kdus: "
3732         step = ""
3733         try:
3734             db_nsr_update = {"_admin.deployed.K8s": []}
3735             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3736
3737             index = 0
3738             updated_cluster_list = []
3739             updated_v3_cluster_list = []
3740
3741             for vnfr_data in db_vnfrs.values():
3742                 vca_id = self.get_vca_id(vnfr_data, {})
3743                 for kdu_index, kdur in enumerate(get_iterable(vnfr_data, "kdur")):
3744                     # Step 0: Prepare and set parameters
3745                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
3746                     vnfd_id = vnfr_data.get("vnfd-id")
3747                     vnfd_with_id = find_in_list(
3748                         db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3749                     )
3750                     kdud = next(
3751                         kdud
3752                         for kdud in vnfd_with_id["kdu"]
3753                         if kdud["name"] == kdur["kdu-name"]
3754                     )
3755                     namespace = kdur.get("k8s-namespace")
3756                     kdu_deployment_name = kdur.get("kdu-deployment-name")
3757                     if kdur.get("helm-chart"):
3758                         kdumodel = kdur["helm-chart"]
3759                         # Default version: helm3, if helm-version is v2 assign v2
3760                         k8sclustertype = "helm-chart-v3"
3761                         self.logger.debug("kdur: {}".format(kdur))
3762                     elif kdur.get("juju-bundle"):
3763                         kdumodel = kdur["juju-bundle"]
3764                         k8sclustertype = "juju-bundle"
3765                     else:
3766                         raise LcmException(
3767                             "kdu type for kdu='{}.{}' is neither helm-chart nor "
3768                             "juju-bundle. Maybe an old NBI version is running".format(
3769                                 vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]
3770                             )
3771                         )
3772                     # check if kdumodel is a file and exists
3773                     try:
3774                         vnfd_with_id = find_in_list(
3775                             db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3776                         )
3777                         storage = deep_get(vnfd_with_id, ("_admin", "storage"))
3778                         if storage:  # may be not present if vnfd has not artifacts
3779                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
3780                             if storage["pkg-dir"]:
3781                                 filename = "{}/{}/{}s/{}".format(
3782                                     storage["folder"],
3783                                     storage["pkg-dir"],
3784                                     k8sclustertype,
3785                                     kdumodel,
3786                                 )
3787                             else:
3788                                 filename = "{}/Scripts/{}s/{}".format(
3789                                     storage["folder"],
3790                                     k8sclustertype,
3791                                     kdumodel,
3792                                 )
3793                             if self.fs.file_exists(
3794                                 filename, mode="file"
3795                             ) or self.fs.file_exists(filename, mode="dir"):
3796                                 kdumodel = self.fs.path + filename
3797                     except (asyncio.TimeoutError, asyncio.CancelledError):
3798                         raise
3799                     except Exception as e:  # it is not a file
3800                         self.logger.warning(f"An exception occurred: {str(e)}")
3801
3802                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
3803                     step = "Synchronize repos for k8s cluster '{}'".format(
3804                         k8s_cluster_id
3805                     )
3806                     cluster_uuid = await _get_cluster_id(k8s_cluster_id, k8sclustertype)
3807
3808                     # Synchronize  repos
3809                     if (
3810                         k8sclustertype == "helm-chart"
3811                         and cluster_uuid not in updated_cluster_list
3812                     ) or (
3813                         k8sclustertype == "helm-chart-v3"
3814                         and cluster_uuid not in updated_v3_cluster_list
3815                     ):
3816                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
3817                             self.k8scluster_map[k8sclustertype].synchronize_repos(
3818                                 cluster_uuid=cluster_uuid
3819                             )
3820                         )
3821                         if del_repo_list or added_repo_dict:
3822                             if k8sclustertype == "helm-chart":
3823                                 unset = {
3824                                     "_admin.helm_charts_added." + item: None
3825                                     for item in del_repo_list
3826                                 }
3827                                 updated = {
3828                                     "_admin.helm_charts_added." + item: name
3829                                     for item, name in added_repo_dict.items()
3830                                 }
3831                                 updated_cluster_list.append(cluster_uuid)
3832                             elif k8sclustertype == "helm-chart-v3":
3833                                 unset = {
3834                                     "_admin.helm_charts_v3_added." + item: None
3835                                     for item in del_repo_list
3836                                 }
3837                                 updated = {
3838                                     "_admin.helm_charts_v3_added." + item: name
3839                                     for item, name in added_repo_dict.items()
3840                                 }
3841                                 updated_v3_cluster_list.append(cluster_uuid)
3842                             self.logger.debug(
3843                                 logging_text + "repos synchronized on k8s cluster "
3844                                 "'{}' to_delete: {}, to_add: {}".format(
3845                                     k8s_cluster_id, del_repo_list, added_repo_dict
3846                                 )
3847                             )
3848                             self.db.set_one(
3849                                 "k8sclusters",
3850                                 {"_id": k8s_cluster_id},
3851                                 updated,
3852                                 unset=unset,
3853                             )
3854
3855                     # Instantiate kdu
3856                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(
3857                         vnfr_data["member-vnf-index-ref"],
3858                         kdur["kdu-name"],
3859                         k8s_cluster_id,
3860                     )
3861                     k8s_instance_info = {
3862                         "kdu-instance": None,
3863                         "k8scluster-uuid": cluster_uuid,
3864                         "k8scluster-type": k8sclustertype,
3865                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
3866                         "kdu-name": kdur["kdu-name"],
3867                         "kdu-model": kdumodel,
3868                         "namespace": namespace,
3869                         "kdu-deployment-name": kdu_deployment_name,
3870                     }
3871                     db_path = "_admin.deployed.K8s.{}".format(index)
3872                     db_nsr_update[db_path] = k8s_instance_info
3873                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
3874                     vnfd_with_id = find_in_list(
3875                         db_vnfds, lambda vnf: vnf["_id"] == vnfd_id
3876                     )
3877                     task = asyncio.ensure_future(
3878                         self._install_kdu(
3879                             nsr_id,
3880                             db_path,
3881                             vnfr_data,
3882                             kdu_index,
3883                             kdud,
3884                             vnfd_with_id,
3885                             k8s_instance_info,
3886                             k8params=desc_params,
3887                             timeout=1800,
3888                             vca_id=vca_id,
3889                         )
3890                     )
3891                     self.lcm_tasks.register(
3892                         "ns",
3893                         nsr_id,
3894                         nslcmop_id,
3895                         "instantiate_KDU-{}".format(index),
3896                         task,
3897                     )
3898                     task_instantiation_info[task] = "Deploying KDU {}".format(
3899                         kdur["kdu-name"]
3900                     )
3901
3902                     index += 1
3903
3904         except (LcmException, asyncio.CancelledError):
3905             raise
3906         except Exception as e:
3907             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
3908             if isinstance(e, (N2VCException, DbException)):
3909                 self.logger.error(logging_text + msg)
3910             else:
3911                 self.logger.critical(logging_text + msg, exc_info=True)
3912             raise LcmException(msg)
3913         finally:
3914             if db_nsr_update:
3915                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3916
3917     def _deploy_n2vc(
3918         self,
3919         logging_text,
3920         db_nsr,
3921         db_vnfr,
3922         nslcmop_id,
3923         nsr_id,
3924         nsi_id,
3925         vnfd_id,
3926         vdu_id,
3927         kdu_name,
3928         member_vnf_index,
3929         vdu_index,
3930         kdu_index,
3931         vdu_name,
3932         deploy_params,
3933         descriptor_config,
3934         base_folder,
3935         task_instantiation_info,
3936         stage,
3937     ):
3938         # launch instantiate_N2VC in a asyncio task and register task object
3939         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
3940         # if not found, create one entry and update database
3941         # fill db_nsr._admin.deployed.VCA.<index>
3942
3943         self.logger.debug(
3944             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
3945         )
3946
3947         charm_name = ""
3948         get_charm_name = False
3949         if "execution-environment-list" in descriptor_config:
3950             ee_list = descriptor_config.get("execution-environment-list", [])
3951         elif "juju" in descriptor_config:
3952             ee_list = [descriptor_config]  # ns charms
3953             if "execution-environment-list" not in descriptor_config:
3954                 # charm name is only required for ns charms
3955                 get_charm_name = True
3956         else:  # other types as script are not supported
3957             ee_list = []
3958
3959         for ee_item in ee_list:
3960             self.logger.debug(
3961                 logging_text
3962                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
3963                     ee_item.get("juju"), ee_item.get("helm-chart")
3964                 )
3965             )
3966             ee_descriptor_id = ee_item.get("id")
3967             vca_name, charm_name, vca_type = self.get_vca_info(
3968                 ee_item, db_nsr, get_charm_name
3969             )
3970             if not vca_type:
3971                 self.logger.debug(
3972                     logging_text + "skipping, non juju/charm/helm configuration"
3973                 )
3974                 continue
3975
3976             vca_index = -1
3977             for vca_index, vca_deployed in enumerate(
3978                 db_nsr["_admin"]["deployed"]["VCA"]
3979             ):
3980                 if not vca_deployed:
3981                     continue
3982                 if (
3983                     vca_deployed.get("member-vnf-index") == member_vnf_index
3984                     and vca_deployed.get("vdu_id") == vdu_id
3985                     and vca_deployed.get("kdu_name") == kdu_name
3986                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
3987                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
3988                 ):
3989                     break
3990             else:
3991                 # not found, create one.
3992                 target = (
3993                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
3994                 )
3995                 if vdu_id:
3996                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
3997                 elif kdu_name:
3998                     target += "/kdu/{}".format(kdu_name)
3999                 vca_deployed = {
4000                     "target_element": target,
4001                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
4002                     "member-vnf-index": member_vnf_index,
4003                     "vdu_id": vdu_id,
4004                     "kdu_name": kdu_name,
4005                     "vdu_count_index": vdu_index,
4006                     "operational-status": "init",  # TODO revise
4007                     "detailed-status": "",  # TODO revise
4008                     "step": "initial-deploy",  # TODO revise
4009                     "vnfd_id": vnfd_id,
4010                     "vdu_name": vdu_name,
4011                     "type": vca_type,
4012                     "ee_descriptor_id": ee_descriptor_id,
4013                     "charm_name": charm_name,
4014                 }
4015                 vca_index += 1
4016
4017                 # create VCA and configurationStatus in db
4018                 db_dict = {
4019                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
4020                     "configurationStatus.{}".format(vca_index): dict(),
4021                 }
4022                 self.update_db_2("nsrs", nsr_id, db_dict)
4023
4024                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
4025
4026             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
4027             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
4028             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
4029
4030             # Launch task
4031             task_n2vc = asyncio.ensure_future(
4032                 self.instantiate_N2VC(
4033                     logging_text=logging_text,
4034                     vca_index=vca_index,
4035                     nsi_id=nsi_id,
4036                     db_nsr=db_nsr,
4037                     db_vnfr=db_vnfr,
4038                     vdu_id=vdu_id,
4039                     kdu_name=kdu_name,
4040                     vdu_index=vdu_index,
4041                     kdu_index=kdu_index,
4042                     deploy_params=deploy_params,
4043                     config_descriptor=descriptor_config,
4044                     base_folder=base_folder,
4045                     nslcmop_id=nslcmop_id,
4046                     stage=stage,
4047                     vca_type=vca_type,
4048                     vca_name=vca_name,
4049                     ee_config_descriptor=ee_item,
4050                 )
4051             )
4052             self.lcm_tasks.register(
4053                 "ns",
4054                 nsr_id,
4055                 nslcmop_id,
4056                 "instantiate_N2VC-{}".format(vca_index),
4057                 task_n2vc,
4058             )
4059             task_instantiation_info[
4060                 task_n2vc
4061             ] = self.task_name_deploy_vca + " {}.{}".format(
4062                 member_vnf_index or "", vdu_id or ""
4063             )
4064
4065     def _format_additional_params(self, params):
4066         params = params or {}
4067         for key, value in params.items():
4068             if str(value).startswith("!!yaml "):
4069                 params[key] = yaml.safe_load(value[7:])
4070         return params
4071
4072     def _get_terminate_primitive_params(self, seq, vnf_index):
4073         primitive = seq.get("name")
4074         primitive_params = {}
4075         params = {
4076             "member_vnf_index": vnf_index,
4077             "primitive": primitive,
4078             "primitive_params": primitive_params,
4079         }
4080         desc_params = {}
4081         return self._map_primitive_params(seq, params, desc_params)
4082
4083     # sub-operations
4084
4085     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
4086         op = deep_get(db_nslcmop, ("_admin", "operations"), [])[op_index]
4087         if op.get("operationState") == "COMPLETED":
4088             # b. Skip sub-operation
4089             # _ns_execute_primitive() or RO.create_action() will NOT be executed
4090             return self.SUBOPERATION_STATUS_SKIP
4091         else:
4092             # c. retry executing sub-operation
4093             # The sub-operation exists, and operationState != 'COMPLETED'
4094             # Update operationState = 'PROCESSING' to indicate a retry.
4095             operationState = "PROCESSING"
4096             detailed_status = "In progress"
4097             self._update_suboperation_status(
4098                 db_nslcmop, op_index, operationState, detailed_status
4099             )
4100             # Return the sub-operation index
4101             # _ns_execute_primitive() or RO.create_action() will be called from scale()
4102             # with arguments extracted from the sub-operation
4103             return op_index
4104
4105     # Find a sub-operation where all keys in a matching dictionary must match
4106     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
4107     def _find_suboperation(self, db_nslcmop, match):
4108         if db_nslcmop and match:
4109             op_list = db_nslcmop.get("_admin", {}).get("operations", [])
4110             for i, op in enumerate(op_list):
4111                 if all(op.get(k) == match[k] for k in match):
4112                     return i
4113         return self.SUBOPERATION_STATUS_NOT_FOUND
4114
4115     # Update status for a sub-operation given its index
4116     def _update_suboperation_status(
4117         self, db_nslcmop, op_index, operationState, detailed_status
4118     ):
4119         # Update DB for HA tasks
4120         q_filter = {"_id": db_nslcmop["_id"]}
4121         update_dict = {
4122             "_admin.operations.{}.operationState".format(op_index): operationState,
4123             "_admin.operations.{}.detailed-status".format(op_index): detailed_status,
4124         }
4125         self.db.set_one(
4126             "nslcmops", q_filter=q_filter, update_dict=update_dict, fail_on_empty=False
4127         )
4128
4129     # Add sub-operation, return the index of the added sub-operation
4130     # Optionally, set operationState, detailed-status, and operationType
4131     # Status and type are currently set for 'scale' sub-operations:
4132     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
4133     # 'detailed-status' : status message
4134     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
4135     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
4136     def _add_suboperation(
4137         self,
4138         db_nslcmop,
4139         vnf_index,
4140         vdu_id,
4141         vdu_count_index,
4142         vdu_name,
4143         primitive,
4144         mapped_primitive_params,
4145         operationState=None,
4146         detailed_status=None,
4147         operationType=None,
4148         RO_nsr_id=None,
4149         RO_scaling_info=None,
4150     ):
4151         if not db_nslcmop:
4152             return self.SUBOPERATION_STATUS_NOT_FOUND
4153         # Get the "_admin.operations" list, if it exists
4154         db_nslcmop_admin = db_nslcmop.get("_admin", {})
4155         op_list = db_nslcmop_admin.get("operations")
4156         # Create or append to the "_admin.operations" list
4157         new_op = {
4158             "member_vnf_index": vnf_index,
4159             "vdu_id": vdu_id,
4160             "vdu_count_index": vdu_count_index,
4161             "primitive": primitive,
4162             "primitive_params": mapped_primitive_params,
4163         }
4164         if operationState:
4165             new_op["operationState"] = operationState
4166         if detailed_status:
4167             new_op["detailed-status"] = detailed_status
4168         if operationType:
4169             new_op["lcmOperationType"] = operationType
4170         if RO_nsr_id:
4171             new_op["RO_nsr_id"] = RO_nsr_id
4172         if RO_scaling_info:
4173             new_op["RO_scaling_info"] = RO_scaling_info
4174         if not op_list:
4175             # No existing operations, create key 'operations' with current operation as first list element
4176             db_nslcmop_admin.update({"operations": [new_op]})
4177             op_list = db_nslcmop_admin.get("operations")
4178         else:
4179             # Existing operations, append operation to list
4180             op_list.append(new_op)
4181
4182         db_nslcmop_update = {"_admin.operations": op_list}
4183         self.update_db_2("nslcmops", db_nslcmop["_id"], db_nslcmop_update)
4184         op_index = len(op_list) - 1
4185         return op_index
4186
4187     # Helper methods for scale() sub-operations
4188
4189     # pre-scale/post-scale:
4190     # Check for 3 different cases:
4191     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
4192     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
4193     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
4194     def _check_or_add_scale_suboperation(
4195         self,
4196         db_nslcmop,
4197         vnf_index,
4198         vnf_config_primitive,
4199         primitive_params,
4200         operationType,
4201         RO_nsr_id=None,
4202         RO_scaling_info=None,
4203     ):
4204         # Find this sub-operation
4205         if RO_nsr_id and RO_scaling_info:
4206             operationType = "SCALE-RO"
4207             match = {
4208                 "member_vnf_index": vnf_index,
4209                 "RO_nsr_id": RO_nsr_id,
4210                 "RO_scaling_info": RO_scaling_info,
4211             }
4212         else:
4213             match = {
4214                 "member_vnf_index": vnf_index,
4215                 "primitive": vnf_config_primitive,
4216                 "primitive_params": primitive_params,
4217                 "lcmOperationType": operationType,
4218             }
4219         op_index = self._find_suboperation(db_nslcmop, match)
4220         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
4221             # a. New sub-operation
4222             # The sub-operation does not exist, add it.
4223             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
4224             # The following parameters are set to None for all kind of scaling:
4225             vdu_id = None
4226             vdu_count_index = None
4227             vdu_name = None
4228             if RO_nsr_id and RO_scaling_info:
4229                 vnf_config_primitive = None
4230                 primitive_params = None
4231             else:
4232                 RO_nsr_id = None
4233                 RO_scaling_info = None
4234             # Initial status for sub-operation
4235             operationState = "PROCESSING"
4236             detailed_status = "In progress"
4237             # Add sub-operation for pre/post-scaling (zero or more operations)
4238             self._add_suboperation(
4239                 db_nslcmop,
4240                 vnf_index,
4241                 vdu_id,
4242                 vdu_count_index,
4243                 vdu_name,
4244                 vnf_config_primitive,
4245                 primitive_params,
4246                 operationState,
4247                 detailed_status,
4248                 operationType,
4249                 RO_nsr_id,
4250                 RO_scaling_info,
4251             )
4252             return self.SUBOPERATION_STATUS_NEW
4253         else:
4254             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
4255             # or op_index (operationState != 'COMPLETED')
4256             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
4257
4258     # Function to return execution_environment id
4259
4260     async def destroy_N2VC(
4261         self,
4262         logging_text,
4263         db_nslcmop,
4264         vca_deployed,
4265         config_descriptor,
4266         vca_index,
4267         destroy_ee=True,
4268         exec_primitives=True,
4269         scaling_in=False,
4270         vca_id: str = None,
4271     ):
4272         """
4273         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
4274         :param logging_text:
4275         :param db_nslcmop:
4276         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
4277         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
4278         :param vca_index: index in the database _admin.deployed.VCA
4279         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
4280         :param exec_primitives: False to do not execute terminate primitives, because the config is not completed or has
4281                             not executed properly
4282         :param scaling_in: True destroys the application, False destroys the model
4283         :return: None or exception
4284         """
4285
4286         self.logger.debug(
4287             logging_text
4288             + " vca_index: {}, vca_deployed: {}, config_descriptor: {}, destroy_ee: {}".format(
4289                 vca_index, vca_deployed, config_descriptor, destroy_ee
4290             )
4291         )
4292
4293         vca_type = vca_deployed.get("type", "lxc_proxy_charm")
4294
4295         # execute terminate_primitives
4296         if exec_primitives:
4297             terminate_primitives = get_ee_sorted_terminate_config_primitive_list(
4298                 config_descriptor.get("terminate-config-primitive"),
4299                 vca_deployed.get("ee_descriptor_id"),
4300             )
4301             vdu_id = vca_deployed.get("vdu_id")
4302             vdu_count_index = vca_deployed.get("vdu_count_index")
4303             vdu_name = vca_deployed.get("vdu_name")
4304             vnf_index = vca_deployed.get("member-vnf-index")
4305             if terminate_primitives and vca_deployed.get("needed_terminate"):
4306                 for seq in terminate_primitives:
4307                     # For each sequence in list, get primitive and call _ns_execute_primitive()
4308                     step = "Calling terminate action for vnf_member_index={} primitive={}".format(
4309                         vnf_index, seq.get("name")
4310                     )
4311                     self.logger.debug(logging_text + step)
4312                     # Create the primitive for each sequence, i.e. "primitive": "touch"
4313                     primitive = seq.get("name")
4314                     mapped_primitive_params = self._get_terminate_primitive_params(
4315                         seq, vnf_index
4316                     )
4317
4318                     # Add sub-operation
4319                     self._add_suboperation(
4320                         db_nslcmop,
4321                         vnf_index,
4322                         vdu_id,
4323                         vdu_count_index,
4324                         vdu_name,
4325                         primitive,
4326                         mapped_primitive_params,
4327                     )
4328                     # Sub-operations: Call _ns_execute_primitive() instead of action()
4329                     try:
4330                         result, result_detail = await self._ns_execute_primitive(
4331                             vca_deployed["ee_id"],
4332                             primitive,
4333                             mapped_primitive_params,
4334                             vca_type=vca_type,
4335                             vca_id=vca_id,
4336                         )
4337                     except LcmException:
4338                         # this happens when VCA is not deployed. In this case it is not needed to terminate
4339                         continue
4340                     result_ok = ["COMPLETED", "PARTIALLY_COMPLETED"]
4341                     if result not in result_ok:
4342                         raise LcmException(
4343                             "terminate_primitive {}  for vnf_member_index={} fails with "
4344                             "error {}".format(seq.get("name"), vnf_index, result_detail)
4345                         )
4346                 # set that this VCA do not need terminated
4347                 db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(
4348                     vca_index
4349                 )
4350                 self.update_db_2(
4351                     "nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False}
4352                 )
4353
4354         # Delete Prometheus Jobs if any
4355         # This uses NSR_ID, so it will destroy any jobs under this index
4356         self.db.del_list("prometheus_jobs", {"nsr_id": db_nslcmop["nsInstanceId"]})
4357
4358         if destroy_ee:
4359             await self.vca_map[vca_type].delete_execution_environment(
4360                 vca_deployed["ee_id"],
4361                 scaling_in=scaling_in,
4362                 vca_type=vca_type,
4363                 vca_id=vca_id,
4364             )
4365
4366     async def _delete_all_N2VC(self, db_nsr: dict, vca_id: str = None):
4367         self._write_all_config_status(db_nsr=db_nsr, status="TERMINATING")
4368         namespace = "." + db_nsr["_id"]
4369         try:
4370             await self.n2vc.delete_namespace(
4371                 namespace=namespace,
4372                 total_timeout=self.timeout.charm_delete,
4373                 vca_id=vca_id,
4374             )
4375         except N2VCNotFound:  # already deleted. Skip
4376             pass
4377         self._write_all_config_status(db_nsr=db_nsr, status="DELETED")
4378
4379     async def terminate(self, nsr_id, nslcmop_id):
4380         # Try to lock HA task here
4381         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
4382         if not task_is_locked_by_me:
4383             return
4384
4385         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
4386         self.logger.debug(logging_text + "Enter")
4387         timeout_ns_terminate = self.timeout.ns_terminate
4388         db_nsr = None
4389         db_nslcmop = None
4390         operation_params = None
4391         exc = None
4392         error_list = []  # annotates all failed error messages
4393         db_nslcmop_update = {}
4394         autoremove = False  # autoremove after terminated
4395         tasks_dict_info = {}
4396         db_nsr_update = {}
4397         stage = [
4398             "Stage 1/3: Preparing task.",
4399             "Waiting for previous operations to terminate.",
4400             "",
4401         ]
4402         # ^ contains [stage, step, VIM-status]
4403         try:
4404             # wait for any previous tasks in process
4405             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
4406
4407             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
4408             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
4409             operation_params = db_nslcmop.get("operationParams") or {}
4410             if operation_params.get("timeout_ns_terminate"):
4411                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
4412             stage[1] = "Getting nsr={} from db.".format(nsr_id)
4413             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4414
4415             db_nsr_update["operational-status"] = "terminating"
4416             db_nsr_update["config-status"] = "terminating"
4417             self._write_ns_status(
4418                 nsr_id=nsr_id,
4419                 ns_state="TERMINATING",
4420                 current_operation="TERMINATING",
4421                 current_operation_id=nslcmop_id,
4422                 other_update=db_nsr_update,
4423             )
4424             self._write_op_status(op_id=nslcmop_id, queuePosition=0, stage=stage)
4425             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
4426             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
4427                 return
4428
4429             stage[1] = "Getting vnf descriptors from db."
4430             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
4431             db_vnfrs_dict = {
4432                 db_vnfr["member-vnf-index-ref"]: db_vnfr for db_vnfr in db_vnfrs_list
4433             }
4434             db_vnfds_from_id = {}
4435             db_vnfds_from_member_index = {}
4436             # Loop over VNFRs
4437             for vnfr in db_vnfrs_list:
4438                 vnfd_id = vnfr["vnfd-id"]
4439                 if vnfd_id not in db_vnfds_from_id:
4440                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
4441                     db_vnfds_from_id[vnfd_id] = vnfd
4442                 db_vnfds_from_member_index[
4443                     vnfr["member-vnf-index-ref"]
4444                 ] = db_vnfds_from_id[vnfd_id]
4445
4446             # Destroy individual execution environments when there are terminating primitives.
4447             # Rest of EE will be deleted at once
4448             # TODO - check before calling _destroy_N2VC
4449             # if not operation_params.get("skip_terminate_primitives"):#
4450             # or not vca.get("needed_terminate"):
4451             stage[0] = "Stage 2/3 execute terminating primitives."
4452             self.logger.debug(logging_text + stage[0])
4453             stage[1] = "Looking execution environment that needs terminate."
4454             self.logger.debug(logging_text + stage[1])
4455
4456             for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
4457                 config_descriptor = None
4458                 vca_member_vnf_index = vca.get("member-vnf-index")
4459                 vca_id = self.get_vca_id(
4460                     db_vnfrs_dict.get(vca_member_vnf_index)
4461                     if vca_member_vnf_index
4462                     else None,
4463                     db_nsr,
4464                 )
4465                 if not vca or not vca.get("ee_id"):
4466                     continue
4467                 if not vca.get("member-vnf-index"):
4468                     # ns
4469                     config_descriptor = db_nsr.get("ns-configuration")
4470                 elif vca.get("vdu_id"):
4471                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4472                     config_descriptor = get_configuration(db_vnfd, vca.get("vdu_id"))
4473                 elif vca.get("kdu_name"):
4474                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4475                     config_descriptor = get_configuration(db_vnfd, vca.get("kdu_name"))
4476                 else:
4477                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4478                     config_descriptor = get_configuration(db_vnfd, db_vnfd["id"])
4479                 vca_type = vca.get("type")
4480                 exec_terminate_primitives = not operation_params.get(
4481                     "skip_terminate_primitives"
4482                 ) and vca.get("needed_terminate")
4483                 # For helm we must destroy_ee. Also for native_charm, as juju_model cannot be deleted if there are
4484                 # pending native charms
4485                 destroy_ee = True if vca_type in ("helm-v3", "native_charm") else False
4486                 # self.logger.debug(logging_text + "vca_index: {}, ee_id: {}, vca_type: {} destroy_ee: {}".format(
4487                 #     vca_index, vca.get("ee_id"), vca_type, destroy_ee))
4488                 task = asyncio.ensure_future(
4489                     self.destroy_N2VC(
4490                         logging_text,
4491                         db_nslcmop,
4492                         vca,
4493                         config_descriptor,
4494                         vca_index,
4495                         destroy_ee,
4496                         exec_terminate_primitives,
4497                         vca_id=vca_id,
4498                     )
4499                 )
4500                 tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
4501
4502             # wait for pending tasks of terminate primitives
4503             if tasks_dict_info:
4504                 self.logger.debug(
4505                     logging_text
4506                     + "Waiting for tasks {}".format(list(tasks_dict_info.keys()))
4507                 )
4508                 error_list = await self._wait_for_tasks(
4509                     logging_text,
4510                     tasks_dict_info,
4511                     min(self.timeout.charm_delete, timeout_ns_terminate),
4512                     stage,
4513                     nslcmop_id,
4514                 )
4515                 tasks_dict_info.clear()
4516                 if error_list:
4517                     return  # raise LcmException("; ".join(error_list))
4518
4519             # remove All execution environments at once
4520             stage[0] = "Stage 3/3 delete all."
4521
4522             if nsr_deployed.get("VCA"):
4523                 stage[1] = "Deleting all execution environments."
4524                 self.logger.debug(logging_text + stage[1])
4525                 helm_vca_list = get_deployed_vca(db_nsr, {"type": "helm-v3"})
4526                 if helm_vca_list:
4527                     # Delete Namespace and Certificates
4528                     await self.vca_map["helm-v3"].delete_tls_certificate(
4529                         namespace=db_nslcmop["nsInstanceId"],
4530                         certificate_name=self.EE_TLS_NAME,
4531                     )
4532                     await self.vca_map["helm-v3"].delete_namespace(
4533                         namespace=db_nslcmop["nsInstanceId"],
4534                     )
4535                 else:
4536                     vca_id = self.get_vca_id({}, db_nsr)
4537                     task_delete_ee = asyncio.ensure_future(
4538                         asyncio.wait_for(
4539                             self._delete_all_N2VC(db_nsr=db_nsr, vca_id=vca_id),
4540                             timeout=self.timeout.charm_delete,
4541                         )
4542                     )
4543                     tasks_dict_info[task_delete_ee] = "Terminating all VCA"
4544
4545             # Delete from k8scluster
4546             stage[1] = "Deleting KDUs."
4547             self.logger.debug(logging_text + stage[1])
4548             # print(nsr_deployed)
4549             for kdu in get_iterable(nsr_deployed, "K8s"):
4550                 if not kdu or not kdu.get("kdu-instance"):
4551                     continue
4552                 kdu_instance = kdu.get("kdu-instance")
4553                 if kdu.get("k8scluster-type") in self.k8scluster_map:
4554                     # TODO: Uninstall kdu instances taking into account they could be deployed in different VIMs
4555                     vca_id = self.get_vca_id({}, db_nsr)
4556                     task_delete_kdu_instance = asyncio.ensure_future(
4557                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
4558                             cluster_uuid=kdu.get("k8scluster-uuid"),
4559                             kdu_instance=kdu_instance,
4560                             vca_id=vca_id,
4561                             namespace=kdu.get("namespace"),
4562                         )
4563                     )
4564                 else:
4565                     self.logger.error(
4566                         logging_text
4567                         + "Unknown k8s deployment type {}".format(
4568                             kdu.get("k8scluster-type")
4569                         )
4570                     )
4571                     continue
4572                 tasks_dict_info[
4573                     task_delete_kdu_instance
4574                 ] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
4575
4576             # remove from RO
4577             stage[1] = "Deleting ns from VIM."
4578             if self.ro_config.ng:
4579                 task_delete_ro = asyncio.ensure_future(
4580                     self._terminate_ng_ro(
4581                         logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
4582                     )
4583                 )
4584                 tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
4585
4586             # rest of staff will be done at finally
4587
4588         except (
4589             ROclient.ROClientException,
4590             DbException,
4591             LcmException,
4592             N2VCException,
4593         ) as e:
4594             self.logger.error(logging_text + "Exit Exception {}".format(e))
4595             exc = e
4596         except asyncio.CancelledError:
4597             self.logger.error(
4598                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
4599             )
4600             exc = "Operation was cancelled"
4601         except Exception as e:
4602             exc = traceback.format_exc()
4603             self.logger.critical(
4604                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
4605                 exc_info=True,
4606             )
4607         finally:
4608             if exc:
4609                 error_list.append(str(exc))
4610             try:
4611                 # wait for pending tasks
4612                 if tasks_dict_info:
4613                     stage[1] = "Waiting for terminate pending tasks."
4614                     self.logger.debug(logging_text + stage[1])
4615                     error_list += await self._wait_for_tasks(
4616                         logging_text,
4617                         tasks_dict_info,
4618                         timeout_ns_terminate,
4619                         stage,
4620                         nslcmop_id,
4621                     )
4622                 stage[1] = stage[2] = ""
4623             except asyncio.CancelledError:
4624                 error_list.append("Cancelled")
4625                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
4626                 await self._wait_for_tasks(
4627                     logging_text,
4628                     tasks_dict_info,
4629                     timeout_ns_terminate,
4630                     stage,
4631                     nslcmop_id,
4632                 )
4633             except Exception as exc:
4634                 error_list.append(str(exc))
4635             # update status at database
4636             if error_list:
4637                 error_detail = "; ".join(error_list)
4638                 # self.logger.error(logging_text + error_detail)
4639                 error_description_nslcmop = "{} Detail: {}".format(
4640                     stage[0], error_detail
4641                 )
4642                 error_description_nsr = "Operation: TERMINATING.{}, {}.".format(
4643                     nslcmop_id, stage[0]
4644                 )
4645
4646                 db_nsr_update["operational-status"] = "failed"
4647                 db_nsr_update["detailed-status"] = (
4648                     error_description_nsr + " Detail: " + error_detail
4649                 )
4650                 db_nslcmop_update["detailed-status"] = error_detail
4651                 nslcmop_operation_state = "FAILED"
4652                 ns_state = "BROKEN"
4653             else:
4654                 error_detail = None
4655                 error_description_nsr = error_description_nslcmop = None
4656                 ns_state = "NOT_INSTANTIATED"
4657                 db_nsr_update["operational-status"] = "terminated"
4658                 db_nsr_update["detailed-status"] = "Done"
4659                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
4660                 db_nslcmop_update["detailed-status"] = "Done"
4661                 nslcmop_operation_state = "COMPLETED"
4662
4663             if db_nsr:
4664                 self._write_ns_status(
4665                     nsr_id=nsr_id,
4666                     ns_state=ns_state,
4667                     current_operation="IDLE",
4668                     current_operation_id=None,
4669                     error_description=error_description_nsr,
4670                     error_detail=error_detail,
4671                     other_update=db_nsr_update,
4672                 )
4673             self._write_op_status(
4674                 op_id=nslcmop_id,
4675                 stage="",
4676                 error_message=error_description_nslcmop,
4677                 operation_state=nslcmop_operation_state,
4678                 other_update=db_nslcmop_update,
4679             )
4680             if ns_state == "NOT_INSTANTIATED":
4681                 try:
4682                     self.db.set_list(
4683                         "vnfrs",
4684                         {"nsr-id-ref": nsr_id},
4685                         {"_admin.nsState": "NOT_INSTANTIATED"},
4686                     )
4687                 except DbException as e:
4688                     self.logger.warn(
4689                         logging_text
4690                         + "Error writing VNFR status for nsr-id-ref: {} -> {}".format(
4691                             nsr_id, e
4692                         )
4693                     )
4694             if operation_params:
4695                 autoremove = operation_params.get("autoremove", False)
4696             if nslcmop_operation_state:
4697                 try:
4698                     await self.msg.aiowrite(
4699                         "ns",
4700                         "terminated",
4701                         {
4702                             "nsr_id": nsr_id,
4703                             "nslcmop_id": nslcmop_id,
4704                             "operationState": nslcmop_operation_state,
4705                             "autoremove": autoremove,
4706                         },
4707                     )
4708                 except Exception as e:
4709                     self.logger.error(
4710                         logging_text + "kafka_write notification Exception {}".format(e)
4711                     )
4712                 self.logger.debug(f"Deleting alerts: ns_id={nsr_id}")
4713                 self.db.del_list("alerts", {"tags.ns_id": nsr_id})
4714
4715             self.logger.debug(logging_text + "Exit")
4716             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
4717
4718     async def _wait_for_tasks(
4719         self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None
4720     ):
4721         time_start = time()
4722         error_detail_list = []
4723         error_list = []
4724         pending_tasks = list(created_tasks_info.keys())
4725         num_tasks = len(pending_tasks)
4726         num_done = 0
4727         stage[1] = "{}/{}.".format(num_done, num_tasks)
4728         self._write_op_status(nslcmop_id, stage)
4729         while pending_tasks:
4730             new_error = None
4731             _timeout = timeout + time_start - time()
4732             done, pending_tasks = await asyncio.wait(
4733                 pending_tasks, timeout=_timeout, return_when=asyncio.FIRST_COMPLETED
4734             )
4735             num_done += len(done)
4736             if not done:  # Timeout
4737                 for task in pending_tasks:
4738                     new_error = created_tasks_info[task] + ": Timeout"
4739                     error_detail_list.append(new_error)
4740                     error_list.append(new_error)
4741                 break
4742             for task in done:
4743                 if task.cancelled():
4744                     exc = "Cancelled"
4745                 else:
4746                     exc = task.exception()
4747                 if exc:
4748                     if isinstance(exc, asyncio.TimeoutError):
4749                         exc = "Timeout"
4750                     new_error = created_tasks_info[task] + ": {}".format(exc)
4751                     error_list.append(created_tasks_info[task])
4752                     error_detail_list.append(new_error)
4753                     if isinstance(
4754                         exc,
4755                         (
4756                             str,
4757                             DbException,
4758                             N2VCException,
4759                             ROclient.ROClientException,
4760                             LcmException,
4761                             K8sException,
4762                             NgRoException,
4763                         ),
4764                     ):
4765                         self.logger.error(logging_text + new_error)
4766                     else:
4767                         exc_traceback = "".join(
4768                             traceback.format_exception(None, exc, exc.__traceback__)
4769                         )
4770                         self.logger.error(
4771                             logging_text
4772                             + created_tasks_info[task]
4773                             + " "
4774                             + exc_traceback
4775                         )
4776                 else:
4777                     self.logger.debug(
4778                         logging_text + created_tasks_info[task] + ": Done"
4779                     )
4780             stage[1] = "{}/{}.".format(num_done, num_tasks)
4781             if new_error:
4782                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
4783                 if nsr_id:  # update also nsr
4784                     self.update_db_2(
4785                         "nsrs",
4786                         nsr_id,
4787                         {
4788                             "errorDescription": "Error at: " + ", ".join(error_list),
4789                             "errorDetail": ". ".join(error_detail_list),
4790                         },
4791                     )
4792             self._write_op_status(nslcmop_id, stage)
4793         return error_detail_list
4794
4795     async def _cancel_pending_tasks(self, logging_text, created_tasks_info):
4796         for task, name in created_tasks_info.items():
4797             self.logger.debug(logging_text + "Cancelling task: " + name)
4798             task.cancel()
4799
4800     @staticmethod
4801     def _map_primitive_params(primitive_desc, params, instantiation_params):
4802         """
4803         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
4804         The default-value is used. If it is between < > it look for a value at instantiation_params
4805         :param primitive_desc: portion of VNFD/NSD that describes primitive
4806         :param params: Params provided by user
4807         :param instantiation_params: Instantiation params provided by user
4808         :return: a dictionary with the calculated params
4809         """
4810         calculated_params = {}
4811         for parameter in primitive_desc.get("parameter", ()):
4812             param_name = parameter["name"]
4813             if param_name in params:
4814                 calculated_params[param_name] = params[param_name]
4815             elif "default-value" in parameter or "value" in parameter:
4816                 if "value" in parameter:
4817                     calculated_params[param_name] = parameter["value"]
4818                 else:
4819                     calculated_params[param_name] = parameter["default-value"]
4820                 if (
4821                     isinstance(calculated_params[param_name], str)
4822                     and calculated_params[param_name].startswith("<")
4823                     and calculated_params[param_name].endswith(">")
4824                 ):
4825                     if calculated_params[param_name][1:-1] in instantiation_params:
4826                         calculated_params[param_name] = instantiation_params[
4827                             calculated_params[param_name][1:-1]
4828                         ]
4829                     else:
4830                         raise LcmException(
4831                             "Parameter {} needed to execute primitive {} not provided".format(
4832                                 calculated_params[param_name], primitive_desc["name"]
4833                             )
4834                         )
4835             else:
4836                 raise LcmException(
4837                     "Parameter {} needed to execute primitive {} not provided".format(
4838                         param_name, primitive_desc["name"]
4839                     )
4840                 )
4841
4842             if isinstance(calculated_params[param_name], (dict, list, tuple)):
4843                 calculated_params[param_name] = yaml.safe_dump(
4844                     calculated_params[param_name], default_flow_style=True, width=256
4845                 )
4846             elif isinstance(calculated_params[param_name], str) and calculated_params[
4847                 param_name
4848             ].startswith("!!yaml "):
4849                 calculated_params[param_name] = calculated_params[param_name][7:]
4850             if parameter.get("data-type") == "INTEGER":
4851                 try:
4852                     calculated_params[param_name] = int(calculated_params[param_name])
4853                 except ValueError:  # error converting string to int
4854                     raise LcmException(
4855                         "Parameter {} of primitive {} must be integer".format(
4856                             param_name, primitive_desc["name"]
4857                         )
4858                     )
4859             elif parameter.get("data-type") == "BOOLEAN":
4860                 calculated_params[param_name] = not (
4861                     (str(calculated_params[param_name])).lower() == "false"
4862                 )
4863
4864         # add always ns_config_info if primitive name is config
4865         if primitive_desc["name"] == "config":
4866             if "ns_config_info" in instantiation_params:
4867                 calculated_params["ns_config_info"] = instantiation_params[
4868                     "ns_config_info"
4869                 ]
4870         return calculated_params
4871
4872     def _look_for_deployed_vca(
4873         self,
4874         deployed_vca,
4875         member_vnf_index,
4876         vdu_id,
4877         vdu_count_index,
4878         kdu_name=None,
4879         ee_descriptor_id=None,
4880     ):
4881         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
4882         for vca in deployed_vca:
4883             if not vca:
4884                 continue
4885             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
4886                 continue
4887             if (
4888                 vdu_count_index is not None
4889                 and vdu_count_index != vca["vdu_count_index"]
4890             ):
4891                 continue
4892             if kdu_name and kdu_name != vca["kdu_name"]:
4893                 continue
4894             if ee_descriptor_id and ee_descriptor_id != vca["ee_descriptor_id"]:
4895                 continue
4896             break
4897         else:
4898             # vca_deployed not found
4899             raise LcmException(
4900                 "charm for member_vnf_index={} vdu_id={}.{} kdu_name={} execution-environment-list.id={}"
4901                 " is not deployed".format(
4902                     member_vnf_index,
4903                     vdu_id,
4904                     vdu_count_index,
4905                     kdu_name,
4906                     ee_descriptor_id,
4907                 )
4908             )
4909         # get ee_id
4910         ee_id = vca.get("ee_id")
4911         vca_type = vca.get(
4912             "type", "lxc_proxy_charm"
4913         )  # default value for backward compatibility - proxy charm
4914         if not ee_id:
4915             raise LcmException(
4916                 "charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
4917                 "execution environment".format(
4918                     member_vnf_index, vdu_id, kdu_name, vdu_count_index
4919                 )
4920             )
4921         return ee_id, vca_type
4922
4923     async def _ns_execute_primitive(
4924         self,
4925         ee_id,
4926         primitive,
4927         primitive_params,
4928         retries=0,
4929         retries_interval=30,
4930         timeout=None,
4931         vca_type=None,
4932         db_dict=None,
4933         vca_id: str = None,
4934     ) -> (str, str):
4935         try:
4936             if primitive == "config":
4937                 primitive_params = {"params": primitive_params}
4938
4939             vca_type = vca_type or "lxc_proxy_charm"
4940
4941             while retries >= 0:
4942                 try:
4943                     output = await asyncio.wait_for(
4944                         self.vca_map[vca_type].exec_primitive(
4945                             ee_id=ee_id,
4946                             primitive_name=primitive,
4947                             params_dict=primitive_params,
4948                             progress_timeout=self.timeout.progress_primitive,
4949                             total_timeout=self.timeout.primitive,
4950                             db_dict=db_dict,
4951                             vca_id=vca_id,
4952                             vca_type=vca_type,
4953                         ),
4954                         timeout=timeout or self.timeout.primitive,
4955                     )
4956                     # execution was OK
4957                     break
4958                 except asyncio.CancelledError:
4959                     raise
4960                 except Exception as e:
4961                     retries -= 1
4962                     if retries >= 0:
4963                         self.logger.debug(
4964                             "Error executing action {} on {} -> {}".format(
4965                                 primitive, ee_id, e
4966                             )
4967                         )
4968                         # wait and retry
4969                         await asyncio.sleep(retries_interval)
4970                     else:
4971                         if isinstance(e, asyncio.TimeoutError):
4972                             e = N2VCException(
4973                                 message="Timed out waiting for action to complete"
4974                             )
4975                         return "FAILED", getattr(e, "message", repr(e))
4976
4977             return "COMPLETED", output
4978
4979         except (LcmException, asyncio.CancelledError):
4980             raise
4981         except Exception as e:
4982             return "FAIL", "Error executing action {}: {}".format(primitive, e)
4983
4984     async def vca_status_refresh(self, nsr_id, nslcmop_id):
4985         """
4986         Updating the vca_status with latest juju information in nsrs record
4987         :param: nsr_id: Id of the nsr
4988         :param: nslcmop_id: Id of the nslcmop
4989         :return: None
4990         """
4991
4992         self.logger.debug("Task ns={} action={} Enter".format(nsr_id, nslcmop_id))
4993         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4994         vca_id = self.get_vca_id({}, db_nsr)
4995         if db_nsr["_admin"]["deployed"]["K8s"]:
4996             for _, k8s in enumerate(db_nsr["_admin"]["deployed"]["K8s"]):
4997                 cluster_uuid, kdu_instance, cluster_type = (
4998                     k8s["k8scluster-uuid"],
4999                     k8s["kdu-instance"],
5000                     k8s["k8scluster-type"],
5001                 )
5002                 await self._on_update_k8s_db(
5003                     cluster_uuid=cluster_uuid,
5004                     kdu_instance=kdu_instance,
5005                     filter={"_id": nsr_id},
5006                     vca_id=vca_id,
5007                     cluster_type=cluster_type,
5008                 )
5009         else:
5010             for vca_index, _ in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
5011                 table, filter = "nsrs", {"_id": nsr_id}
5012                 path = "_admin.deployed.VCA.{}.".format(vca_index)
5013                 await self._on_update_n2vc_db(table, filter, path, {})
5014
5015         self.logger.debug("Task ns={} action={} Exit".format(nsr_id, nslcmop_id))
5016         self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_vca_status_refresh")
5017
5018     async def action(self, nsr_id, nslcmop_id):
5019         # Try to lock HA task here
5020         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5021         if not task_is_locked_by_me:
5022             return
5023
5024         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
5025         self.logger.debug(logging_text + "Enter")
5026         # get all needed from database
5027         db_nsr = None
5028         db_nslcmop = None
5029         db_nsr_update = {}
5030         db_nslcmop_update = {}
5031         nslcmop_operation_state = None
5032         error_description_nslcmop = None
5033         exc = None
5034         step = ""
5035         try:
5036             # wait for any previous tasks in process
5037             step = "Waiting for previous operations to terminate"
5038             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5039
5040             self._write_ns_status(
5041                 nsr_id=nsr_id,
5042                 ns_state=None,
5043                 current_operation="RUNNING ACTION",
5044                 current_operation_id=nslcmop_id,
5045             )
5046
5047             step = "Getting information from database"
5048             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5049             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5050             if db_nslcmop["operationParams"].get("primitive_params"):
5051                 db_nslcmop["operationParams"]["primitive_params"] = json.loads(
5052                     db_nslcmop["operationParams"]["primitive_params"]
5053                 )
5054
5055             nsr_deployed = db_nsr["_admin"].get("deployed")
5056             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
5057             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
5058             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
5059             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
5060             primitive = db_nslcmop["operationParams"]["primitive"]
5061             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
5062             timeout_ns_action = db_nslcmop["operationParams"].get(
5063                 "timeout_ns_action", self.timeout.primitive
5064             )
5065
5066             if vnf_index:
5067                 step = "Getting vnfr from database"
5068                 db_vnfr = self.db.get_one(
5069                     "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
5070                 )
5071                 if db_vnfr.get("kdur"):
5072                     kdur_list = []
5073                     for kdur in db_vnfr["kdur"]:
5074                         if kdur.get("additionalParams"):
5075                             kdur["additionalParams"] = json.loads(
5076                                 kdur["additionalParams"]
5077                             )
5078                         kdur_list.append(kdur)
5079                     db_vnfr["kdur"] = kdur_list
5080                 step = "Getting vnfd from database"
5081                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
5082
5083                 # Sync filesystem before running a primitive
5084                 self.fs.sync(db_vnfr["vnfd-id"])
5085             else:
5086                 step = "Getting nsd from database"
5087                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
5088
5089             vca_id = self.get_vca_id(db_vnfr, db_nsr)
5090             # for backward compatibility
5091             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
5092                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
5093                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
5094                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5095
5096             # look for primitive
5097             config_primitive_desc = descriptor_configuration = None
5098             if vdu_id:
5099                 descriptor_configuration = get_configuration(db_vnfd, vdu_id)
5100             elif kdu_name:
5101                 descriptor_configuration = get_configuration(db_vnfd, kdu_name)
5102             elif vnf_index:
5103                 descriptor_configuration = get_configuration(db_vnfd, db_vnfd["id"])
5104             else:
5105                 descriptor_configuration = db_nsd.get("ns-configuration")
5106
5107             if descriptor_configuration and descriptor_configuration.get(
5108                 "config-primitive"
5109             ):
5110                 for config_primitive in descriptor_configuration["config-primitive"]:
5111                     if config_primitive["name"] == primitive:
5112                         config_primitive_desc = config_primitive
5113                         break
5114
5115             if not config_primitive_desc:
5116                 if not (kdu_name and primitive in ("upgrade", "rollback", "status")):
5117                     raise LcmException(
5118                         "Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".format(
5119                             primitive
5120                         )
5121                     )
5122                 primitive_name = primitive
5123                 ee_descriptor_id = None
5124             else:
5125                 primitive_name = config_primitive_desc.get(
5126                     "execution-environment-primitive", primitive
5127                 )
5128                 ee_descriptor_id = config_primitive_desc.get(
5129                     "execution-environment-ref"
5130                 )
5131
5132             if vnf_index:
5133                 if vdu_id:
5134                     vdur = next(
5135                         (x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None
5136                     )
5137                     desc_params = parse_yaml_strings(vdur.get("additionalParams"))
5138                 elif kdu_name:
5139                     kdur = next(
5140                         (x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None
5141                     )
5142                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
5143                 else:
5144                     desc_params = parse_yaml_strings(
5145                         db_vnfr.get("additionalParamsForVnf")
5146                     )
5147             else:
5148                 desc_params = parse_yaml_strings(db_nsr.get("additionalParamsForNs"))
5149             if kdu_name and get_configuration(db_vnfd, kdu_name):
5150                 kdu_configuration = get_configuration(db_vnfd, kdu_name)
5151                 actions = set()
5152                 for primitive in kdu_configuration.get("initial-config-primitive", []):
5153                     actions.add(primitive["name"])
5154                 for primitive in kdu_configuration.get("config-primitive", []):
5155                     actions.add(primitive["name"])
5156                 kdu = find_in_list(
5157                     nsr_deployed["K8s"],
5158                     lambda kdu: kdu_name == kdu["kdu-name"]
5159                     and kdu["member-vnf-index"] == vnf_index,
5160                 )
5161                 kdu_action = (
5162                     True
5163                     if primitive_name in actions
5164                     and kdu["k8scluster-type"] != "helm-chart-v3"
5165                     else False
5166                 )
5167
5168             # TODO check if ns is in a proper status
5169             if kdu_name and (
5170                 primitive_name in ("upgrade", "rollback", "status") or kdu_action
5171             ):
5172                 # kdur and desc_params already set from before
5173                 if primitive_params:
5174                     desc_params.update(primitive_params)
5175                 # TODO Check if we will need something at vnf level
5176                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
5177                     if (
5178                         kdu_name == kdu["kdu-name"]
5179                         and kdu["member-vnf-index"] == vnf_index
5180                     ):
5181                         break
5182                 else:
5183                     raise LcmException(
5184                         "KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index)
5185                     )
5186
5187                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
5188                     msg = "unknown k8scluster-type '{}'".format(
5189                         kdu.get("k8scluster-type")
5190                     )
5191                     raise LcmException(msg)
5192
5193                 db_dict = {
5194                     "collection": "nsrs",
5195                     "filter": {"_id": nsr_id},
5196                     "path": "_admin.deployed.K8s.{}".format(index),
5197                 }
5198                 self.logger.debug(
5199                     logging_text
5200                     + "Exec k8s {} on {}.{}".format(primitive_name, vnf_index, kdu_name)
5201                 )
5202                 step = "Executing kdu {}".format(primitive_name)
5203                 if primitive_name == "upgrade":
5204                     if desc_params.get("kdu_model"):
5205                         kdu_model = desc_params.get("kdu_model")
5206                         del desc_params["kdu_model"]
5207                     else:
5208                         kdu_model = kdu.get("kdu-model")
5209                         if kdu_model.count("/") < 2:  # helm chart is not embedded
5210                             parts = kdu_model.split(sep=":")
5211                             if len(parts) == 2:
5212                                 kdu_model = parts[0]
5213                     if desc_params.get("kdu_atomic_upgrade"):
5214                         atomic_upgrade = desc_params.get(
5215                             "kdu_atomic_upgrade"
5216                         ).lower() in ("yes", "true", "1")
5217                         del desc_params["kdu_atomic_upgrade"]
5218                     else:
5219                         atomic_upgrade = True
5220
5221                     detailed_status = await asyncio.wait_for(
5222                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
5223                             cluster_uuid=kdu.get("k8scluster-uuid"),
5224                             kdu_instance=kdu.get("kdu-instance"),
5225                             atomic=atomic_upgrade,
5226                             kdu_model=kdu_model,
5227                             params=desc_params,
5228                             db_dict=db_dict,
5229                             timeout=timeout_ns_action,
5230                         ),
5231                         timeout=timeout_ns_action + 10,
5232                     )
5233                     self.logger.debug(
5234                         logging_text + " Upgrade of kdu {} done".format(detailed_status)
5235                     )
5236                 elif primitive_name == "rollback":
5237                     detailed_status = await asyncio.wait_for(
5238                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
5239                             cluster_uuid=kdu.get("k8scluster-uuid"),
5240                             kdu_instance=kdu.get("kdu-instance"),
5241                             db_dict=db_dict,
5242                         ),
5243                         timeout=timeout_ns_action,
5244                     )
5245                 elif primitive_name == "status":
5246                     detailed_status = await asyncio.wait_for(
5247                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
5248                             cluster_uuid=kdu.get("k8scluster-uuid"),
5249                             kdu_instance=kdu.get("kdu-instance"),
5250                             vca_id=vca_id,
5251                         ),
5252                         timeout=timeout_ns_action,
5253                     )
5254                 else:
5255                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(
5256                         kdu["kdu-name"], nsr_id
5257                     )
5258                     params = self._map_primitive_params(
5259                         config_primitive_desc, primitive_params, desc_params
5260                     )
5261
5262                     detailed_status = await asyncio.wait_for(
5263                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
5264                             cluster_uuid=kdu.get("k8scluster-uuid"),
5265                             kdu_instance=kdu_instance,
5266                             primitive_name=primitive_name,
5267                             params=params,
5268                             db_dict=db_dict,
5269                             timeout=timeout_ns_action,
5270                             vca_id=vca_id,
5271                         ),
5272                         timeout=timeout_ns_action,
5273                     )
5274
5275                 if detailed_status:
5276                     nslcmop_operation_state = "COMPLETED"
5277                 else:
5278                     detailed_status = ""
5279                     nslcmop_operation_state = "FAILED"
5280             else:
5281                 ee_id, vca_type = self._look_for_deployed_vca(
5282                     nsr_deployed["VCA"],
5283                     member_vnf_index=vnf_index,
5284                     vdu_id=vdu_id,
5285                     vdu_count_index=vdu_count_index,
5286                     ee_descriptor_id=ee_descriptor_id,
5287                 )
5288                 for vca_index, vca_deployed in enumerate(
5289                     db_nsr["_admin"]["deployed"]["VCA"]
5290                 ):
5291                     if vca_deployed.get("member-vnf-index") == vnf_index:
5292                         db_dict = {
5293                             "collection": "nsrs",
5294                             "filter": {"_id": nsr_id},
5295                             "path": "_admin.deployed.VCA.{}.".format(vca_index),
5296                         }
5297                         break
5298                 (
5299                     nslcmop_operation_state,
5300                     detailed_status,
5301                 ) = await self._ns_execute_primitive(
5302                     ee_id,
5303                     primitive=primitive_name,
5304                     primitive_params=self._map_primitive_params(
5305                         config_primitive_desc, primitive_params, desc_params
5306                     ),
5307                     timeout=timeout_ns_action,
5308                     vca_type=vca_type,
5309                     db_dict=db_dict,
5310                     vca_id=vca_id,
5311                 )
5312
5313             db_nslcmop_update["detailed-status"] = detailed_status
5314             error_description_nslcmop = (
5315                 detailed_status if nslcmop_operation_state == "FAILED" else ""
5316             )
5317             self.logger.debug(
5318                 logging_text
5319                 + "Done with result {} {}".format(
5320                     nslcmop_operation_state, detailed_status
5321                 )
5322             )
5323             return  # database update is called inside finally
5324
5325         except (DbException, LcmException, N2VCException, K8sException) as e:
5326             self.logger.error(logging_text + "Exit Exception {}".format(e))
5327             exc = e
5328         except asyncio.CancelledError:
5329             self.logger.error(
5330                 logging_text + "Cancelled Exception while '{}'".format(step)
5331             )
5332             exc = "Operation was cancelled"
5333         except asyncio.TimeoutError:
5334             self.logger.error(logging_text + "Timeout while '{}'".format(step))
5335             exc = "Timeout"
5336         except Exception as e:
5337             exc = traceback.format_exc()
5338             self.logger.critical(
5339                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
5340                 exc_info=True,
5341             )
5342         finally:
5343             if exc:
5344                 db_nslcmop_update[
5345                     "detailed-status"
5346                 ] = (
5347                     detailed_status
5348                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
5349                 nslcmop_operation_state = "FAILED"
5350             if db_nsr:
5351                 self._write_ns_status(
5352                     nsr_id=nsr_id,
5353                     ns_state=db_nsr[
5354                         "nsState"
5355                     ],  # TODO check if degraded. For the moment use previous status
5356                     current_operation="IDLE",
5357                     current_operation_id=None,
5358                     # error_description=error_description_nsr,
5359                     # error_detail=error_detail,
5360                     other_update=db_nsr_update,
5361                 )
5362
5363             self._write_op_status(
5364                 op_id=nslcmop_id,
5365                 stage="",
5366                 error_message=error_description_nslcmop,
5367                 operation_state=nslcmop_operation_state,
5368                 other_update=db_nslcmop_update,
5369             )
5370
5371             if nslcmop_operation_state:
5372                 try:
5373                     await self.msg.aiowrite(
5374                         "ns",
5375                         "actioned",
5376                         {
5377                             "nsr_id": nsr_id,
5378                             "nslcmop_id": nslcmop_id,
5379                             "operationState": nslcmop_operation_state,
5380                         },
5381                     )
5382                 except Exception as e:
5383                     self.logger.error(
5384                         logging_text + "kafka_write notification Exception {}".format(e)
5385                     )
5386             self.logger.debug(logging_text + "Exit")
5387             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
5388             return nslcmop_operation_state, detailed_status
5389
5390     async def terminate_vdus(
5391         self, db_vnfr, member_vnf_index, db_nsr, update_db_nslcmops, stage, logging_text
5392     ):
5393         """This method terminates VDUs
5394
5395         Args:
5396             db_vnfr: VNF instance record
5397             member_vnf_index: VNF index to identify the VDUs to be removed
5398             db_nsr: NS instance record
5399             update_db_nslcmops: Nslcmop update record
5400         """
5401         vca_scaling_info = []
5402         scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5403         scaling_info["scaling_direction"] = "IN"
5404         scaling_info["vdu-delete"] = {}
5405         scaling_info["kdu-delete"] = {}
5406         db_vdur = db_vnfr.get("vdur")
5407         vdur_list = copy(db_vdur)
5408         count_index = 0
5409         for index, vdu in enumerate(vdur_list):
5410             vca_scaling_info.append(
5411                 {
5412                     "osm_vdu_id": vdu["vdu-id-ref"],
5413                     "member-vnf-index": member_vnf_index,
5414                     "type": "delete",
5415                     "vdu_index": count_index,
5416                 }
5417             )
5418             scaling_info["vdu-delete"][vdu["vdu-id-ref"]] = count_index
5419             scaling_info["vdu"].append(
5420                 {
5421                     "name": vdu.get("name") or vdu.get("vdu-name"),
5422                     "vdu_id": vdu["vdu-id-ref"],
5423                     "interface": [],
5424                 }
5425             )
5426             for interface in vdu["interfaces"]:
5427                 scaling_info["vdu"][index]["interface"].append(
5428                     {
5429                         "name": interface["name"],
5430                         "ip_address": interface["ip-address"],
5431                         "mac_address": interface.get("mac-address"),
5432                     }
5433                 )
5434             self.logger.info("NS update scaling info{}".format(scaling_info))
5435             stage[2] = "Terminating VDUs"
5436             if scaling_info.get("vdu-delete"):
5437                 # scale_process = "RO"
5438                 if self.ro_config.ng:
5439                     await self._scale_ng_ro(
5440                         logging_text,
5441                         db_nsr,
5442                         update_db_nslcmops,
5443                         db_vnfr,
5444                         scaling_info,
5445                         stage,
5446                     )
5447
5448     async def remove_vnf(self, nsr_id, nslcmop_id, vnf_instance_id):
5449         """This method is to Remove VNF instances from NS.
5450
5451         Args:
5452             nsr_id: NS instance id
5453             nslcmop_id: nslcmop id of update
5454             vnf_instance_id: id of the VNF instance to be removed
5455
5456         Returns:
5457             result: (str, str) COMPLETED/FAILED, details
5458         """
5459         try:
5460             db_nsr_update = {}
5461             logging_text = "Task ns={} update ".format(nsr_id)
5462             check_vnfr_count = len(self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}))
5463             self.logger.info("check_vnfr_count {}".format(check_vnfr_count))
5464             if check_vnfr_count > 1:
5465                 stage = ["", "", ""]
5466                 step = "Getting nslcmop from database"
5467                 self.logger.debug(
5468                     step + " after having waited for previous tasks to be completed"
5469                 )
5470                 # db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5471                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5472                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
5473                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
5474                 """ db_vnfr = self.db.get_one(
5475                     "vnfrs", {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id}) """
5476
5477                 update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5478                 await self.terminate_vdus(
5479                     db_vnfr,
5480                     member_vnf_index,
5481                     db_nsr,
5482                     update_db_nslcmops,
5483                     stage,
5484                     logging_text,
5485                 )
5486
5487                 constituent_vnfr = db_nsr.get("constituent-vnfr-ref")
5488                 constituent_vnfr.remove(db_vnfr.get("_id"))
5489                 db_nsr_update["constituent-vnfr-ref"] = db_nsr.get(
5490                     "constituent-vnfr-ref"
5491                 )
5492                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5493                 self.db.del_one("vnfrs", {"_id": db_vnfr.get("_id")})
5494                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5495                 return "COMPLETED", "Done"
5496             else:
5497                 step = "Terminate VNF Failed with"
5498                 raise LcmException(
5499                     "{} Cannot terminate the last VNF in this NS.".format(
5500                         vnf_instance_id
5501                     )
5502                 )
5503         except (LcmException, asyncio.CancelledError):
5504             raise
5505         except Exception as e:
5506             self.logger.debug("Error removing VNF {}".format(e))
5507             return "FAILED", "Error removing VNF {}".format(e)
5508
5509     async def _ns_redeploy_vnf(
5510         self,
5511         nsr_id,
5512         nslcmop_id,
5513         db_vnfd,
5514         db_vnfr,
5515         db_nsr,
5516     ):
5517         """This method updates and redeploys VNF instances
5518
5519         Args:
5520             nsr_id: NS instance id
5521             nslcmop_id:   nslcmop id
5522             db_vnfd: VNF descriptor
5523             db_vnfr: VNF instance record
5524             db_nsr: NS instance record
5525
5526         Returns:
5527             result: (str, str) COMPLETED/FAILED, details
5528         """
5529         try:
5530             count_index = 0
5531             stage = ["", "", ""]
5532             logging_text = "Task ns={} update ".format(nsr_id)
5533             latest_vnfd_revision = db_vnfd["_admin"].get("revision")
5534             member_vnf_index = db_vnfr["member-vnf-index-ref"]
5535
5536             # Terminate old VNF resources
5537             update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5538             await self.terminate_vdus(
5539                 db_vnfr,
5540                 member_vnf_index,
5541                 db_nsr,
5542                 update_db_nslcmops,
5543                 stage,
5544                 logging_text,
5545             )
5546
5547             # old_vnfd_id = db_vnfr["vnfd-id"]
5548             # new_db_vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
5549             new_db_vnfd = db_vnfd
5550             # new_vnfd_ref = new_db_vnfd["id"]
5551             # new_vnfd_id = vnfd_id
5552
5553             # Create VDUR
5554             new_vnfr_cp = []
5555             for cp in new_db_vnfd.get("ext-cpd", ()):
5556                 vnf_cp = {
5557                     "name": cp.get("id"),
5558                     "connection-point-id": cp.get("int-cpd", {}).get("cpd"),
5559                     "connection-point-vdu-id": cp.get("int-cpd", {}).get("vdu-id"),
5560                     "id": cp.get("id"),
5561                 }
5562                 new_vnfr_cp.append(vnf_cp)
5563             new_vdur = update_db_nslcmops["operationParams"]["newVdur"]
5564             # new_vdur = self._create_vdur_descriptor_from_vnfd(db_nsd, db_vnfd, old_db_vnfd, vnfd_id, db_nsr, member_vnf_index)
5565             # new_vnfr_update = {"vnfd-ref": new_vnfd_ref, "vnfd-id": new_vnfd_id, "connection-point": new_vnfr_cp, "vdur": new_vdur, "ip-address": ""}
5566             new_vnfr_update = {
5567                 "revision": latest_vnfd_revision,
5568                 "connection-point": new_vnfr_cp,
5569                 "vdur": new_vdur,
5570                 "ip-address": "",
5571             }
5572             self.update_db_2("vnfrs", db_vnfr["_id"], new_vnfr_update)
5573             updated_db_vnfr = self.db.get_one(
5574                 "vnfrs",
5575                 {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id},
5576             )
5577
5578             # Instantiate new VNF resources
5579             # update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5580             vca_scaling_info = []
5581             scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5582             scaling_info["scaling_direction"] = "OUT"
5583             scaling_info["vdu-create"] = {}
5584             scaling_info["kdu-create"] = {}
5585             vdud_instantiate_list = db_vnfd["vdu"]
5586             for index, vdud in enumerate(vdud_instantiate_list):
5587                 cloud_init_text = self._get_vdu_cloud_init_content(vdud, db_vnfd)
5588                 if cloud_init_text:
5589                     additional_params = (
5590                         self._get_vdu_additional_params(updated_db_vnfr, vdud["id"])
5591                         or {}
5592                     )
5593                 cloud_init_list = []
5594                 if cloud_init_text:
5595                     # TODO Information of its own ip is not available because db_vnfr is not updated.
5596                     additional_params["OSM"] = get_osm_params(
5597                         updated_db_vnfr, vdud["id"], 1
5598                     )
5599                     cloud_init_list.append(
5600                         self._parse_cloud_init(
5601                             cloud_init_text,
5602                             additional_params,
5603                             db_vnfd["id"],
5604                             vdud["id"],
5605                         )
5606                     )
5607                     vca_scaling_info.append(
5608                         {
5609                             "osm_vdu_id": vdud["id"],
5610                             "member-vnf-index": member_vnf_index,
5611                             "type": "create",
5612                             "vdu_index": count_index,
5613                         }
5614                     )
5615                 scaling_info["vdu-create"][vdud["id"]] = count_index
5616             if self.ro_config.ng:
5617                 self.logger.debug(
5618                     "New Resources to be deployed: {}".format(scaling_info)
5619                 )
5620                 await self._scale_ng_ro(
5621                     logging_text,
5622                     db_nsr,
5623                     update_db_nslcmops,
5624                     updated_db_vnfr,
5625                     scaling_info,
5626                     stage,
5627                 )
5628                 return "COMPLETED", "Done"
5629         except (LcmException, asyncio.CancelledError):
5630             raise
5631         except Exception as e:
5632             self.logger.debug("Error updating VNF {}".format(e))
5633             return "FAILED", "Error updating VNF {}".format(e)
5634
5635     async def _ns_charm_upgrade(
5636         self,
5637         ee_id,
5638         charm_id,
5639         charm_type,
5640         path,
5641         timeout: float = None,
5642     ) -> (str, str):
5643         """This method upgrade charms in VNF instances
5644
5645         Args:
5646             ee_id:  Execution environment id
5647             path:   Local path to the charm
5648             charm_id: charm-id
5649             charm_type: Charm type can be lxc-proxy-charm, native-charm or k8s-proxy-charm
5650             timeout: (Float)    Timeout for the ns update operation
5651
5652         Returns:
5653             result: (str, str) COMPLETED/FAILED, details
5654         """
5655         try:
5656             charm_type = charm_type or "lxc_proxy_charm"
5657             output = await self.vca_map[charm_type].upgrade_charm(
5658                 ee_id=ee_id,
5659                 path=path,
5660                 charm_id=charm_id,
5661                 charm_type=charm_type,
5662                 timeout=timeout or self.timeout.ns_update,
5663             )
5664
5665             if output:
5666                 return "COMPLETED", output
5667
5668         except (LcmException, asyncio.CancelledError):
5669             raise
5670
5671         except Exception as e:
5672             self.logger.debug("Error upgrading charm {}".format(path))
5673
5674             return "FAILED", "Error upgrading charm {}: {}".format(path, e)
5675
5676     async def update(self, nsr_id, nslcmop_id):
5677         """Update NS according to different update types
5678
5679         This method performs upgrade of VNF instances then updates the revision
5680         number in VNF record
5681
5682         Args:
5683             nsr_id: Network service will be updated
5684             nslcmop_id: ns lcm operation id
5685
5686         Returns:
5687              It may raise DbException, LcmException, N2VCException, K8sException
5688
5689         """
5690         # Try to lock HA task here
5691         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5692         if not task_is_locked_by_me:
5693             return
5694
5695         logging_text = "Task ns={} update={} ".format(nsr_id, nslcmop_id)
5696         self.logger.debug(logging_text + "Enter")
5697
5698         # Set the required variables to be filled up later
5699         db_nsr = None
5700         db_nslcmop_update = {}
5701         vnfr_update = {}
5702         nslcmop_operation_state = None
5703         db_nsr_update = {}
5704         error_description_nslcmop = ""
5705         exc = None
5706         change_type = "updated"
5707         detailed_status = ""
5708         member_vnf_index = None
5709
5710         try:
5711             # wait for any previous tasks in process
5712             step = "Waiting for previous operations to terminate"
5713             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5714             self._write_ns_status(
5715                 nsr_id=nsr_id,
5716                 ns_state=None,
5717                 current_operation="UPDATING",
5718                 current_operation_id=nslcmop_id,
5719             )
5720
5721             step = "Getting nslcmop from database"
5722             db_nslcmop = self.db.get_one(
5723                 "nslcmops", {"_id": nslcmop_id}, fail_on_empty=False
5724             )
5725             update_type = db_nslcmop["operationParams"]["updateType"]
5726
5727             step = "Getting nsr from database"
5728             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5729             old_operational_status = db_nsr["operational-status"]
5730             db_nsr_update["operational-status"] = "updating"
5731             self.update_db_2("nsrs", nsr_id, db_nsr_update)
5732             nsr_deployed = db_nsr["_admin"].get("deployed")
5733
5734             if update_type == "CHANGE_VNFPKG":
5735                 # Get the input parameters given through update request
5736                 vnf_instance_id = db_nslcmop["operationParams"][
5737                     "changeVnfPackageData"
5738                 ].get("vnfInstanceId")
5739
5740                 vnfd_id = db_nslcmop["operationParams"]["changeVnfPackageData"].get(
5741                     "vnfdId"
5742                 )
5743                 timeout_seconds = db_nslcmop["operationParams"].get("timeout_ns_update")
5744
5745                 step = "Getting vnfr from database"
5746                 db_vnfr = self.db.get_one(
5747                     "vnfrs", {"_id": vnf_instance_id}, fail_on_empty=False
5748                 )
5749
5750                 step = "Getting vnfds from database"
5751                 # Latest VNFD
5752                 latest_vnfd = self.db.get_one(
5753                     "vnfds", {"_id": vnfd_id}, fail_on_empty=False
5754                 )
5755                 latest_vnfd_revision = latest_vnfd["_admin"].get("revision")
5756
5757                 # Current VNFD
5758                 current_vnf_revision = db_vnfr.get("revision", 1)
5759                 current_vnfd = self.db.get_one(
5760                     "vnfds_revisions",
5761                     {"_id": vnfd_id + ":" + str(current_vnf_revision)},
5762                     fail_on_empty=False,
5763                 )
5764                 # Charm artifact paths will be filled up later
5765                 (
5766                     current_charm_artifact_path,
5767                     target_charm_artifact_path,
5768                     charm_artifact_paths,
5769                     helm_artifacts,
5770                 ) = ([], [], [], [])
5771
5772                 step = "Checking if revision has changed in VNFD"
5773                 if current_vnf_revision != latest_vnfd_revision:
5774                     change_type = "policy_updated"
5775
5776                     # There is new revision of VNFD, update operation is required
5777                     current_vnfd_path = vnfd_id + ":" + str(current_vnf_revision)
5778                     latest_vnfd_path = vnfd_id + ":" + str(latest_vnfd_revision)
5779
5780                     step = "Removing the VNFD packages if they exist in the local path"
5781                     shutil.rmtree(self.fs.path + current_vnfd_path, ignore_errors=True)
5782                     shutil.rmtree(self.fs.path + latest_vnfd_path, ignore_errors=True)
5783
5784                     step = "Get the VNFD packages from FSMongo"
5785                     self.fs.sync(from_path=latest_vnfd_path)
5786                     self.fs.sync(from_path=current_vnfd_path)
5787
5788                     step = (
5789                         "Get the charm-type, charm-id, ee-id if there is deployed VCA"
5790                     )
5791                     current_base_folder = current_vnfd["_admin"]["storage"]
5792                     latest_base_folder = latest_vnfd["_admin"]["storage"]
5793
5794                     for vca_index, vca_deployed in enumerate(
5795                         get_iterable(nsr_deployed, "VCA")
5796                     ):
5797                         vnf_index = db_vnfr.get("member-vnf-index-ref")
5798
5799                         # Getting charm-id and charm-type
5800                         if vca_deployed.get("member-vnf-index") == vnf_index:
5801                             vca_id = self.get_vca_id(db_vnfr, db_nsr)
5802                             vca_type = vca_deployed.get("type")
5803                             vdu_count_index = vca_deployed.get("vdu_count_index")
5804
5805                             # Getting ee-id
5806                             ee_id = vca_deployed.get("ee_id")
5807
5808                             step = "Getting descriptor config"
5809                             if current_vnfd.get("kdu"):
5810                                 search_key = "kdu_name"
5811                             else:
5812                                 search_key = "vnfd_id"
5813
5814                             entity_id = vca_deployed.get(search_key)
5815
5816                             descriptor_config = get_configuration(
5817                                 current_vnfd, entity_id
5818                             )
5819
5820                             if "execution-environment-list" in descriptor_config:
5821                                 ee_list = descriptor_config.get(
5822                                     "execution-environment-list", []
5823                                 )
5824                             else:
5825                                 ee_list = []
5826
5827                             # There could be several charm used in the same VNF
5828                             for ee_item in ee_list:
5829                                 if ee_item.get("juju"):
5830                                     step = "Getting charm name"
5831                                     charm_name = ee_item["juju"].get("charm")
5832
5833                                     step = "Setting Charm artifact paths"
5834                                     current_charm_artifact_path.append(
5835                                         get_charm_artifact_path(
5836                                             current_base_folder,
5837                                             charm_name,
5838                                             vca_type,
5839                                             current_vnf_revision,
5840                                         )
5841                                     )
5842                                     target_charm_artifact_path.append(
5843                                         get_charm_artifact_path(
5844                                             latest_base_folder,
5845                                             charm_name,
5846                                             vca_type,
5847                                             latest_vnfd_revision,
5848                                         )
5849                                     )
5850                                 elif ee_item.get("helm-chart"):
5851                                     # add chart to list and all parameters
5852                                     step = "Getting helm chart name"
5853                                     chart_name = ee_item.get("helm-chart")
5854                                     vca_type = "helm-v3"
5855                                     step = "Setting Helm chart artifact paths"
5856
5857                                     helm_artifacts.append(
5858                                         {
5859                                             "current_artifact_path": get_charm_artifact_path(
5860                                                 current_base_folder,
5861                                                 chart_name,
5862                                                 vca_type,
5863                                                 current_vnf_revision,
5864                                             ),
5865                                             "target_artifact_path": get_charm_artifact_path(
5866                                                 latest_base_folder,
5867                                                 chart_name,
5868                                                 vca_type,
5869                                                 latest_vnfd_revision,
5870                                             ),
5871                                             "ee_id": ee_id,
5872                                             "vca_index": vca_index,
5873                                             "vdu_index": vdu_count_index,
5874                                         }
5875                                     )
5876
5877                             charm_artifact_paths = zip(
5878                                 current_charm_artifact_path, target_charm_artifact_path
5879                             )
5880
5881                     step = "Checking if software version has changed in VNFD"
5882                     if find_software_version(current_vnfd) != find_software_version(
5883                         latest_vnfd
5884                     ):
5885                         step = "Checking if existing VNF has charm"
5886                         for current_charm_path, target_charm_path in list(
5887                             charm_artifact_paths
5888                         ):
5889                             if current_charm_path:
5890                                 raise LcmException(
5891                                     "Software version change is not supported as VNF instance {} has charm.".format(
5892                                         vnf_instance_id
5893                                     )
5894                                 )
5895
5896                         step = "Checking whether the descriptor has SFC"
5897                         if db_nsr.get("nsd", {}).get("vnffgd"):
5898                             raise LcmException(
5899                                 "Ns update is not allowed for NS with SFC"
5900                             )
5901
5902                         # There is no change in the charm package, then redeploy the VNF
5903                         # based on new descriptor
5904                         step = "Redeploying VNF"
5905                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5906                         (result, detailed_status) = await self._ns_redeploy_vnf(
5907                             nsr_id, nslcmop_id, latest_vnfd, db_vnfr, db_nsr
5908                         )
5909                         if result == "FAILED":
5910                             nslcmop_operation_state = result
5911                             error_description_nslcmop = detailed_status
5912                             old_operational_status = "failed"
5913                         db_nslcmop_update["detailed-status"] = detailed_status
5914                         db_nsr_update["detailed-status"] = detailed_status
5915                         scaling_aspect = get_scaling_aspect(latest_vnfd)
5916                         scaling_group_desc = db_nsr.get("_admin").get(
5917                             "scaling-group", None
5918                         )
5919                         if scaling_group_desc:
5920                             for aspect in scaling_aspect:
5921                                 scaling_group_id = aspect.get("id")
5922                                 for scale_index, scaling_group in enumerate(
5923                                     scaling_group_desc
5924                                 ):
5925                                     if scaling_group.get("name") == scaling_group_id:
5926                                         db_nsr_update[
5927                                             "_admin.scaling-group.{}.nb-scale-op".format(
5928                                                 scale_index
5929                                             )
5930                                         ] = 0
5931                         self.logger.debug(
5932                             logging_text
5933                             + " step {} Done with result {} {}".format(
5934                                 step, nslcmop_operation_state, detailed_status
5935                             )
5936                         )
5937
5938                     else:
5939                         step = "Checking if any charm package has changed or not"
5940                         for current_charm_path, target_charm_path in list(
5941                             charm_artifact_paths
5942                         ):
5943                             if (
5944                                 current_charm_path
5945                                 and target_charm_path
5946                                 and self.check_charm_hash_changed(
5947                                     current_charm_path, target_charm_path
5948                                 )
5949                             ):
5950                                 step = "Checking whether VNF uses juju bundle"
5951                                 if check_juju_bundle_existence(current_vnfd):
5952                                     raise LcmException(
5953                                         "Charm upgrade is not supported for the instance which"
5954                                         " uses juju-bundle: {}".format(
5955                                             check_juju_bundle_existence(current_vnfd)
5956                                         )
5957                                     )
5958
5959                                 step = "Upgrading Charm"
5960                                 (
5961                                     result,
5962                                     detailed_status,
5963                                 ) = await self._ns_charm_upgrade(
5964                                     ee_id=ee_id,
5965                                     charm_id=vca_id,
5966                                     charm_type=vca_type,
5967                                     path=self.fs.path + target_charm_path,
5968                                     timeout=timeout_seconds,
5969                                 )
5970
5971                                 if result == "FAILED":
5972                                     nslcmop_operation_state = result
5973                                     error_description_nslcmop = detailed_status
5974
5975                                 db_nslcmop_update["detailed-status"] = detailed_status
5976                                 self.logger.debug(
5977                                     logging_text
5978                                     + " step {} Done with result {} {}".format(
5979                                         step, nslcmop_operation_state, detailed_status
5980                                     )
5981                                 )
5982
5983                         step = "Updating policies"
5984                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5985                         result = "COMPLETED"
5986                         detailed_status = "Done"
5987                         db_nslcmop_update["detailed-status"] = "Done"
5988
5989                     # helm base EE
5990                     for item in helm_artifacts:
5991                         if not (
5992                             item["current_artifact_path"]
5993                             and item["target_artifact_path"]
5994                             and self.check_charm_hash_changed(
5995                                 item["current_artifact_path"],
5996                                 item["target_artifact_path"],
5997                             )
5998                         ):
5999                             continue
6000                         db_update_entry = "_admin.deployed.VCA.{}.".format(
6001                             item["vca_index"]
6002                         )
6003                         vnfr_id = db_vnfr["_id"]
6004                         osm_config = {"osm": {"ns_id": nsr_id, "vnf_id": vnfr_id}}
6005                         db_dict = {
6006                             "collection": "nsrs",
6007                             "filter": {"_id": nsr_id},
6008                             "path": db_update_entry,
6009                         }
6010                         vca_type, namespace, helm_id = get_ee_id_parts(item["ee_id"])
6011                         await self.vca_map[vca_type].upgrade_execution_environment(
6012                             namespace=namespace,
6013                             helm_id=helm_id,
6014                             db_dict=db_dict,
6015                             config=osm_config,
6016                             artifact_path=item["target_artifact_path"],
6017                             vca_type=vca_type,
6018                         )
6019                         vnf_id = db_vnfr.get("vnfd-ref")
6020                         config_descriptor = get_configuration(latest_vnfd, vnf_id)
6021                         self.logger.debug("get ssh key block")
6022                         rw_mgmt_ip = None
6023                         if deep_get(
6024                             config_descriptor,
6025                             ("config-access", "ssh-access", "required"),
6026                         ):
6027                             # Needed to inject a ssh key
6028                             user = deep_get(
6029                                 config_descriptor,
6030                                 ("config-access", "ssh-access", "default-user"),
6031                             )
6032                             step = (
6033                                 "Install configuration Software, getting public ssh key"
6034                             )
6035                             pub_key = await self.vca_map[
6036                                 vca_type
6037                             ].get_ee_ssh_public__key(
6038                                 ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
6039                             )
6040
6041                             step = (
6042                                 "Insert public key into VM user={} ssh_key={}".format(
6043                                     user, pub_key
6044                                 )
6045                             )
6046                             self.logger.debug(logging_text + step)
6047
6048                             # wait for RO (ip-address) Insert pub_key into VM
6049                             rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
6050                                 logging_text,
6051                                 nsr_id,
6052                                 vnfr_id,
6053                                 None,
6054                                 item["vdu_index"],
6055                                 user=user,
6056                                 pub_key=pub_key,
6057                             )
6058
6059                         initial_config_primitive_list = config_descriptor.get(
6060                             "initial-config-primitive"
6061                         )
6062                         config_primitive = next(
6063                             (
6064                                 p
6065                                 for p in initial_config_primitive_list
6066                                 if p["name"] == "config"
6067                             ),
6068                             None,
6069                         )
6070                         if not config_primitive:
6071                             continue
6072
6073                         deploy_params = {"OSM": get_osm_params(db_vnfr)}
6074                         if rw_mgmt_ip:
6075                             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
6076                         if db_vnfr.get("additionalParamsForVnf"):
6077                             deploy_params.update(
6078                                 parse_yaml_strings(
6079                                     db_vnfr["additionalParamsForVnf"].copy()
6080                                 )
6081                             )
6082                         primitive_params_ = self._map_primitive_params(
6083                             config_primitive, {}, deploy_params
6084                         )
6085
6086                         step = "execute primitive '{}' params '{}'".format(
6087                             config_primitive["name"], primitive_params_
6088                         )
6089                         self.logger.debug(logging_text + step)
6090                         await self.vca_map[vca_type].exec_primitive(
6091                             ee_id=ee_id,
6092                             primitive_name=config_primitive["name"],
6093                             params_dict=primitive_params_,
6094                             db_dict=db_dict,
6095                             vca_id=vca_id,
6096                             vca_type=vca_type,
6097                         )
6098
6099                         step = "Updating policies"
6100                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
6101                         detailed_status = "Done"
6102                         db_nslcmop_update["detailed-status"] = "Done"
6103
6104                     #  If nslcmop_operation_state is None, so any operation is not failed.
6105                     if not nslcmop_operation_state:
6106                         nslcmop_operation_state = "COMPLETED"
6107
6108                         # If update CHANGE_VNFPKG nslcmop_operation is successful
6109                         # vnf revision need to be updated
6110                         vnfr_update["revision"] = latest_vnfd_revision
6111                         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
6112
6113                     self.logger.debug(
6114                         logging_text
6115                         + " task Done with result {} {}".format(
6116                             nslcmop_operation_state, detailed_status
6117                         )
6118                     )
6119             elif update_type == "REMOVE_VNF":
6120                 # This part is included in https://osm.etsi.org/gerrit/11876
6121                 vnf_instance_id = db_nslcmop["operationParams"]["removeVnfInstanceId"]
6122                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
6123                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
6124                 step = "Removing VNF"
6125                 (result, detailed_status) = await self.remove_vnf(
6126                     nsr_id, nslcmop_id, vnf_instance_id
6127                 )
6128                 if result == "FAILED":
6129                     nslcmop_operation_state = result
6130                     error_description_nslcmop = detailed_status
6131                 db_nslcmop_update["detailed-status"] = detailed_status
6132                 change_type = "vnf_terminated"
6133                 if not nslcmop_operation_state:
6134                     nslcmop_operation_state = "COMPLETED"
6135                 self.logger.debug(
6136                     logging_text
6137                     + " task Done with result {} {}".format(
6138                         nslcmop_operation_state, detailed_status
6139                     )
6140                 )
6141
6142             elif update_type == "OPERATE_VNF":
6143                 vnf_id = db_nslcmop["operationParams"]["operateVnfData"][
6144                     "vnfInstanceId"
6145                 ]
6146                 operation_type = db_nslcmop["operationParams"]["operateVnfData"][
6147                     "changeStateTo"
6148                 ]
6149                 additional_param = db_nslcmop["operationParams"]["operateVnfData"][
6150                     "additionalParam"
6151                 ]
6152                 (result, detailed_status) = await self.rebuild_start_stop(
6153                     nsr_id, nslcmop_id, vnf_id, additional_param, operation_type
6154                 )
6155                 if result == "FAILED":
6156                     nslcmop_operation_state = result
6157                     error_description_nslcmop = detailed_status
6158                 db_nslcmop_update["detailed-status"] = detailed_status
6159                 if not nslcmop_operation_state:
6160                     nslcmop_operation_state = "COMPLETED"
6161                 self.logger.debug(
6162                     logging_text
6163                     + " task Done with result {} {}".format(
6164                         nslcmop_operation_state, detailed_status
6165                     )
6166                 )
6167
6168             #  If nslcmop_operation_state is None, so any operation is not failed.
6169             #  All operations are executed in overall.
6170             if not nslcmop_operation_state:
6171                 nslcmop_operation_state = "COMPLETED"
6172             db_nsr_update["operational-status"] = old_operational_status
6173
6174         except (DbException, LcmException, N2VCException, K8sException) as e:
6175             self.logger.error(logging_text + "Exit Exception {}".format(e))
6176             exc = e
6177         except asyncio.CancelledError:
6178             self.logger.error(
6179                 logging_text + "Cancelled Exception while '{}'".format(step)
6180             )
6181             exc = "Operation was cancelled"
6182         except asyncio.TimeoutError:
6183             self.logger.error(logging_text + "Timeout while '{}'".format(step))
6184             exc = "Timeout"
6185         except Exception as e:
6186             exc = traceback.format_exc()
6187             self.logger.critical(
6188                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
6189                 exc_info=True,
6190             )
6191         finally:
6192             if exc:
6193                 db_nslcmop_update[
6194                     "detailed-status"
6195                 ] = (
6196                     detailed_status
6197                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
6198                 nslcmop_operation_state = "FAILED"
6199                 db_nsr_update["operational-status"] = old_operational_status
6200             if db_nsr:
6201                 self._write_ns_status(
6202                     nsr_id=nsr_id,
6203                     ns_state=db_nsr["nsState"],
6204                     current_operation="IDLE",
6205                     current_operation_id=None,
6206                     other_update=db_nsr_update,
6207                 )
6208
6209             self._write_op_status(
6210                 op_id=nslcmop_id,
6211                 stage="",
6212                 error_message=error_description_nslcmop,
6213                 operation_state=nslcmop_operation_state,
6214                 other_update=db_nslcmop_update,
6215             )
6216
6217             if nslcmop_operation_state:
6218                 try:
6219                     msg = {
6220                         "nsr_id": nsr_id,
6221                         "nslcmop_id": nslcmop_id,
6222                         "operationState": nslcmop_operation_state,
6223                     }
6224                     if (
6225                         change_type in ("vnf_terminated", "policy_updated")
6226                         and member_vnf_index
6227                     ):
6228                         msg.update({"vnf_member_index": member_vnf_index})
6229                     await self.msg.aiowrite("ns", change_type, msg)
6230                 except Exception as e:
6231                     self.logger.error(
6232                         logging_text + "kafka_write notification Exception {}".format(e)
6233                     )
6234             self.logger.debug(logging_text + "Exit")
6235             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_update")
6236             return nslcmop_operation_state, detailed_status
6237
6238     async def scale(self, nsr_id, nslcmop_id):
6239         # Try to lock HA task here
6240         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
6241         if not task_is_locked_by_me:
6242             return
6243
6244         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
6245         stage = ["", "", ""]
6246         tasks_dict_info = {}
6247         # ^ stage, step, VIM progress
6248         self.logger.debug(logging_text + "Enter")
6249         # get all needed from database
6250         db_nsr = None
6251         db_nslcmop_update = {}
6252         db_nsr_update = {}
6253         exc = None
6254         # in case of error, indicates what part of scale was failed to put nsr at error status
6255         scale_process = None
6256         old_operational_status = ""
6257         old_config_status = ""
6258         nsi_id = None
6259         prom_job_name = ""
6260         try:
6261             # wait for any previous tasks in process
6262             step = "Waiting for previous operations to terminate"
6263             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
6264             self._write_ns_status(
6265                 nsr_id=nsr_id,
6266                 ns_state=None,
6267                 current_operation="SCALING",
6268                 current_operation_id=nslcmop_id,
6269             )
6270
6271             step = "Getting nslcmop from database"
6272             self.logger.debug(
6273                 step + " after having waited for previous tasks to be completed"
6274             )
6275             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
6276
6277             step = "Getting nsr from database"
6278             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
6279             old_operational_status = db_nsr["operational-status"]
6280             old_config_status = db_nsr["config-status"]
6281
6282             step = "Checking whether the descriptor has SFC"
6283             if db_nsr.get("nsd", {}).get("vnffgd"):
6284                 raise LcmException("Scaling is not allowed for NS with SFC")
6285
6286             step = "Parsing scaling parameters"
6287             db_nsr_update["operational-status"] = "scaling"
6288             self.update_db_2("nsrs", nsr_id, db_nsr_update)
6289             nsr_deployed = db_nsr["_admin"].get("deployed")
6290
6291             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"][
6292                 "scaleByStepData"
6293             ]["member-vnf-index"]
6294             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"][
6295                 "scaleByStepData"
6296             ]["scaling-group-descriptor"]
6297             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
6298             # for backward compatibility
6299             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
6300                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
6301                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
6302                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6303
6304             step = "Getting vnfr from database"
6305             db_vnfr = self.db.get_one(
6306                 "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
6307             )
6308
6309             vca_id = self.get_vca_id(db_vnfr, db_nsr)
6310
6311             step = "Getting vnfd from database"
6312             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
6313
6314             base_folder = db_vnfd["_admin"]["storage"]
6315
6316             step = "Getting scaling-group-descriptor"
6317             scaling_descriptor = find_in_list(
6318                 get_scaling_aspect(db_vnfd),
6319                 lambda scale_desc: scale_desc["name"] == scaling_group,
6320             )
6321             if not scaling_descriptor:
6322                 raise LcmException(
6323                     "input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
6324                     "at vnfd:scaling-group-descriptor".format(scaling_group)
6325                 )
6326
6327             step = "Sending scale order to VIM"
6328             # TODO check if ns is in a proper status
6329             nb_scale_op = 0
6330             if not db_nsr["_admin"].get("scaling-group"):
6331                 self.update_db_2(
6332                     "nsrs",
6333                     nsr_id,
6334                     {
6335                         "_admin.scaling-group": [
6336                             {
6337                                 "name": scaling_group,
6338                                 "vnf_index": vnf_index,
6339                                 "nb-scale-op": 0,
6340                             }
6341                         ]
6342                     },
6343                 )
6344                 admin_scale_index = 0
6345             else:
6346                 for admin_scale_index, admin_scale_info in enumerate(
6347                     db_nsr["_admin"]["scaling-group"]
6348                 ):
6349                     if (
6350                         admin_scale_info["name"] == scaling_group
6351                         and admin_scale_info["vnf_index"] == vnf_index
6352                     ):
6353                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
6354                         break
6355                 else:  # not found, set index one plus last element and add new entry with the name
6356                     admin_scale_index += 1
6357                     db_nsr_update[
6358                         "_admin.scaling-group.{}.name".format(admin_scale_index)
6359                     ] = scaling_group
6360                     db_nsr_update[
6361                         "_admin.scaling-group.{}.vnf_index".format(admin_scale_index)
6362                     ] = vnf_index
6363
6364             vca_scaling_info = []
6365             scaling_info = {"scaling_group_name": scaling_group, "vdu": [], "kdu": []}
6366             if scaling_type == "SCALE_OUT":
6367                 if "aspect-delta-details" not in scaling_descriptor:
6368                     raise LcmException(
6369                         "Aspect delta details not fount in scaling descriptor {}".format(
6370                             scaling_descriptor["name"]
6371                         )
6372                     )
6373                 # count if max-instance-count is reached
6374                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6375
6376                 scaling_info["scaling_direction"] = "OUT"
6377                 scaling_info["vdu-create"] = {}
6378                 scaling_info["kdu-create"] = {}
6379                 for delta in deltas:
6380                     for vdu_delta in delta.get("vdu-delta", {}):
6381                         vdud = get_vdu(db_vnfd, vdu_delta["id"])
6382                         # vdu_index also provides the number of instance of the targeted vdu
6383                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6384                         if vdu_index <= len(db_vnfr["vdur"]):
6385                             vdu_name_id = db_vnfr["vdur"][vdu_index - 1]["vdu-name"]
6386                             prom_job_name = (
6387                                 db_vnfr["_id"] + vdu_name_id + str(vdu_index - 1)
6388                             )
6389                             prom_job_name = prom_job_name.replace("_", "")
6390                             prom_job_name = prom_job_name.replace("-", "")
6391                         else:
6392                             prom_job_name = None
6393                         cloud_init_text = self._get_vdu_cloud_init_content(
6394                             vdud, db_vnfd
6395                         )
6396                         if cloud_init_text:
6397                             additional_params = (
6398                                 self._get_vdu_additional_params(db_vnfr, vdud["id"])
6399                                 or {}
6400                             )
6401                         cloud_init_list = []
6402
6403                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6404                         max_instance_count = 10
6405                         if vdu_profile and "max-number-of-instances" in vdu_profile:
6406                             max_instance_count = vdu_profile.get(
6407                                 "max-number-of-instances", 10
6408                             )
6409
6410                         default_instance_num = get_number_of_instances(
6411                             db_vnfd, vdud["id"]
6412                         )
6413                         instances_number = vdu_delta.get("number-of-instances", 1)
6414                         nb_scale_op += instances_number
6415
6416                         new_instance_count = nb_scale_op + default_instance_num
6417                         # Control if new count is over max and vdu count is less than max.
6418                         # Then assign new instance count
6419                         if new_instance_count > max_instance_count > vdu_count:
6420                             instances_number = new_instance_count - max_instance_count
6421                         else:
6422                             instances_number = instances_number
6423
6424                         if new_instance_count > max_instance_count:
6425                             raise LcmException(
6426                                 "reached the limit of {} (max-instance-count) "
6427                                 "scaling-out operations for the "
6428                                 "scaling-group-descriptor '{}'".format(
6429                                     nb_scale_op, scaling_group
6430                                 )
6431                             )
6432                         for x in range(vdu_delta.get("number-of-instances", 1)):
6433                             if cloud_init_text:
6434                                 # TODO Information of its own ip is not available because db_vnfr is not updated.
6435                                 additional_params["OSM"] = get_osm_params(
6436                                     db_vnfr, vdu_delta["id"], vdu_index + x
6437                                 )
6438                                 cloud_init_list.append(
6439                                     self._parse_cloud_init(
6440                                         cloud_init_text,
6441                                         additional_params,
6442                                         db_vnfd["id"],
6443                                         vdud["id"],
6444                                     )
6445                                 )
6446                                 vca_scaling_info.append(
6447                                     {
6448                                         "osm_vdu_id": vdu_delta["id"],
6449                                         "member-vnf-index": vnf_index,
6450                                         "type": "create",
6451                                         "vdu_index": vdu_index + x,
6452                                     }
6453                                 )
6454                         scaling_info["vdu-create"][vdu_delta["id"]] = instances_number
6455                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6456                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6457                         kdu_name = kdu_profile["kdu-name"]
6458                         resource_name = kdu_profile.get("resource-name", "")
6459
6460                         # Might have different kdus in the same delta
6461                         # Should have list for each kdu
6462                         if not scaling_info["kdu-create"].get(kdu_name, None):
6463                             scaling_info["kdu-create"][kdu_name] = []
6464
6465                         kdur = get_kdur(db_vnfr, kdu_name)
6466                         if kdur.get("helm-chart"):
6467                             k8s_cluster_type = "helm-chart-v3"
6468                             self.logger.debug("kdur: {}".format(kdur))
6469                         elif kdur.get("juju-bundle"):
6470                             k8s_cluster_type = "juju-bundle"
6471                         else:
6472                             raise LcmException(
6473                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6474                                 "juju-bundle. Maybe an old NBI version is running".format(
6475                                     db_vnfr["member-vnf-index-ref"], kdu_name
6476                                 )
6477                             )
6478
6479                         max_instance_count = 10
6480                         if kdu_profile and "max-number-of-instances" in kdu_profile:
6481                             max_instance_count = kdu_profile.get(
6482                                 "max-number-of-instances", 10
6483                             )
6484
6485                         nb_scale_op += kdu_delta.get("number-of-instances", 1)
6486                         deployed_kdu, _ = get_deployed_kdu(
6487                             nsr_deployed, kdu_name, vnf_index
6488                         )
6489                         if deployed_kdu is None:
6490                             raise LcmException(
6491                                 "KDU '{}' for vnf '{}' not deployed".format(
6492                                     kdu_name, vnf_index
6493                                 )
6494                             )
6495                         kdu_instance = deployed_kdu.get("kdu-instance")
6496                         instance_num = await self.k8scluster_map[
6497                             k8s_cluster_type
6498                         ].get_scale_count(
6499                             resource_name,
6500                             kdu_instance,
6501                             vca_id=vca_id,
6502                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6503                             kdu_model=deployed_kdu.get("kdu-model"),
6504                         )
6505                         kdu_replica_count = instance_num + kdu_delta.get(
6506                             "number-of-instances", 1
6507                         )
6508
6509                         # Control if new count is over max and instance_num is less than max.
6510                         # Then assign max instance number to kdu replica count
6511                         if kdu_replica_count > max_instance_count > instance_num:
6512                             kdu_replica_count = max_instance_count
6513                         if kdu_replica_count > max_instance_count:
6514                             raise LcmException(
6515                                 "reached the limit of {} (max-instance-count) "
6516                                 "scaling-out operations for the "
6517                                 "scaling-group-descriptor '{}'".format(
6518                                     instance_num, scaling_group
6519                                 )
6520                             )
6521
6522                         for x in range(kdu_delta.get("number-of-instances", 1)):
6523                             vca_scaling_info.append(
6524                                 {
6525                                     "osm_kdu_id": kdu_name,
6526                                     "member-vnf-index": vnf_index,
6527                                     "type": "create",
6528                                     "kdu_index": instance_num + x - 1,
6529                                 }
6530                             )
6531                         scaling_info["kdu-create"][kdu_name].append(
6532                             {
6533                                 "member-vnf-index": vnf_index,
6534                                 "type": "create",
6535                                 "k8s-cluster-type": k8s_cluster_type,
6536                                 "resource-name": resource_name,
6537                                 "scale": kdu_replica_count,
6538                             }
6539                         )
6540             elif scaling_type == "SCALE_IN":
6541                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6542
6543                 scaling_info["scaling_direction"] = "IN"
6544                 scaling_info["vdu-delete"] = {}
6545                 scaling_info["kdu-delete"] = {}
6546
6547                 for delta in deltas:
6548                     for vdu_delta in delta.get("vdu-delta", {}):
6549                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6550                         min_instance_count = 0
6551                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6552                         if vdu_profile and "min-number-of-instances" in vdu_profile:
6553                             min_instance_count = vdu_profile["min-number-of-instances"]
6554
6555                         default_instance_num = get_number_of_instances(
6556                             db_vnfd, vdu_delta["id"]
6557                         )
6558                         instance_num = vdu_delta.get("number-of-instances", 1)
6559                         nb_scale_op -= instance_num
6560
6561                         new_instance_count = nb_scale_op + default_instance_num
6562
6563                         if new_instance_count < min_instance_count < vdu_count:
6564                             instances_number = min_instance_count - new_instance_count
6565                         else:
6566                             instances_number = instance_num
6567
6568                         if new_instance_count < min_instance_count:
6569                             raise LcmException(
6570                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6571                                 "scaling-group-descriptor '{}'".format(
6572                                     nb_scale_op, scaling_group
6573                                 )
6574                             )
6575                         for x in range(vdu_delta.get("number-of-instances", 1)):
6576                             vca_scaling_info.append(
6577                                 {
6578                                     "osm_vdu_id": vdu_delta["id"],
6579                                     "member-vnf-index": vnf_index,
6580                                     "type": "delete",
6581                                     "vdu_index": vdu_index - 1 - x,
6582                                 }
6583                             )
6584                         scaling_info["vdu-delete"][vdu_delta["id"]] = instances_number
6585                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6586                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6587                         kdu_name = kdu_profile["kdu-name"]
6588                         resource_name = kdu_profile.get("resource-name", "")
6589
6590                         if not scaling_info["kdu-delete"].get(kdu_name, None):
6591                             scaling_info["kdu-delete"][kdu_name] = []
6592
6593                         kdur = get_kdur(db_vnfr, kdu_name)
6594                         if kdur.get("helm-chart"):
6595                             k8s_cluster_type = "helm-chart-v3"
6596                             self.logger.debug("kdur: {}".format(kdur))
6597                         elif kdur.get("juju-bundle"):
6598                             k8s_cluster_type = "juju-bundle"
6599                         else:
6600                             raise LcmException(
6601                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6602                                 "juju-bundle. Maybe an old NBI version is running".format(
6603                                     db_vnfr["member-vnf-index-ref"], kdur["kdu-name"]
6604                                 )
6605                             )
6606
6607                         min_instance_count = 0
6608                         if kdu_profile and "min-number-of-instances" in kdu_profile:
6609                             min_instance_count = kdu_profile["min-number-of-instances"]
6610
6611                         nb_scale_op -= kdu_delta.get("number-of-instances", 1)
6612                         deployed_kdu, _ = get_deployed_kdu(
6613                             nsr_deployed, kdu_name, vnf_index
6614                         )
6615                         if deployed_kdu is None:
6616                             raise LcmException(
6617                                 "KDU '{}' for vnf '{}' not deployed".format(
6618                                     kdu_name, vnf_index
6619                                 )
6620                             )
6621                         kdu_instance = deployed_kdu.get("kdu-instance")
6622                         instance_num = await self.k8scluster_map[
6623                             k8s_cluster_type
6624                         ].get_scale_count(
6625                             resource_name,
6626                             kdu_instance,
6627                             vca_id=vca_id,
6628                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6629                             kdu_model=deployed_kdu.get("kdu-model"),
6630                         )
6631                         kdu_replica_count = instance_num - kdu_delta.get(
6632                             "number-of-instances", 1
6633                         )
6634
6635                         if kdu_replica_count < min_instance_count < instance_num:
6636                             kdu_replica_count = min_instance_count
6637                         if kdu_replica_count < min_instance_count:
6638                             raise LcmException(
6639                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6640                                 "scaling-group-descriptor '{}'".format(
6641                                     instance_num, scaling_group
6642                                 )
6643                             )
6644
6645                         for x in range(kdu_delta.get("number-of-instances", 1)):
6646                             vca_scaling_info.append(
6647                                 {
6648                                     "osm_kdu_id": kdu_name,
6649                                     "member-vnf-index": vnf_index,
6650                                     "type": "delete",
6651                                     "kdu_index": instance_num - x - 1,
6652                                 }
6653                             )
6654                         scaling_info["kdu-delete"][kdu_name].append(
6655                             {
6656                                 "member-vnf-index": vnf_index,
6657                                 "type": "delete",
6658                                 "k8s-cluster-type": k8s_cluster_type,
6659                                 "resource-name": resource_name,
6660                                 "scale": kdu_replica_count,
6661                             }
6662                         )
6663
6664             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
6665             vdu_delete = copy(scaling_info.get("vdu-delete"))
6666             if scaling_info["scaling_direction"] == "IN":
6667                 for vdur in reversed(db_vnfr["vdur"]):
6668                     if vdu_delete.get(vdur["vdu-id-ref"]):
6669                         vdu_delete[vdur["vdu-id-ref"]] -= 1
6670                         scaling_info["vdu"].append(
6671                             {
6672                                 "name": vdur.get("name") or vdur.get("vdu-name"),
6673                                 "vdu_id": vdur["vdu-id-ref"],
6674                                 "interface": [],
6675                             }
6676                         )
6677                         for interface in vdur["interfaces"]:
6678                             scaling_info["vdu"][-1]["interface"].append(
6679                                 {
6680                                     "name": interface["name"],
6681                                     "ip_address": interface["ip-address"],
6682                                     "mac_address": interface.get("mac-address"),
6683                                 }
6684                             )
6685                 # vdu_delete = vdu_scaling_info.pop("vdu-delete")
6686
6687             # PRE-SCALE BEGIN
6688             step = "Executing pre-scale vnf-config-primitive"
6689             if scaling_descriptor.get("scaling-config-action"):
6690                 for scaling_config_action in scaling_descriptor[
6691                     "scaling-config-action"
6692                 ]:
6693                     if (
6694                         scaling_config_action.get("trigger") == "pre-scale-in"
6695                         and scaling_type == "SCALE_IN"
6696                     ) or (
6697                         scaling_config_action.get("trigger") == "pre-scale-out"
6698                         and scaling_type == "SCALE_OUT"
6699                     ):
6700                         vnf_config_primitive = scaling_config_action[
6701                             "vnf-config-primitive-name-ref"
6702                         ]
6703                         step = db_nslcmop_update[
6704                             "detailed-status"
6705                         ] = "executing pre-scale scaling-config-action '{}'".format(
6706                             vnf_config_primitive
6707                         )
6708
6709                         # look for primitive
6710                         for config_primitive in (
6711                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
6712                         ).get("config-primitive", ()):
6713                             if config_primitive["name"] == vnf_config_primitive:
6714                                 break
6715                         else:
6716                             raise LcmException(
6717                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
6718                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
6719                                 "primitive".format(scaling_group, vnf_config_primitive)
6720                             )
6721
6722                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
6723                         if db_vnfr.get("additionalParamsForVnf"):
6724                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
6725
6726                         scale_process = "VCA"
6727                         db_nsr_update["config-status"] = "configuring pre-scaling"
6728                         primitive_params = self._map_primitive_params(
6729                             config_primitive, {}, vnfr_params
6730                         )
6731
6732                         # Pre-scale retry check: Check if this sub-operation has been executed before
6733                         op_index = self._check_or_add_scale_suboperation(
6734                             db_nslcmop,
6735                             vnf_index,
6736                             vnf_config_primitive,
6737                             primitive_params,
6738                             "PRE-SCALE",
6739                         )
6740                         if op_index == self.SUBOPERATION_STATUS_SKIP:
6741                             # Skip sub-operation
6742                             result = "COMPLETED"
6743                             result_detail = "Done"
6744                             self.logger.debug(
6745                                 logging_text
6746                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
6747                                     vnf_config_primitive, result, result_detail
6748                                 )
6749                             )
6750                         else:
6751                             if op_index == self.SUBOPERATION_STATUS_NEW:
6752                                 # New sub-operation: Get index of this sub-operation
6753                                 op_index = (
6754                                     len(db_nslcmop.get("_admin", {}).get("operations"))
6755                                     - 1
6756                                 )
6757                                 self.logger.debug(
6758                                     logging_text
6759                                     + "vnf_config_primitive={} New sub-operation".format(
6760                                         vnf_config_primitive
6761                                     )
6762                                 )
6763                             else:
6764                                 # retry:  Get registered params for this existing sub-operation
6765                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
6766                                     op_index
6767                                 ]
6768                                 vnf_index = op.get("member_vnf_index")
6769                                 vnf_config_primitive = op.get("primitive")
6770                                 primitive_params = op.get("primitive_params")
6771                                 self.logger.debug(
6772                                     logging_text
6773                                     + "vnf_config_primitive={} Sub-operation retry".format(
6774                                         vnf_config_primitive
6775                                     )
6776                                 )
6777                             # Execute the primitive, either with new (first-time) or registered (reintent) args
6778                             ee_descriptor_id = config_primitive.get(
6779                                 "execution-environment-ref"
6780                             )
6781                             primitive_name = config_primitive.get(
6782                                 "execution-environment-primitive", vnf_config_primitive
6783                             )
6784                             ee_id, vca_type = self._look_for_deployed_vca(
6785                                 nsr_deployed["VCA"],
6786                                 member_vnf_index=vnf_index,
6787                                 vdu_id=None,
6788                                 vdu_count_index=None,
6789                                 ee_descriptor_id=ee_descriptor_id,
6790                             )
6791                             result, result_detail = await self._ns_execute_primitive(
6792                                 ee_id,
6793                                 primitive_name,
6794                                 primitive_params,
6795                                 vca_type=vca_type,
6796                                 vca_id=vca_id,
6797                             )
6798                             self.logger.debug(
6799                                 logging_text
6800                                 + "vnf_config_primitive={} Done with result {} {}".format(
6801                                     vnf_config_primitive, result, result_detail
6802                                 )
6803                             )
6804                             # Update operationState = COMPLETED | FAILED
6805                             self._update_suboperation_status(
6806                                 db_nslcmop, op_index, result, result_detail
6807                             )
6808
6809                         if result == "FAILED":
6810                             raise LcmException(result_detail)
6811                         db_nsr_update["config-status"] = old_config_status
6812                         scale_process = None
6813             # PRE-SCALE END
6814
6815             db_nsr_update[
6816                 "_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)
6817             ] = nb_scale_op
6818             db_nsr_update[
6819                 "_admin.scaling-group.{}.time".format(admin_scale_index)
6820             ] = time()
6821
6822             # SCALE-IN VCA - BEGIN
6823             if vca_scaling_info:
6824                 step = db_nslcmop_update[
6825                     "detailed-status"
6826                 ] = "Deleting the execution environments"
6827                 scale_process = "VCA"
6828                 for vca_info in vca_scaling_info:
6829                     if vca_info["type"] == "delete" and not vca_info.get("osm_kdu_id"):
6830                         member_vnf_index = str(vca_info["member-vnf-index"])
6831                         self.logger.debug(
6832                             logging_text + "vdu info: {}".format(vca_info)
6833                         )
6834                         if vca_info.get("osm_vdu_id"):
6835                             vdu_id = vca_info["osm_vdu_id"]
6836                             vdu_index = int(vca_info["vdu_index"])
6837                             stage[
6838                                 1
6839                             ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
6840                                 member_vnf_index, vdu_id, vdu_index
6841                             )
6842                         stage[2] = step = "Scaling in VCA"
6843                         self._write_op_status(op_id=nslcmop_id, stage=stage)
6844                         vca_update = db_nsr["_admin"]["deployed"]["VCA"]
6845                         config_update = db_nsr["configurationStatus"]
6846                         for vca_index, vca in enumerate(vca_update):
6847                             if (
6848                                 (vca or vca.get("ee_id"))
6849                                 and vca["member-vnf-index"] == member_vnf_index
6850                                 and vca["vdu_count_index"] == vdu_index
6851                             ):
6852                                 if vca.get("vdu_id"):
6853                                     config_descriptor = get_configuration(
6854                                         db_vnfd, vca.get("vdu_id")
6855                                     )
6856                                 elif vca.get("kdu_name"):
6857                                     config_descriptor = get_configuration(
6858                                         db_vnfd, vca.get("kdu_name")
6859                                     )
6860                                 else:
6861                                     config_descriptor = get_configuration(
6862                                         db_vnfd, db_vnfd["id"]
6863                                     )
6864                                 operation_params = (
6865                                     db_nslcmop.get("operationParams") or {}
6866                                 )
6867                                 exec_terminate_primitives = not operation_params.get(
6868                                     "skip_terminate_primitives"
6869                                 ) and vca.get("needed_terminate")
6870                                 task = asyncio.ensure_future(
6871                                     asyncio.wait_for(
6872                                         self.destroy_N2VC(
6873                                             logging_text,
6874                                             db_nslcmop,
6875                                             vca,
6876                                             config_descriptor,
6877                                             vca_index,
6878                                             destroy_ee=True,
6879                                             exec_primitives=exec_terminate_primitives,
6880                                             scaling_in=True,
6881                                             vca_id=vca_id,
6882                                         ),
6883                                         timeout=self.timeout.charm_delete,
6884                                     )
6885                                 )
6886                                 tasks_dict_info[task] = "Terminating VCA {}".format(
6887                                     vca.get("ee_id")
6888                                 )
6889                                 del vca_update[vca_index]
6890                                 del config_update[vca_index]
6891                         # wait for pending tasks of terminate primitives
6892                         if tasks_dict_info:
6893                             self.logger.debug(
6894                                 logging_text
6895                                 + "Waiting for tasks {}".format(
6896                                     list(tasks_dict_info.keys())
6897                                 )
6898                             )
6899                             error_list = await self._wait_for_tasks(
6900                                 logging_text,
6901                                 tasks_dict_info,
6902                                 min(
6903                                     self.timeout.charm_delete, self.timeout.ns_terminate
6904                                 ),
6905                                 stage,
6906                                 nslcmop_id,
6907                             )
6908                             tasks_dict_info.clear()
6909                             if error_list:
6910                                 raise LcmException("; ".join(error_list))
6911
6912                         db_vca_and_config_update = {
6913                             "_admin.deployed.VCA": vca_update,
6914                             "configurationStatus": config_update,
6915                         }
6916                         self.update_db_2(
6917                             "nsrs", db_nsr["_id"], db_vca_and_config_update
6918                         )
6919             scale_process = None
6920             # SCALE-IN VCA - END
6921
6922             # SCALE RO - BEGIN
6923             if scaling_info.get("vdu-create") or scaling_info.get("vdu-delete"):
6924                 scale_process = "RO"
6925                 if self.ro_config.ng:
6926                     await self._scale_ng_ro(
6927                         logging_text, db_nsr, db_nslcmop, db_vnfr, scaling_info, stage
6928                     )
6929             scaling_info.pop("vdu-create", None)
6930             scaling_info.pop("vdu-delete", None)
6931
6932             scale_process = None
6933             # SCALE RO - END
6934
6935             # SCALE KDU - BEGIN
6936             if scaling_info.get("kdu-create") or scaling_info.get("kdu-delete"):
6937                 scale_process = "KDU"
6938                 await self._scale_kdu(
6939                     logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
6940                 )
6941             scaling_info.pop("kdu-create", None)
6942             scaling_info.pop("kdu-delete", None)
6943
6944             scale_process = None
6945             # SCALE KDU - END
6946
6947             if db_nsr_update:
6948                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6949
6950             # SCALE-UP VCA - BEGIN
6951             if vca_scaling_info:
6952                 step = db_nslcmop_update[
6953                     "detailed-status"
6954                 ] = "Creating new execution environments"
6955                 scale_process = "VCA"
6956                 for vca_info in vca_scaling_info:
6957                     if vca_info["type"] == "create" and not vca_info.get("osm_kdu_id"):
6958                         member_vnf_index = str(vca_info["member-vnf-index"])
6959                         self.logger.debug(
6960                             logging_text + "vdu info: {}".format(vca_info)
6961                         )
6962                         vnfd_id = db_vnfr["vnfd-ref"]
6963                         if vca_info.get("osm_vdu_id"):
6964                             vdu_index = int(vca_info["vdu_index"])
6965                             deploy_params = {"OSM": get_osm_params(db_vnfr)}
6966                             if db_vnfr.get("additionalParamsForVnf"):
6967                                 deploy_params.update(
6968                                     parse_yaml_strings(
6969                                         db_vnfr["additionalParamsForVnf"].copy()
6970                                     )
6971                                 )
6972                             descriptor_config = get_configuration(
6973                                 db_vnfd, db_vnfd["id"]
6974                             )
6975                             if descriptor_config:
6976                                 vdu_id = None
6977                                 vdu_name = None
6978                                 kdu_name = None
6979                                 kdu_index = None
6980                                 self._deploy_n2vc(
6981                                     logging_text=logging_text
6982                                     + "member_vnf_index={} ".format(member_vnf_index),
6983                                     db_nsr=db_nsr,
6984                                     db_vnfr=db_vnfr,
6985                                     nslcmop_id=nslcmop_id,
6986                                     nsr_id=nsr_id,
6987                                     nsi_id=nsi_id,
6988                                     vnfd_id=vnfd_id,
6989                                     vdu_id=vdu_id,
6990                                     kdu_name=kdu_name,
6991                                     kdu_index=kdu_index,
6992                                     member_vnf_index=member_vnf_index,
6993                                     vdu_index=vdu_index,
6994                                     vdu_name=vdu_name,
6995                                     deploy_params=deploy_params,
6996                                     descriptor_config=descriptor_config,
6997                                     base_folder=base_folder,
6998                                     task_instantiation_info=tasks_dict_info,
6999                                     stage=stage,
7000                                 )
7001                             vdu_id = vca_info["osm_vdu_id"]
7002                             vdur = find_in_list(
7003                                 db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
7004                             )
7005                             descriptor_config = get_configuration(db_vnfd, vdu_id)
7006                             if vdur.get("additionalParams"):
7007                                 deploy_params_vdu = parse_yaml_strings(
7008                                     vdur["additionalParams"]
7009                                 )
7010                             else:
7011                                 deploy_params_vdu = deploy_params
7012                             deploy_params_vdu["OSM"] = get_osm_params(
7013                                 db_vnfr, vdu_id, vdu_count_index=vdu_index
7014                             )
7015                             if descriptor_config:
7016                                 vdu_name = None
7017                                 kdu_name = None
7018                                 kdu_index = None
7019                                 stage[
7020                                     1
7021                                 ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
7022                                     member_vnf_index, vdu_id, vdu_index
7023                                 )
7024                                 stage[2] = step = "Scaling out VCA"
7025                                 self._write_op_status(op_id=nslcmop_id, stage=stage)
7026                                 self._deploy_n2vc(
7027                                     logging_text=logging_text
7028                                     + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
7029                                         member_vnf_index, vdu_id, vdu_index
7030                                     ),
7031                                     db_nsr=db_nsr,
7032                                     db_vnfr=db_vnfr,
7033                                     nslcmop_id=nslcmop_id,
7034                                     nsr_id=nsr_id,
7035                                     nsi_id=nsi_id,
7036                                     vnfd_id=vnfd_id,
7037                                     vdu_id=vdu_id,
7038                                     kdu_name=kdu_name,
7039                                     member_vnf_index=member_vnf_index,
7040                                     vdu_index=vdu_index,
7041                                     kdu_index=kdu_index,
7042                                     vdu_name=vdu_name,
7043                                     deploy_params=deploy_params_vdu,
7044                                     descriptor_config=descriptor_config,
7045                                     base_folder=base_folder,
7046                                     task_instantiation_info=tasks_dict_info,
7047                                     stage=stage,
7048                                 )
7049             # SCALE-UP VCA - END
7050             scale_process = None
7051
7052             # POST-SCALE BEGIN
7053             # execute primitive service POST-SCALING
7054             step = "Executing post-scale vnf-config-primitive"
7055             if scaling_descriptor.get("scaling-config-action"):
7056                 for scaling_config_action in scaling_descriptor[
7057                     "scaling-config-action"
7058                 ]:
7059                     if (
7060                         scaling_config_action.get("trigger") == "post-scale-in"
7061                         and scaling_type == "SCALE_IN"
7062                     ) or (
7063                         scaling_config_action.get("trigger") == "post-scale-out"
7064                         and scaling_type == "SCALE_OUT"
7065                     ):
7066                         vnf_config_primitive = scaling_config_action[
7067                             "vnf-config-primitive-name-ref"
7068                         ]
7069                         step = db_nslcmop_update[
7070                             "detailed-status"
7071                         ] = "executing post-scale scaling-config-action '{}'".format(
7072                             vnf_config_primitive
7073                         )
7074
7075                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
7076                         if db_vnfr.get("additionalParamsForVnf"):
7077                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
7078
7079                         # look for primitive
7080                         for config_primitive in (
7081                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
7082                         ).get("config-primitive", ()):
7083                             if config_primitive["name"] == vnf_config_primitive:
7084                                 break
7085                         else:
7086                             raise LcmException(
7087                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-"
7088                                 "action[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:"
7089                                 "config-primitive".format(
7090                                     scaling_group, vnf_config_primitive
7091                                 )
7092                             )
7093                         scale_process = "VCA"
7094                         db_nsr_update["config-status"] = "configuring post-scaling"
7095                         primitive_params = self._map_primitive_params(
7096                             config_primitive, {}, vnfr_params
7097                         )
7098
7099                         # Post-scale retry check: Check if this sub-operation has been executed before
7100                         op_index = self._check_or_add_scale_suboperation(
7101                             db_nslcmop,
7102                             vnf_index,
7103                             vnf_config_primitive,
7104                             primitive_params,
7105                             "POST-SCALE",
7106                         )
7107                         if op_index == self.SUBOPERATION_STATUS_SKIP:
7108                             # Skip sub-operation
7109                             result = "COMPLETED"
7110                             result_detail = "Done"
7111                             self.logger.debug(
7112                                 logging_text
7113                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
7114                                     vnf_config_primitive, result, result_detail
7115                                 )
7116                             )
7117                         else:
7118                             if op_index == self.SUBOPERATION_STATUS_NEW:
7119                                 # New sub-operation: Get index of this sub-operation
7120                                 op_index = (
7121                                     len(db_nslcmop.get("_admin", {}).get("operations"))
7122                                     - 1
7123                                 )
7124                                 self.logger.debug(
7125                                     logging_text
7126                                     + "vnf_config_primitive={} New sub-operation".format(
7127                                         vnf_config_primitive
7128                                     )
7129                                 )
7130                             else:
7131                                 # retry:  Get registered params for this existing sub-operation
7132                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
7133                                     op_index
7134                                 ]
7135                                 vnf_index = op.get("member_vnf_index")
7136                                 vnf_config_primitive = op.get("primitive")
7137                                 primitive_params = op.get("primitive_params")
7138                                 self.logger.debug(
7139                                     logging_text
7140                                     + "vnf_config_primitive={} Sub-operation retry".format(
7141                                         vnf_config_primitive
7142                                     )
7143                                 )
7144                             # Execute the primitive, either with new (first-time) or registered (reintent) args
7145                             ee_descriptor_id = config_primitive.get(
7146                                 "execution-environment-ref"
7147                             )
7148                             primitive_name = config_primitive.get(
7149                                 "execution-environment-primitive", vnf_config_primitive
7150                             )
7151                             ee_id, vca_type = self._look_for_deployed_vca(
7152                                 nsr_deployed["VCA"],
7153                                 member_vnf_index=vnf_index,
7154                                 vdu_id=None,
7155                                 vdu_count_index=None,
7156                                 ee_descriptor_id=ee_descriptor_id,
7157                             )
7158                             result, result_detail = await self._ns_execute_primitive(
7159                                 ee_id,
7160                                 primitive_name,
7161                                 primitive_params,
7162                                 vca_type=vca_type,
7163                                 vca_id=vca_id,
7164                             )
7165                             self.logger.debug(
7166                                 logging_text
7167                                 + "vnf_config_primitive={} Done with result {} {}".format(
7168                                     vnf_config_primitive, result, result_detail
7169                                 )
7170                             )
7171                             # Update operationState = COMPLETED | FAILED
7172                             self._update_suboperation_status(
7173                                 db_nslcmop, op_index, result, result_detail
7174                             )
7175
7176                         if result == "FAILED":
7177                             raise LcmException(result_detail)
7178                         db_nsr_update["config-status"] = old_config_status
7179                         scale_process = None
7180             # POST-SCALE END
7181             # Check if each vnf has exporter for metric collection if so update prometheus job records
7182             if scaling_type == "SCALE_OUT":
7183                 if "exporters-endpoints" in db_vnfd.get("df")[0]:
7184                     vnfr_id = db_vnfr["id"]
7185                     db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
7186                     exporter_config = db_vnfd.get("df")[0].get("exporters-endpoints")
7187                     self.logger.debug("exporter config :{}".format(exporter_config))
7188                     artifact_path = "{}/{}/{}".format(
7189                         base_folder["folder"],
7190                         base_folder["pkg-dir"],
7191                         "exporter-endpoint",
7192                     )
7193                     ee_id = None
7194                     ee_config_descriptor = exporter_config
7195                     rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
7196                         logging_text,
7197                         nsr_id,
7198                         vnfr_id,
7199                         vdu_id=db_vnfr["vdur"][-1]["vdu-id-ref"],
7200                         vdu_index=db_vnfr["vdur"][-1]["count-index"],
7201                         user=None,
7202                         pub_key=None,
7203                     )
7204                     self.logger.debug("rw_mgmt_ip:{}".format(rw_mgmt_ip))
7205                     self.logger.debug("Artifact_path:{}".format(artifact_path))
7206                     vdu_id_for_prom = None
7207                     vdu_index_for_prom = None
7208                     for x in get_iterable(db_vnfr, "vdur"):
7209                         vdu_id_for_prom = x.get("vdu-id-ref")
7210                         vdu_index_for_prom = x.get("count-index")
7211                     vnfr_id = vnfr_id + vdu_id + str(vdu_index)
7212                     vnfr_id = vnfr_id.replace("_", "")
7213                     prometheus_jobs = await self.extract_prometheus_scrape_jobs(
7214                         ee_id=ee_id,
7215                         artifact_path=artifact_path,
7216                         ee_config_descriptor=ee_config_descriptor,
7217                         vnfr_id=vnfr_id,
7218                         nsr_id=nsr_id,
7219                         target_ip=rw_mgmt_ip,
7220                         element_type="VDU",
7221                         vdu_id=vdu_id_for_prom,
7222                         vdu_index=vdu_index_for_prom,
7223                     )
7224
7225                     self.logger.debug("Prometheus job:{}".format(prometheus_jobs))
7226                     if prometheus_jobs:
7227                         db_nsr_update[
7228                             "_admin.deployed.prometheus_jobs"
7229                         ] = prometheus_jobs
7230                         self.update_db_2(
7231                             "nsrs",
7232                             nsr_id,
7233                             db_nsr_update,
7234                         )
7235
7236                         for job in prometheus_jobs:
7237                             self.db.set_one(
7238                                 "prometheus_jobs",
7239                                 {"job_name": ""},
7240                                 job,
7241                                 upsert=True,
7242                                 fail_on_empty=False,
7243                             )
7244             db_nsr_update[
7245                 "detailed-status"
7246             ] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
7247             db_nsr_update["operational-status"] = (
7248                 "running"
7249                 if old_operational_status == "failed"
7250                 else old_operational_status
7251             )
7252             db_nsr_update["config-status"] = old_config_status
7253             return
7254         except (
7255             ROclient.ROClientException,
7256             DbException,
7257             LcmException,
7258             NgRoException,
7259         ) as e:
7260             self.logger.error(logging_text + "Exit Exception {}".format(e))
7261             exc = e
7262         except asyncio.CancelledError:
7263             self.logger.error(
7264                 logging_text + "Cancelled Exception while '{}'".format(step)
7265             )
7266             exc = "Operation was cancelled"
7267         except Exception as e:
7268             exc = traceback.format_exc()
7269             self.logger.critical(
7270                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
7271                 exc_info=True,
7272             )
7273         finally:
7274             error_list = list()
7275             if exc:
7276                 error_list.append(str(exc))
7277             self._write_ns_status(
7278                 nsr_id=nsr_id,
7279                 ns_state=None,
7280                 current_operation="IDLE",
7281                 current_operation_id=None,
7282             )
7283             try:
7284                 if tasks_dict_info:
7285                     stage[1] = "Waiting for instantiate pending tasks."
7286                     self.logger.debug(logging_text + stage[1])
7287                     exc = await self._wait_for_tasks(
7288                         logging_text,
7289                         tasks_dict_info,
7290                         self.timeout.ns_deploy,
7291                         stage,
7292                         nslcmop_id,
7293                         nsr_id=nsr_id,
7294                     )
7295             except asyncio.CancelledError:
7296                 error_list.append("Cancelled")
7297                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
7298                 await self._wait_for_tasks(
7299                     logging_text,
7300                     tasks_dict_info,
7301                     self.timeout.ns_deploy,
7302                     stage,
7303                     nslcmop_id,
7304                     nsr_id=nsr_id,
7305                 )
7306             if error_list:
7307                 error_detail = "; ".join(error_list)
7308                 db_nslcmop_update[
7309                     "detailed-status"
7310                 ] = error_description_nslcmop = "FAILED {}: {}".format(
7311                     step, error_detail
7312                 )
7313                 nslcmop_operation_state = "FAILED"
7314                 if db_nsr:
7315                     db_nsr_update["operational-status"] = old_operational_status
7316                     db_nsr_update["config-status"] = old_config_status
7317                     db_nsr_update["detailed-status"] = ""
7318                     if scale_process:
7319                         if "VCA" in scale_process:
7320                             db_nsr_update["config-status"] = "failed"
7321                         if "RO" in scale_process:
7322                             db_nsr_update["operational-status"] = "failed"
7323                         db_nsr_update[
7324                             "detailed-status"
7325                         ] = "FAILED scaling nslcmop={} {}: {}".format(
7326                             nslcmop_id, step, error_detail
7327                         )
7328             else:
7329                 error_description_nslcmop = None
7330                 nslcmop_operation_state = "COMPLETED"
7331                 db_nslcmop_update["detailed-status"] = "Done"
7332                 if scaling_type == "SCALE_IN" and prom_job_name is not None:
7333                     self.db.del_one(
7334                         "prometheus_jobs",
7335                         {"job_name": prom_job_name},
7336                         fail_on_empty=False,
7337                     )
7338
7339             self._write_op_status(
7340                 op_id=nslcmop_id,
7341                 stage="",
7342                 error_message=error_description_nslcmop,
7343                 operation_state=nslcmop_operation_state,
7344                 other_update=db_nslcmop_update,
7345             )
7346             if db_nsr:
7347                 self._write_ns_status(
7348                     nsr_id=nsr_id,
7349                     ns_state=None,
7350                     current_operation="IDLE",
7351                     current_operation_id=None,
7352                     other_update=db_nsr_update,
7353                 )
7354
7355             if nslcmop_operation_state:
7356                 try:
7357                     msg = {
7358                         "nsr_id": nsr_id,
7359                         "nslcmop_id": nslcmop_id,
7360                         "operationState": nslcmop_operation_state,
7361                     }
7362                     await self.msg.aiowrite("ns", "scaled", msg)
7363                 except Exception as e:
7364                     self.logger.error(
7365                         logging_text + "kafka_write notification Exception {}".format(e)
7366                     )
7367             self.logger.debug(logging_text + "Exit")
7368             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")
7369
7370     async def _scale_kdu(
7371         self, logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
7372     ):
7373         _scaling_info = scaling_info.get("kdu-create") or scaling_info.get("kdu-delete")
7374         for kdu_name in _scaling_info:
7375             for kdu_scaling_info in _scaling_info[kdu_name]:
7376                 deployed_kdu, index = get_deployed_kdu(
7377                     nsr_deployed, kdu_name, kdu_scaling_info["member-vnf-index"]
7378                 )
7379                 cluster_uuid = deployed_kdu["k8scluster-uuid"]
7380                 kdu_instance = deployed_kdu["kdu-instance"]
7381                 kdu_model = deployed_kdu.get("kdu-model")
7382                 scale = int(kdu_scaling_info["scale"])
7383                 k8s_cluster_type = kdu_scaling_info["k8s-cluster-type"]
7384
7385                 db_dict = {
7386                     "collection": "nsrs",
7387                     "filter": {"_id": nsr_id},
7388                     "path": "_admin.deployed.K8s.{}".format(index),
7389                 }
7390
7391                 step = "scaling application {}".format(
7392                     kdu_scaling_info["resource-name"]
7393                 )
7394                 self.logger.debug(logging_text + step)
7395
7396                 if kdu_scaling_info["type"] == "delete":
7397                     kdu_config = get_configuration(db_vnfd, kdu_name)
7398                     if (
7399                         kdu_config
7400                         and kdu_config.get("terminate-config-primitive")
7401                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7402                     ):
7403                         terminate_config_primitive_list = kdu_config.get(
7404                             "terminate-config-primitive"
7405                         )
7406                         terminate_config_primitive_list.sort(
7407                             key=lambda val: int(val["seq"])
7408                         )
7409
7410                         for (
7411                             terminate_config_primitive
7412                         ) in terminate_config_primitive_list:
7413                             primitive_params_ = self._map_primitive_params(
7414                                 terminate_config_primitive, {}, {}
7415                             )
7416                             step = "execute terminate config primitive"
7417                             self.logger.debug(logging_text + step)
7418                             await asyncio.wait_for(
7419                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7420                                     cluster_uuid=cluster_uuid,
7421                                     kdu_instance=kdu_instance,
7422                                     primitive_name=terminate_config_primitive["name"],
7423                                     params=primitive_params_,
7424                                     db_dict=db_dict,
7425                                     total_timeout=self.timeout.primitive,
7426                                     vca_id=vca_id,
7427                                 ),
7428                                 timeout=self.timeout.primitive
7429                                 * self.timeout.primitive_outer_factor,
7430                             )
7431
7432                 await asyncio.wait_for(
7433                     self.k8scluster_map[k8s_cluster_type].scale(
7434                         kdu_instance=kdu_instance,
7435                         scale=scale,
7436                         resource_name=kdu_scaling_info["resource-name"],
7437                         total_timeout=self.timeout.scale_on_error,
7438                         vca_id=vca_id,
7439                         cluster_uuid=cluster_uuid,
7440                         kdu_model=kdu_model,
7441                         atomic=True,
7442                         db_dict=db_dict,
7443                     ),
7444                     timeout=self.timeout.scale_on_error
7445                     * self.timeout.scale_on_error_outer_factor,
7446                 )
7447
7448                 if kdu_scaling_info["type"] == "create":
7449                     kdu_config = get_configuration(db_vnfd, kdu_name)
7450                     if (
7451                         kdu_config
7452                         and kdu_config.get("initial-config-primitive")
7453                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7454                     ):
7455                         initial_config_primitive_list = kdu_config.get(
7456                             "initial-config-primitive"
7457                         )
7458                         initial_config_primitive_list.sort(
7459                             key=lambda val: int(val["seq"])
7460                         )
7461
7462                         for initial_config_primitive in initial_config_primitive_list:
7463                             primitive_params_ = self._map_primitive_params(
7464                                 initial_config_primitive, {}, {}
7465                             )
7466                             step = "execute initial config primitive"
7467                             self.logger.debug(logging_text + step)
7468                             await asyncio.wait_for(
7469                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7470                                     cluster_uuid=cluster_uuid,
7471                                     kdu_instance=kdu_instance,
7472                                     primitive_name=initial_config_primitive["name"],
7473                                     params=primitive_params_,
7474                                     db_dict=db_dict,
7475                                     vca_id=vca_id,
7476                                 ),
7477                                 timeout=600,
7478                             )
7479
7480     async def _scale_ng_ro(
7481         self, logging_text, db_nsr, db_nslcmop, db_vnfr, vdu_scaling_info, stage
7482     ):
7483         nsr_id = db_nslcmop["nsInstanceId"]
7484         db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7485         db_vnfrs = {}
7486
7487         # read from db: vnfd's for every vnf
7488         db_vnfds = []
7489
7490         # for each vnf in ns, read vnfd
7491         for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
7492             db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
7493             vnfd_id = vnfr["vnfd-id"]  # vnfd uuid for this vnf
7494             # if we haven't this vnfd, read it from db
7495             if not find_in_list(db_vnfds, lambda a_vnfd: a_vnfd["id"] == vnfd_id):
7496                 # read from db
7497                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7498                 db_vnfds.append(vnfd)
7499         n2vc_key = self.n2vc.get_public_key()
7500         n2vc_key_list = [n2vc_key]
7501         self.scale_vnfr(
7502             db_vnfr,
7503             vdu_scaling_info.get("vdu-create"),
7504             vdu_scaling_info.get("vdu-delete"),
7505             mark_delete=True,
7506         )
7507         # db_vnfr has been updated, update db_vnfrs to use it
7508         db_vnfrs[db_vnfr["member-vnf-index-ref"]] = db_vnfr
7509         await self._instantiate_ng_ro(
7510             logging_text,
7511             nsr_id,
7512             db_nsd,
7513             db_nsr,
7514             db_nslcmop,
7515             db_vnfrs,
7516             db_vnfds,
7517             n2vc_key_list,
7518             stage=stage,
7519             start_deploy=time(),
7520             timeout_ns_deploy=self.timeout.ns_deploy,
7521         )
7522         if vdu_scaling_info.get("vdu-delete"):
7523             self.scale_vnfr(
7524                 db_vnfr, None, vdu_scaling_info["vdu-delete"], mark_delete=False
7525             )
7526
7527     async def extract_prometheus_scrape_jobs(
7528         self,
7529         ee_id: str,
7530         artifact_path: str,
7531         ee_config_descriptor: dict,
7532         vnfr_id: str,
7533         nsr_id: str,
7534         target_ip: str,
7535         element_type: str,
7536         vnf_member_index: str = "",
7537         vdu_id: str = "",
7538         vdu_index: int = None,
7539         kdu_name: str = "",
7540         kdu_index: int = None,
7541     ) -> dict:
7542         """Method to extract prometheus scrape jobs from EE's Prometheus template job file
7543             This method will wait until the corresponding VDU or KDU is fully instantiated
7544
7545         Args:
7546             ee_id (str): Execution Environment ID
7547             artifact_path (str): Path where the EE's content is (including the Prometheus template file)
7548             ee_config_descriptor (dict): Execution Environment's configuration descriptor
7549             vnfr_id (str): VNFR ID where this EE applies
7550             nsr_id (str): NSR ID where this EE applies
7551             target_ip (str): VDU/KDU instance IP address
7552             element_type (str): NS or VNF or VDU or KDU
7553             vnf_member_index (str, optional): VNF index where this EE applies. Defaults to "".
7554             vdu_id (str, optional): VDU ID where this EE applies. Defaults to "".
7555             vdu_index (int, optional): VDU index where this EE applies. Defaults to None.
7556             kdu_name (str, optional): KDU name where this EE applies. Defaults to "".
7557             kdu_index (int, optional): KDU index where this EE applies. Defaults to None.
7558
7559         Raises:
7560             LcmException: When the VDU or KDU instance was not found in an hour
7561
7562         Returns:
7563             _type_: Prometheus jobs
7564         """
7565         # default the vdur and kdur names to an empty string, to avoid any later
7566         # problem with Prometheus when the element type is not VDU or KDU
7567         vdur_name = ""
7568         kdur_name = ""
7569
7570         # look if exist a file called 'prometheus*.j2' and
7571         artifact_content = self.fs.dir_ls(artifact_path)
7572         job_file = next(
7573             (
7574                 f
7575                 for f in artifact_content
7576                 if f.startswith("prometheus") and f.endswith(".j2")
7577             ),
7578             None,
7579         )
7580         if not job_file:
7581             return
7582         self.logger.debug("Artifact path{}".format(artifact_path))
7583         self.logger.debug("job file{}".format(job_file))
7584         with self.fs.file_open((artifact_path, job_file), "r") as f:
7585             job_data = f.read()
7586
7587         # obtain the VDUR or KDUR, if the element type is VDU or KDU
7588         if element_type in ("VDU", "KDU"):
7589             for _ in range(360):
7590                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
7591                 if vdu_id and vdu_index is not None:
7592                     vdur = next(
7593                         (
7594                             x
7595                             for x in get_iterable(db_vnfr, "vdur")
7596                             if (
7597                                 x.get("vdu-id-ref") == vdu_id
7598                                 and x.get("count-index") == vdu_index
7599                             )
7600                         ),
7601                         {},
7602                     )
7603                     if vdur.get("name"):
7604                         vdur_name = vdur.get("name")
7605                         break
7606                 if kdu_name and kdu_index is not None:
7607                     kdur = next(
7608                         (
7609                             x
7610                             for x in get_iterable(db_vnfr, "kdur")
7611                             if (
7612                                 x.get("kdu-name") == kdu_name
7613                                 and x.get("count-index") == kdu_index
7614                             )
7615                         ),
7616                         {},
7617                     )
7618                     if kdur.get("name"):
7619                         kdur_name = kdur.get("name")
7620                         break
7621
7622                 await asyncio.sleep(10)
7623             else:
7624                 if vdu_id and vdu_index is not None:
7625                     raise LcmException(
7626                         f"Timeout waiting VDU with name={vdu_id} and index={vdu_index} to be intantiated"
7627                     )
7628                 if kdu_name and kdu_index is not None:
7629                     raise LcmException(
7630                         f"Timeout waiting KDU with name={kdu_name} and index={kdu_index} to be intantiated"
7631                     )
7632
7633         if ee_id is not None:
7634             _, namespace, helm_id = get_ee_id_parts(
7635                 ee_id
7636             )  # get namespace and EE gRPC service name
7637             host_name = f'{helm_id}-{ee_config_descriptor["metric-service"]}.{namespace}.svc'  # svc_name.namespace.svc
7638             host_port = "80"
7639             vnfr_id = vnfr_id.replace("-", "")
7640             variables = {
7641                 "JOB_NAME": vnfr_id,
7642                 "TARGET_IP": target_ip,
7643                 "EXPORTER_POD_IP": host_name,
7644                 "EXPORTER_POD_PORT": host_port,
7645                 "NSR_ID": nsr_id,
7646                 "VNF_MEMBER_INDEX": vnf_member_index,
7647                 "VDUR_NAME": vdur_name,
7648                 "KDUR_NAME": kdur_name,
7649                 "ELEMENT_TYPE": element_type,
7650             }
7651         else:
7652             metric_path = ee_config_descriptor["metric-path"]
7653             target_port = ee_config_descriptor["metric-port"]
7654             vnfr_id = vnfr_id.replace("-", "")
7655             variables = {
7656                 "JOB_NAME": vnfr_id,
7657                 "TARGET_IP": target_ip,
7658                 "TARGET_PORT": target_port,
7659                 "METRIC_PATH": metric_path,
7660             }
7661
7662         job_list = parse_job(job_data, variables)
7663         # ensure job_name is using the vnfr_id. Adding the metadata nsr_id
7664         for job in job_list:
7665             if (
7666                 not isinstance(job.get("job_name"), str)
7667                 or vnfr_id not in job["job_name"]
7668             ):
7669                 job["job_name"] = vnfr_id + "_" + str(SystemRandom().randint(1, 10000))
7670             job["nsr_id"] = nsr_id
7671             job["vnfr_id"] = vnfr_id
7672         return job_list
7673
7674     async def rebuild_start_stop(
7675         self, nsr_id, nslcmop_id, vnf_id, additional_param, operation_type
7676     ):
7677         logging_text = "Task ns={} {}={} ".format(nsr_id, operation_type, nslcmop_id)
7678         self.logger.info(logging_text + "Enter")
7679         stage = ["Preparing the environment", ""]
7680         # database nsrs record
7681         db_nsr_update = {}
7682         vdu_vim_name = None
7683         vim_vm_id = None
7684         # in case of error, indicates what part of scale was failed to put nsr at error status
7685         start_deploy = time()
7686         try:
7687             db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_id})
7688             vim_account_id = db_vnfr.get("vim-account-id")
7689             vim_info_key = "vim:" + vim_account_id
7690             vdu_id = additional_param["vdu_id"]
7691             vdurs = [item for item in db_vnfr["vdur"] if item["vdu-id-ref"] == vdu_id]
7692             vdur = find_in_list(
7693                 vdurs, lambda vdu: vdu["count-index"] == additional_param["count-index"]
7694             )
7695             if vdur:
7696                 vdu_vim_name = vdur["name"]
7697                 vim_vm_id = vdur["vim_info"][vim_info_key]["vim_id"]
7698                 target_vim, _ = next(k_v for k_v in vdur["vim_info"].items())
7699             else:
7700                 raise LcmException("Target vdu is not found")
7701             self.logger.info("vdu_vim_name >> {} ".format(vdu_vim_name))
7702             # wait for any previous tasks in process
7703             stage[1] = "Waiting for previous operations to terminate"
7704             self.logger.info(stage[1])
7705             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7706
7707             stage[1] = "Reading from database."
7708             self.logger.info(stage[1])
7709             self._write_ns_status(
7710                 nsr_id=nsr_id,
7711                 ns_state=None,
7712                 current_operation=operation_type.upper(),
7713                 current_operation_id=nslcmop_id,
7714             )
7715             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
7716
7717             # read from db: ns
7718             stage[1] = "Getting nsr={} from db.".format(nsr_id)
7719             db_nsr_update["operational-status"] = operation_type
7720             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7721             # Payload for RO
7722             desc = {
7723                 operation_type: {
7724                     "vim_vm_id": vim_vm_id,
7725                     "vnf_id": vnf_id,
7726                     "vdu_index": additional_param["count-index"],
7727                     "vdu_id": vdur["id"],
7728                     "target_vim": target_vim,
7729                     "vim_account_id": vim_account_id,
7730                 }
7731             }
7732             stage[1] = "Sending rebuild request to RO... {}".format(desc)
7733             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
7734             self.logger.info("ro nsr id: {}".format(nsr_id))
7735             result_dict = await self.RO.operate(nsr_id, desc, operation_type)
7736             self.logger.info("response from RO: {}".format(result_dict))
7737             action_id = result_dict["action_id"]
7738             await self._wait_ng_ro(
7739                 nsr_id,
7740                 action_id,
7741                 nslcmop_id,
7742                 start_deploy,
7743                 self.timeout.operate,
7744                 None,
7745                 "start_stop_rebuild",
7746             )
7747             return "COMPLETED", "Done"
7748         except (ROclient.ROClientException, DbException, LcmException) as e:
7749             self.logger.error("Exit Exception {}".format(e))
7750             exc = e
7751         except asyncio.CancelledError:
7752             self.logger.error("Cancelled Exception while '{}'".format(stage))
7753             exc = "Operation was cancelled"
7754         except Exception as e:
7755             exc = traceback.format_exc()
7756             self.logger.critical(
7757                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
7758             )
7759             return "FAILED", "Error in operate VNF {}".format(exc)
7760
7761     async def migrate(self, nsr_id, nslcmop_id):
7762         """
7763         Migrate VNFs and VDUs instances in a NS
7764
7765         :param: nsr_id: NS Instance ID
7766         :param: nslcmop_id: nslcmop ID of migrate
7767
7768         """
7769         # Try to lock HA task here
7770         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7771         if not task_is_locked_by_me:
7772             return
7773         logging_text = "Task ns={} migrate ".format(nsr_id)
7774         self.logger.debug(logging_text + "Enter")
7775         # get all needed from database
7776         db_nslcmop = None
7777         db_nslcmop_update = {}
7778         nslcmop_operation_state = None
7779         db_nsr_update = {}
7780         target = {}
7781         exc = None
7782         # in case of error, indicates what part of scale was failed to put nsr at error status
7783         start_deploy = time()
7784
7785         try:
7786             # wait for any previous tasks in process
7787             step = "Waiting for previous operations to terminate"
7788             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7789
7790             self._write_ns_status(
7791                 nsr_id=nsr_id,
7792                 ns_state=None,
7793                 current_operation="MIGRATING",
7794                 current_operation_id=nslcmop_id,
7795             )
7796             step = "Getting nslcmop from database"
7797             self.logger.debug(
7798                 step + " after having waited for previous tasks to be completed"
7799             )
7800             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7801             migrate_params = db_nslcmop.get("operationParams")
7802
7803             target = {}
7804             target.update(migrate_params)
7805             desc = await self.RO.migrate(nsr_id, target)
7806             self.logger.debug("RO return > {}".format(desc))
7807             action_id = desc["action_id"]
7808             await self._wait_ng_ro(
7809                 nsr_id,
7810                 action_id,
7811                 nslcmop_id,
7812                 start_deploy,
7813                 self.timeout.migrate,
7814                 operation="migrate",
7815             )
7816         except (ROclient.ROClientException, DbException, LcmException) as e:
7817             self.logger.error("Exit Exception {}".format(e))
7818             exc = e
7819         except asyncio.CancelledError:
7820             self.logger.error("Cancelled Exception while '{}'".format(step))
7821             exc = "Operation was cancelled"
7822         except Exception as e:
7823             exc = traceback.format_exc()
7824             self.logger.critical(
7825                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
7826             )
7827         finally:
7828             self._write_ns_status(
7829                 nsr_id=nsr_id,
7830                 ns_state=None,
7831                 current_operation="IDLE",
7832                 current_operation_id=None,
7833             )
7834             if exc:
7835                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
7836                 nslcmop_operation_state = "FAILED"
7837             else:
7838                 nslcmop_operation_state = "COMPLETED"
7839                 db_nslcmop_update["detailed-status"] = "Done"
7840                 db_nsr_update["detailed-status"] = "Done"
7841
7842             self._write_op_status(
7843                 op_id=nslcmop_id,
7844                 stage="",
7845                 error_message="",
7846                 operation_state=nslcmop_operation_state,
7847                 other_update=db_nslcmop_update,
7848             )
7849             if nslcmop_operation_state:
7850                 try:
7851                     msg = {
7852                         "nsr_id": nsr_id,
7853                         "nslcmop_id": nslcmop_id,
7854                         "operationState": nslcmop_operation_state,
7855                     }
7856                     await self.msg.aiowrite("ns", "migrated", msg)
7857                 except Exception as e:
7858                     self.logger.error(
7859                         logging_text + "kafka_write notification Exception {}".format(e)
7860                     )
7861             self.logger.debug(logging_text + "Exit")
7862             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_migrate")
7863
7864     async def heal(self, nsr_id, nslcmop_id):
7865         """
7866         Heal NS
7867
7868         :param nsr_id: ns instance to heal
7869         :param nslcmop_id: operation to run
7870         :return:
7871         """
7872
7873         # Try to lock HA task here
7874         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7875         if not task_is_locked_by_me:
7876             return
7877
7878         logging_text = "Task ns={} heal={} ".format(nsr_id, nslcmop_id)
7879         stage = ["", "", ""]
7880         tasks_dict_info = {}
7881         # ^ stage, step, VIM progress
7882         self.logger.debug(logging_text + "Enter")
7883         # get all needed from database
7884         db_nsr = None
7885         db_nslcmop_update = {}
7886         db_nsr_update = {}
7887         db_vnfrs = {}  # vnf's info indexed by _id
7888         exc = None
7889         old_operational_status = ""
7890         old_config_status = ""
7891         nsi_id = None
7892         try:
7893             # wait for any previous tasks in process
7894             step = "Waiting for previous operations to terminate"
7895             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7896             self._write_ns_status(
7897                 nsr_id=nsr_id,
7898                 ns_state=None,
7899                 current_operation="HEALING",
7900                 current_operation_id=nslcmop_id,
7901             )
7902
7903             step = "Getting nslcmop from database"
7904             self.logger.debug(
7905                 step + " after having waited for previous tasks to be completed"
7906             )
7907             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7908
7909             step = "Getting nsr from database"
7910             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
7911             old_operational_status = db_nsr["operational-status"]
7912             old_config_status = db_nsr["config-status"]
7913
7914             db_nsr_update = {
7915                 "operational-status": "healing",
7916                 "_admin.deployed.RO.operational-status": "healing",
7917             }
7918             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7919
7920             step = "Sending heal order to VIM"
7921             await self.heal_RO(
7922                 logging_text=logging_text,
7923                 nsr_id=nsr_id,
7924                 db_nslcmop=db_nslcmop,
7925                 stage=stage,
7926             )
7927             # VCA tasks
7928             # read from db: nsd
7929             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
7930             self.logger.debug(logging_text + stage[1])
7931             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7932             self.fs.sync(db_nsr["nsd-id"])
7933             db_nsr["nsd"] = nsd
7934             # read from db: vnfr's of this ns
7935             step = "Getting vnfrs from db"
7936             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
7937             for vnfr in db_vnfrs_list:
7938                 db_vnfrs[vnfr["_id"]] = vnfr
7939             self.logger.debug("ns.heal db_vnfrs={}".format(db_vnfrs))
7940
7941             # Check for each target VNF
7942             target_list = db_nslcmop.get("operationParams", {}).get("healVnfData", {})
7943             for target_vnf in target_list:
7944                 # Find this VNF in the list from DB
7945                 vnfr_id = target_vnf.get("vnfInstanceId", None)
7946                 if vnfr_id:
7947                     db_vnfr = db_vnfrs[vnfr_id]
7948                     vnfd_id = db_vnfr.get("vnfd-id")
7949                     vnfd_ref = db_vnfr.get("vnfd-ref")
7950                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7951                     base_folder = vnfd["_admin"]["storage"]
7952                     vdu_id = None
7953                     vdu_index = 0
7954                     vdu_name = None
7955                     kdu_name = None
7956                     nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
7957                     member_vnf_index = db_vnfr.get("member-vnf-index-ref")
7958
7959                     # Check each target VDU and deploy N2VC
7960                     target_vdu_list = target_vnf.get("additionalParams", {}).get(
7961                         "vdu", []
7962                     )
7963                     if not target_vdu_list:
7964                         # Codigo nuevo para crear diccionario
7965                         target_vdu_list = []
7966                         for existing_vdu in db_vnfr.get("vdur"):
7967                             vdu_name = existing_vdu.get("vdu-name", None)
7968                             vdu_index = existing_vdu.get("count-index", 0)
7969                             vdu_run_day1 = target_vnf.get("additionalParams", {}).get(
7970                                 "run-day1", False
7971                             )
7972                             vdu_to_be_healed = {
7973                                 "vdu-id": vdu_name,
7974                                 "count-index": vdu_index,
7975                                 "run-day1": vdu_run_day1,
7976                             }
7977                             target_vdu_list.append(vdu_to_be_healed)
7978                     for target_vdu in target_vdu_list:
7979                         deploy_params_vdu = target_vdu
7980                         # Set run-day1 vnf level value if not vdu level value exists
7981                         if not deploy_params_vdu.get("run-day1") and target_vnf.get(
7982                             "additionalParams", {}
7983                         ).get("run-day1"):
7984                             deploy_params_vdu["run-day1"] = target_vnf[
7985                                 "additionalParams"
7986                             ].get("run-day1")
7987                         vdu_name = target_vdu.get("vdu-id", None)
7988                         # TODO: Get vdu_id from vdud.
7989                         vdu_id = vdu_name
7990                         # For multi instance VDU count-index is mandatory
7991                         # For single session VDU count-indes is 0
7992                         vdu_index = target_vdu.get("count-index", 0)
7993
7994                         # n2vc_redesign STEP 3 to 6 Deploy N2VC
7995                         stage[1] = "Deploying Execution Environments."
7996                         self.logger.debug(logging_text + stage[1])
7997
7998                         # VNF Level charm. Normal case when proxy charms.
7999                         # If target instance is management machine continue with actions: recreate EE for native charms or reinject juju key for proxy charms.
8000                         descriptor_config = get_configuration(vnfd, vnfd_ref)
8001                         if descriptor_config:
8002                             # Continue if healed machine is management machine
8003                             vnf_ip_address = db_vnfr.get("ip-address")
8004                             target_instance = None
8005                             for instance in db_vnfr.get("vdur", None):
8006                                 if (
8007                                     instance["vdu-name"] == vdu_name
8008                                     and instance["count-index"] == vdu_index
8009                                 ):
8010                                     target_instance = instance
8011                                     break
8012                             if vnf_ip_address == target_instance.get("ip-address"):
8013                                 self._heal_n2vc(
8014                                     logging_text=logging_text
8015                                     + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
8016                                         member_vnf_index, vdu_name, vdu_index
8017                                     ),
8018                                     db_nsr=db_nsr,
8019                                     db_vnfr=db_vnfr,
8020                                     nslcmop_id=nslcmop_id,
8021                                     nsr_id=nsr_id,
8022                                     nsi_id=nsi_id,
8023                                     vnfd_id=vnfd_ref,
8024                                     vdu_id=None,
8025                                     kdu_name=None,
8026                                     member_vnf_index=member_vnf_index,
8027                                     vdu_index=0,
8028                                     vdu_name=None,
8029                                     deploy_params=deploy_params_vdu,
8030                                     descriptor_config=descriptor_config,
8031                                     base_folder=base_folder,
8032                                     task_instantiation_info=tasks_dict_info,
8033                                     stage=stage,
8034                                 )
8035
8036                         # VDU Level charm. Normal case with native charms.
8037                         descriptor_config = get_configuration(vnfd, vdu_name)
8038                         if descriptor_config:
8039                             self._heal_n2vc(
8040                                 logging_text=logging_text
8041                                 + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
8042                                     member_vnf_index, vdu_name, vdu_index
8043                                 ),
8044                                 db_nsr=db_nsr,
8045                                 db_vnfr=db_vnfr,
8046                                 nslcmop_id=nslcmop_id,
8047                                 nsr_id=nsr_id,
8048                                 nsi_id=nsi_id,
8049                                 vnfd_id=vnfd_ref,
8050                                 vdu_id=vdu_id,
8051                                 kdu_name=kdu_name,
8052                                 member_vnf_index=member_vnf_index,
8053                                 vdu_index=vdu_index,
8054                                 vdu_name=vdu_name,
8055                                 deploy_params=deploy_params_vdu,
8056                                 descriptor_config=descriptor_config,
8057                                 base_folder=base_folder,
8058                                 task_instantiation_info=tasks_dict_info,
8059                                 stage=stage,
8060                             )
8061         except (
8062             ROclient.ROClientException,
8063             DbException,
8064             LcmException,
8065             NgRoException,
8066         ) as e:
8067             self.logger.error(logging_text + "Exit Exception {}".format(e))
8068             exc = e
8069         except asyncio.CancelledError:
8070             self.logger.error(
8071                 logging_text + "Cancelled Exception while '{}'".format(step)
8072             )
8073             exc = "Operation was cancelled"
8074         except Exception as e:
8075             exc = traceback.format_exc()
8076             self.logger.critical(
8077                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
8078                 exc_info=True,
8079             )
8080         finally:
8081             error_list = list()
8082             if exc:
8083                 error_list.append(str(exc))
8084             try:
8085                 if tasks_dict_info:
8086                     stage[1] = "Waiting for healing pending tasks."
8087                     self.logger.debug(logging_text + stage[1])
8088                     exc = await self._wait_for_tasks(
8089                         logging_text,
8090                         tasks_dict_info,
8091                         self.timeout.ns_deploy,
8092                         stage,
8093                         nslcmop_id,
8094                         nsr_id=nsr_id,
8095                     )
8096             except asyncio.CancelledError:
8097                 error_list.append("Cancelled")
8098                 await self._cancel_pending_tasks(logging_text, tasks_dict_info)
8099                 await self._wait_for_tasks(
8100                     logging_text,
8101                     tasks_dict_info,
8102                     self.timeout.ns_deploy,
8103                     stage,
8104                     nslcmop_id,
8105                     nsr_id=nsr_id,
8106                 )
8107             if error_list:
8108                 error_detail = "; ".join(error_list)
8109                 db_nslcmop_update[
8110                     "detailed-status"
8111                 ] = error_description_nslcmop = "FAILED {}: {}".format(
8112                     step, error_detail
8113                 )
8114                 nslcmop_operation_state = "FAILED"
8115                 if db_nsr:
8116                     db_nsr_update["operational-status"] = old_operational_status
8117                     db_nsr_update["config-status"] = old_config_status
8118                     db_nsr_update[
8119                         "detailed-status"
8120                     ] = "FAILED healing nslcmop={} {}: {}".format(
8121                         nslcmop_id, step, error_detail
8122                     )
8123                     for task, task_name in tasks_dict_info.items():
8124                         if not task.done() or task.cancelled() or task.exception():
8125                             if task_name.startswith(self.task_name_deploy_vca):
8126                                 # A N2VC task is pending
8127                                 db_nsr_update["config-status"] = "failed"
8128                             else:
8129                                 # RO task is pending
8130                                 db_nsr_update["operational-status"] = "failed"
8131             else:
8132                 error_description_nslcmop = None
8133                 nslcmop_operation_state = "COMPLETED"
8134                 db_nslcmop_update["detailed-status"] = "Done"
8135                 db_nsr_update["detailed-status"] = "Done"
8136                 db_nsr_update["operational-status"] = "running"
8137                 db_nsr_update["config-status"] = "configured"
8138
8139             self._write_op_status(
8140                 op_id=nslcmop_id,
8141                 stage="",
8142                 error_message=error_description_nslcmop,
8143                 operation_state=nslcmop_operation_state,
8144                 other_update=db_nslcmop_update,
8145             )
8146             if db_nsr:
8147                 self._write_ns_status(
8148                     nsr_id=nsr_id,
8149                     ns_state=None,
8150                     current_operation="IDLE",
8151                     current_operation_id=None,
8152                     other_update=db_nsr_update,
8153                 )
8154
8155             if nslcmop_operation_state:
8156                 try:
8157                     msg = {
8158                         "nsr_id": nsr_id,
8159                         "nslcmop_id": nslcmop_id,
8160                         "operationState": nslcmop_operation_state,
8161                     }
8162                     await self.msg.aiowrite("ns", "healed", msg)
8163                 except Exception as e:
8164                     self.logger.error(
8165                         logging_text + "kafka_write notification Exception {}".format(e)
8166                     )
8167             self.logger.debug(logging_text + "Exit")
8168             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_heal")
8169
8170     async def heal_RO(
8171         self,
8172         logging_text,
8173         nsr_id,
8174         db_nslcmop,
8175         stage,
8176     ):
8177         """
8178         Heal at RO
8179         :param logging_text: preffix text to use at logging
8180         :param nsr_id: nsr identity
8181         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
8182         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
8183         :return: None or exception
8184         """
8185
8186         def get_vim_account(vim_account_id):
8187             nonlocal db_vims
8188             if vim_account_id in db_vims:
8189                 return db_vims[vim_account_id]
8190             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
8191             db_vims[vim_account_id] = db_vim
8192             return db_vim
8193
8194         try:
8195             start_heal = time()
8196             ns_params = db_nslcmop.get("operationParams")
8197             if ns_params and ns_params.get("timeout_ns_heal"):
8198                 timeout_ns_heal = ns_params["timeout_ns_heal"]
8199             else:
8200                 timeout_ns_heal = self.timeout.ns_heal
8201
8202             db_vims = {}
8203
8204             nslcmop_id = db_nslcmop["_id"]
8205             target = {
8206                 "action_id": nslcmop_id,
8207             }
8208             self.logger.warning(
8209                 "db_nslcmop={} and timeout_ns_heal={}".format(
8210                     db_nslcmop, timeout_ns_heal
8211                 )
8212             )
8213             target.update(db_nslcmop.get("operationParams", {}))
8214
8215             self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
8216             desc = await self.RO.recreate(nsr_id, target)
8217             self.logger.debug("RO return > {}".format(desc))
8218             action_id = desc["action_id"]
8219             # waits for RO to complete because Reinjecting juju key at ro can find VM in state Deleted
8220             await self._wait_ng_ro(
8221                 nsr_id,
8222                 action_id,
8223                 nslcmop_id,
8224                 start_heal,
8225                 timeout_ns_heal,
8226                 stage,
8227                 operation="healing",
8228             )
8229
8230             # Updating NSR
8231             db_nsr_update = {
8232                 "_admin.deployed.RO.operational-status": "running",
8233                 "detailed-status": " ".join(stage),
8234             }
8235             self.update_db_2("nsrs", nsr_id, db_nsr_update)
8236             self._write_op_status(nslcmop_id, stage)
8237             self.logger.debug(
8238                 logging_text + "ns healed at RO. RO_id={}".format(action_id)
8239             )
8240
8241         except Exception as e:
8242             stage[2] = "ERROR healing at VIM"
8243             # self.set_vnfr_at_error(db_vnfrs, str(e))
8244             self.logger.error(
8245                 "Error healing at VIM {}".format(e),
8246                 exc_info=not isinstance(
8247                     e,
8248                     (
8249                         ROclient.ROClientException,
8250                         LcmException,
8251                         DbException,
8252                         NgRoException,
8253                     ),
8254                 ),
8255             )
8256             raise
8257
8258     def _heal_n2vc(
8259         self,
8260         logging_text,
8261         db_nsr,
8262         db_vnfr,
8263         nslcmop_id,
8264         nsr_id,
8265         nsi_id,
8266         vnfd_id,
8267         vdu_id,
8268         kdu_name,
8269         member_vnf_index,
8270         vdu_index,
8271         vdu_name,
8272         deploy_params,
8273         descriptor_config,
8274         base_folder,
8275         task_instantiation_info,
8276         stage,
8277     ):
8278         # launch instantiate_N2VC in a asyncio task and register task object
8279         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
8280         # if not found, create one entry and update database
8281         # fill db_nsr._admin.deployed.VCA.<index>
8282
8283         self.logger.debug(
8284             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
8285         )
8286
8287         charm_name = ""
8288         get_charm_name = False
8289         if "execution-environment-list" in descriptor_config:
8290             ee_list = descriptor_config.get("execution-environment-list", [])
8291         elif "juju" in descriptor_config:
8292             ee_list = [descriptor_config]  # ns charms
8293             if "execution-environment-list" not in descriptor_config:
8294                 # charm name is only required for ns charms
8295                 get_charm_name = True
8296         else:  # other types as script are not supported
8297             ee_list = []
8298
8299         for ee_item in ee_list:
8300             self.logger.debug(
8301                 logging_text
8302                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
8303                     ee_item.get("juju"), ee_item.get("helm-chart")
8304                 )
8305             )
8306             ee_descriptor_id = ee_item.get("id")
8307             vca_name, charm_name, vca_type = self.get_vca_info(
8308                 ee_item, db_nsr, get_charm_name
8309             )
8310             if not vca_type:
8311                 self.logger.debug(
8312                     logging_text + "skipping, non juju/charm/helm configuration"
8313                 )
8314                 continue
8315
8316             vca_index = -1
8317             for vca_index, vca_deployed in enumerate(
8318                 db_nsr["_admin"]["deployed"]["VCA"]
8319             ):
8320                 if not vca_deployed:
8321                     continue
8322                 if (
8323                     vca_deployed.get("member-vnf-index") == member_vnf_index
8324                     and vca_deployed.get("vdu_id") == vdu_id
8325                     and vca_deployed.get("kdu_name") == kdu_name
8326                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
8327                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
8328                 ):
8329                     break
8330             else:
8331                 # not found, create one.
8332                 target = (
8333                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
8334                 )
8335                 if vdu_id:
8336                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
8337                 elif kdu_name:
8338                     target += "/kdu/{}".format(kdu_name)
8339                 vca_deployed = {
8340                     "target_element": target,
8341                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
8342                     "member-vnf-index": member_vnf_index,
8343                     "vdu_id": vdu_id,
8344                     "kdu_name": kdu_name,
8345                     "vdu_count_index": vdu_index,
8346                     "operational-status": "init",  # TODO revise
8347                     "detailed-status": "",  # TODO revise
8348                     "step": "initial-deploy",  # TODO revise
8349                     "vnfd_id": vnfd_id,
8350                     "vdu_name": vdu_name,
8351                     "type": vca_type,
8352                     "ee_descriptor_id": ee_descriptor_id,
8353                     "charm_name": charm_name,
8354                 }
8355                 vca_index += 1
8356
8357                 # create VCA and configurationStatus in db
8358                 db_dict = {
8359                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
8360                     "configurationStatus.{}".format(vca_index): dict(),
8361                 }
8362                 self.update_db_2("nsrs", nsr_id, db_dict)
8363
8364                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
8365
8366             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
8367             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
8368             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
8369
8370             # Launch task
8371             task_n2vc = asyncio.ensure_future(
8372                 self.heal_N2VC(
8373                     logging_text=logging_text,
8374                     vca_index=vca_index,
8375                     nsi_id=nsi_id,
8376                     db_nsr=db_nsr,
8377                     db_vnfr=db_vnfr,
8378                     vdu_id=vdu_id,
8379                     kdu_name=kdu_name,
8380                     vdu_index=vdu_index,
8381                     deploy_params=deploy_params,
8382                     config_descriptor=descriptor_config,
8383                     base_folder=base_folder,
8384                     nslcmop_id=nslcmop_id,
8385                     stage=stage,
8386                     vca_type=vca_type,
8387                     vca_name=vca_name,
8388                     ee_config_descriptor=ee_item,
8389                 )
8390             )
8391             self.lcm_tasks.register(
8392                 "ns",
8393                 nsr_id,
8394                 nslcmop_id,
8395                 "instantiate_N2VC-{}".format(vca_index),
8396                 task_n2vc,
8397             )
8398             task_instantiation_info[
8399                 task_n2vc
8400             ] = self.task_name_deploy_vca + " {}.{}".format(
8401                 member_vnf_index or "", vdu_id or ""
8402             )
8403
8404     async def heal_N2VC(
8405         self,
8406         logging_text,
8407         vca_index,
8408         nsi_id,
8409         db_nsr,
8410         db_vnfr,
8411         vdu_id,
8412         kdu_name,
8413         vdu_index,
8414         config_descriptor,
8415         deploy_params,
8416         base_folder,
8417         nslcmop_id,
8418         stage,
8419         vca_type,
8420         vca_name,
8421         ee_config_descriptor,
8422     ):
8423         nsr_id = db_nsr["_id"]
8424         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
8425         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
8426         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
8427         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
8428         db_dict = {
8429             "collection": "nsrs",
8430             "filter": {"_id": nsr_id},
8431             "path": db_update_entry,
8432         }
8433         step = ""
8434         try:
8435             element_type = "NS"
8436             element_under_configuration = nsr_id
8437
8438             vnfr_id = None
8439             if db_vnfr:
8440                 vnfr_id = db_vnfr["_id"]
8441                 osm_config["osm"]["vnf_id"] = vnfr_id
8442
8443             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
8444
8445             if vca_type == "native_charm":
8446                 index_number = 0
8447             else:
8448                 index_number = vdu_index or 0
8449
8450             if vnfr_id:
8451                 element_type = "VNF"
8452                 element_under_configuration = vnfr_id
8453                 namespace += ".{}-{}".format(vnfr_id, index_number)
8454                 if vdu_id:
8455                     namespace += ".{}-{}".format(vdu_id, index_number)
8456                     element_type = "VDU"
8457                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
8458                     osm_config["osm"]["vdu_id"] = vdu_id
8459                 elif kdu_name:
8460                     namespace += ".{}".format(kdu_name)
8461                     element_type = "KDU"
8462                     element_under_configuration = kdu_name
8463                     osm_config["osm"]["kdu_name"] = kdu_name
8464
8465             # Get artifact path
8466             if base_folder["pkg-dir"]:
8467                 artifact_path = "{}/{}/{}/{}".format(
8468                     base_folder["folder"],
8469                     base_folder["pkg-dir"],
8470                     "charms"
8471                     if vca_type
8472                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
8473                     else "helm-charts",
8474                     vca_name,
8475                 )
8476             else:
8477                 artifact_path = "{}/Scripts/{}/{}/".format(
8478                     base_folder["folder"],
8479                     "charms"
8480                     if vca_type
8481                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
8482                     else "helm-charts",
8483                     vca_name,
8484                 )
8485
8486             self.logger.debug("Artifact path > {}".format(artifact_path))
8487
8488             # get initial_config_primitive_list that applies to this element
8489             initial_config_primitive_list = config_descriptor.get(
8490                 "initial-config-primitive"
8491             )
8492
8493             self.logger.debug(
8494                 "Initial config primitive list > {}".format(
8495                     initial_config_primitive_list
8496                 )
8497             )
8498
8499             # add config if not present for NS charm
8500             ee_descriptor_id = ee_config_descriptor.get("id")
8501             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
8502             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
8503                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
8504             )
8505
8506             self.logger.debug(
8507                 "Initial config primitive list #2 > {}".format(
8508                     initial_config_primitive_list
8509                 )
8510             )
8511             # n2vc_redesign STEP 3.1
8512             # find old ee_id if exists
8513             ee_id = vca_deployed.get("ee_id")
8514
8515             vca_id = self.get_vca_id(db_vnfr, db_nsr)
8516             # create or register execution environment in VCA. Only for native charms when healing
8517             if vca_type == "native_charm":
8518                 step = "Waiting to VM being up and getting IP address"
8519                 self.logger.debug(logging_text + step)
8520                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
8521                     logging_text,
8522                     nsr_id,
8523                     vnfr_id,
8524                     vdu_id,
8525                     vdu_index,
8526                     user=None,
8527                     pub_key=None,
8528                 )
8529                 credentials = {"hostname": rw_mgmt_ip}
8530                 # get username
8531                 username = deep_get(
8532                     config_descriptor, ("config-access", "ssh-access", "default-user")
8533                 )
8534                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
8535                 #  merged. Meanwhile let's get username from initial-config-primitive
8536                 if not username and initial_config_primitive_list:
8537                     for config_primitive in initial_config_primitive_list:
8538                         for param in config_primitive.get("parameter", ()):
8539                             if param["name"] == "ssh-username":
8540                                 username = param["value"]
8541                                 break
8542                 if not username:
8543                     raise LcmException(
8544                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
8545                         "'config-access.ssh-access.default-user'"
8546                     )
8547                 credentials["username"] = username
8548
8549                 # n2vc_redesign STEP 3.2
8550                 # TODO: Before healing at RO it is needed to destroy native charm units to be deleted.
8551                 self._write_configuration_status(
8552                     nsr_id=nsr_id,
8553                     vca_index=vca_index,
8554                     status="REGISTERING",
8555                     element_under_configuration=element_under_configuration,
8556                     element_type=element_type,
8557                 )
8558
8559                 step = "register execution environment {}".format(credentials)
8560                 self.logger.debug(logging_text + step)
8561                 ee_id = await self.vca_map[vca_type].register_execution_environment(
8562                     credentials=credentials,
8563                     namespace=namespace,
8564                     db_dict=db_dict,
8565                     vca_id=vca_id,
8566                 )
8567
8568                 # update ee_id en db
8569                 db_dict_ee_id = {
8570                     "_admin.deployed.VCA.{}.ee_id".format(vca_index): ee_id,
8571                 }
8572                 self.update_db_2("nsrs", nsr_id, db_dict_ee_id)
8573
8574             # for compatibility with MON/POL modules, the need model and application name at database
8575             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
8576             # Not sure if this need to be done when healing
8577             """
8578             ee_id_parts = ee_id.split(".")
8579             db_nsr_update = {db_update_entry + "ee_id": ee_id}
8580             if len(ee_id_parts) >= 2:
8581                 model_name = ee_id_parts[0]
8582                 application_name = ee_id_parts[1]
8583                 db_nsr_update[db_update_entry + "model"] = model_name
8584                 db_nsr_update[db_update_entry + "application"] = application_name
8585             """
8586
8587             # n2vc_redesign STEP 3.3
8588             # Install configuration software. Only for native charms.
8589             step = "Install configuration Software"
8590
8591             self._write_configuration_status(
8592                 nsr_id=nsr_id,
8593                 vca_index=vca_index,
8594                 status="INSTALLING SW",
8595                 element_under_configuration=element_under_configuration,
8596                 element_type=element_type,
8597                 # other_update=db_nsr_update,
8598                 other_update=None,
8599             )
8600
8601             # TODO check if already done
8602             self.logger.debug(logging_text + step)
8603             config = None
8604             if vca_type == "native_charm":
8605                 config_primitive = next(
8606                     (p for p in initial_config_primitive_list if p["name"] == "config"),
8607                     None,
8608                 )
8609                 if config_primitive:
8610                     config = self._map_primitive_params(
8611                         config_primitive, {}, deploy_params
8612                     )
8613                 await self.vca_map[vca_type].install_configuration_sw(
8614                     ee_id=ee_id,
8615                     artifact_path=artifact_path,
8616                     db_dict=db_dict,
8617                     config=config,
8618                     num_units=1,
8619                     vca_id=vca_id,
8620                     vca_type=vca_type,
8621                 )
8622
8623             # write in db flag of configuration_sw already installed
8624             self.update_db_2(
8625                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
8626             )
8627
8628             # Not sure if this need to be done when healing
8629             """
8630             # add relations for this VCA (wait for other peers related with this VCA)
8631             await self._add_vca_relations(
8632                 logging_text=logging_text,
8633                 nsr_id=nsr_id,
8634                 vca_type=vca_type,
8635                 vca_index=vca_index,
8636             )
8637             """
8638
8639             # if SSH access is required, then get execution environment SSH public
8640             # if native charm we have waited already to VM be UP
8641             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm-v3"):
8642                 pub_key = None
8643                 user = None
8644                 # self.logger.debug("get ssh key block")
8645                 if deep_get(
8646                     config_descriptor, ("config-access", "ssh-access", "required")
8647                 ):
8648                     # self.logger.debug("ssh key needed")
8649                     # Needed to inject a ssh key
8650                     user = deep_get(
8651                         config_descriptor,
8652                         ("config-access", "ssh-access", "default-user"),
8653                     )
8654                     step = "Install configuration Software, getting public ssh key"
8655                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
8656                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
8657                     )
8658
8659                     step = "Insert public key into VM user={} ssh_key={}".format(
8660                         user, pub_key
8661                     )
8662                 else:
8663                     # self.logger.debug("no need to get ssh key")
8664                     step = "Waiting to VM being up and getting IP address"
8665                 self.logger.debug(logging_text + step)
8666
8667                 # n2vc_redesign STEP 5.1
8668                 # wait for RO (ip-address) Insert pub_key into VM
8669                 # IMPORTANT: We need do wait for RO to complete healing operation.
8670                 await self._wait_heal_ro(nsr_id, self.timeout.ns_heal)
8671                 if vnfr_id:
8672                     if kdu_name:
8673                         rw_mgmt_ip = await self.wait_kdu_up(
8674                             logging_text, nsr_id, vnfr_id, kdu_name
8675                         )
8676                     else:
8677                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
8678                             logging_text,
8679                             nsr_id,
8680                             vnfr_id,
8681                             vdu_id,
8682                             vdu_index,
8683                             user=user,
8684                             pub_key=pub_key,
8685                         )
8686                 else:
8687                     rw_mgmt_ip = None  # This is for a NS configuration
8688
8689                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
8690
8691             # store rw_mgmt_ip in deploy params for later replacement
8692             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
8693
8694             # Day1 operations.
8695             # get run-day1 operation parameter
8696             runDay1 = deploy_params.get("run-day1", False)
8697             self.logger.debug(
8698                 "Healing vnf={}, vdu={}, runDay1 ={}".format(vnfr_id, vdu_id, runDay1)
8699             )
8700             if runDay1:
8701                 # n2vc_redesign STEP 6  Execute initial config primitive
8702                 step = "execute initial config primitive"
8703
8704                 # wait for dependent primitives execution (NS -> VNF -> VDU)
8705                 if initial_config_primitive_list:
8706                     await self._wait_dependent_n2vc(
8707                         nsr_id, vca_deployed_list, vca_index
8708                     )
8709
8710                 # stage, in function of element type: vdu, kdu, vnf or ns
8711                 my_vca = vca_deployed_list[vca_index]
8712                 if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
8713                     # VDU or KDU
8714                     stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
8715                 elif my_vca.get("member-vnf-index"):
8716                     # VNF
8717                     stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
8718                 else:
8719                     # NS
8720                     stage[0] = "Stage 5/5: running Day-1 primitives for NS."
8721
8722                 self._write_configuration_status(
8723                     nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
8724                 )
8725
8726                 self._write_op_status(op_id=nslcmop_id, stage=stage)
8727
8728                 check_if_terminated_needed = True
8729                 for initial_config_primitive in initial_config_primitive_list:
8730                     # adding information on the vca_deployed if it is a NS execution environment
8731                     if not vca_deployed["member-vnf-index"]:
8732                         deploy_params["ns_config_info"] = json.dumps(
8733                             self._get_ns_config_info(nsr_id)
8734                         )
8735                     # TODO check if already done
8736                     primitive_params_ = self._map_primitive_params(
8737                         initial_config_primitive, {}, deploy_params
8738                     )
8739
8740                     step = "execute primitive '{}' params '{}'".format(
8741                         initial_config_primitive["name"], primitive_params_
8742                     )
8743                     self.logger.debug(logging_text + step)
8744                     await self.vca_map[vca_type].exec_primitive(
8745                         ee_id=ee_id,
8746                         primitive_name=initial_config_primitive["name"],
8747                         params_dict=primitive_params_,
8748                         db_dict=db_dict,
8749                         vca_id=vca_id,
8750                         vca_type=vca_type,
8751                     )
8752                     # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
8753                     if check_if_terminated_needed:
8754                         if config_descriptor.get("terminate-config-primitive"):
8755                             self.update_db_2(
8756                                 "nsrs",
8757                                 nsr_id,
8758                                 {db_update_entry + "needed_terminate": True},
8759                             )
8760                         check_if_terminated_needed = False
8761
8762                     # TODO register in database that primitive is done
8763
8764             # STEP 7 Configure metrics
8765             # Not sure if this need to be done when healing
8766             """
8767             if vca_type == "helm" or vca_type == "helm-v3":
8768                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
8769                     ee_id=ee_id,
8770                     artifact_path=artifact_path,
8771                     ee_config_descriptor=ee_config_descriptor,
8772                     vnfr_id=vnfr_id,
8773                     nsr_id=nsr_id,
8774                     target_ip=rw_mgmt_ip,
8775                 )
8776                 if prometheus_jobs:
8777                     self.update_db_2(
8778                         "nsrs",
8779                         nsr_id,
8780                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
8781                     )
8782
8783                     for job in prometheus_jobs:
8784                         self.db.set_one(
8785                             "prometheus_jobs",
8786                             {"job_name": job["job_name"]},
8787                             job,
8788                             upsert=True,
8789                             fail_on_empty=False,
8790                         )
8791
8792             """
8793             step = "instantiated at VCA"
8794             self.logger.debug(logging_text + step)
8795
8796             self._write_configuration_status(
8797                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
8798             )
8799
8800         except Exception as e:  # TODO not use Exception but N2VC exception
8801             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
8802             if not isinstance(
8803                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
8804             ):
8805                 self.logger.error(
8806                     "Exception while {} : {}".format(step, e), exc_info=True
8807                 )
8808             self._write_configuration_status(
8809                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
8810             )
8811             raise LcmException("{} {}".format(step, e)) from e
8812
8813     async def _wait_heal_ro(
8814         self,
8815         nsr_id,
8816         timeout=600,
8817     ):
8818         start_time = time()
8819         while time() <= start_time + timeout:
8820             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
8821             operational_status_ro = db_nsr["_admin"]["deployed"]["RO"][
8822                 "operational-status"
8823             ]
8824             self.logger.debug("Wait Heal RO > {}".format(operational_status_ro))
8825             if operational_status_ro != "healing":
8826                 break
8827             await asyncio.sleep(15)
8828         else:  # timeout_ns_deploy
8829             raise NgRoException("Timeout waiting ns to deploy")
8830
8831     async def vertical_scale(self, nsr_id, nslcmop_id):
8832         """
8833         Vertical Scale the VDUs in a NS
8834
8835         :param: nsr_id: NS Instance ID
8836         :param: nslcmop_id: nslcmop ID of migrate
8837
8838         """
8839         # Try to lock HA task here
8840         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
8841         if not task_is_locked_by_me:
8842             return
8843         logging_text = "Task ns={} vertical scale ".format(nsr_id)
8844         self.logger.debug(logging_text + "Enter")
8845         # get all needed from database
8846         db_nslcmop = None
8847         db_nslcmop_update = {}
8848         nslcmop_operation_state = None
8849         old_db_update = {}
8850         q_filter = {}
8851         old_vdu_index = None
8852         old_flavor_id = None
8853         db_nsr_update = {}
8854         target = {}
8855         exc = None
8856         # in case of error, indicates what part of scale was failed to put nsr at error status
8857         start_deploy = time()
8858
8859         try:
8860             # wait for any previous tasks in process
8861             step = "Waiting for previous operations to terminate"
8862             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
8863
8864             self._write_ns_status(
8865                 nsr_id=nsr_id,
8866                 ns_state=None,
8867                 current_operation="VerticalScale",
8868                 current_operation_id=nslcmop_id,
8869             )
8870             step = "Getting nslcmop from database"
8871             self.logger.debug(
8872                 step + " after having waited for previous tasks to be completed"
8873             )
8874             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
8875             operationParams = db_nslcmop.get("operationParams")
8876             # Update the VNFRS and NSRS with the requested flavour detail, So that ro tasks can function properly
8877             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
8878             db_flavor = db_nsr.get("flavor")
8879             db_flavor_index = str(len(db_flavor))
8880             change_vnf_flavor_data = operationParams["changeVnfFlavorData"]
8881             flavor_dict = change_vnf_flavor_data["additionalParams"]
8882             count_index = flavor_dict["vduCountIndex"]
8883             vdu_id_ref = flavor_dict["vduid"]
8884             flavor_dict_update = {
8885                 "id": db_flavor_index,
8886                 "memory-mb": flavor_dict["virtualMemory"],
8887                 "name": f"{vdu_id_ref}-{count_index}-flv",
8888                 "storage-gb": flavor_dict["sizeOfStorage"],
8889                 "vcpu-count": flavor_dict["numVirtualCpu"],
8890             }
8891             db_flavor.append(flavor_dict_update)
8892             db_update = {}
8893             db_update["flavor"] = db_flavor
8894             ns_q_filter = {
8895                 "_id": nsr_id,
8896             }
8897             self.db.set_one(
8898                 "nsrs",
8899                 q_filter=ns_q_filter,
8900                 update_dict=db_update,
8901                 fail_on_empty=True,
8902             )
8903             db_vnfr = self.db.get_one(
8904                 "vnfrs", {"_id": change_vnf_flavor_data["vnfInstanceId"]}
8905             )
8906             for vdu_index, vdur in enumerate(db_vnfr.get("vdur", ())):
8907                 if (
8908                     vdur.get("count-index") == count_index
8909                     and vdur.get("vdu-id-ref") == vdu_id_ref
8910                 ):
8911                     old_flavor_id = vdur.get("ns-flavor-id", 0)
8912                     old_vdu_index = vdu_index
8913                     filter_text = {
8914                         "_id": change_vnf_flavor_data["vnfInstanceId"],
8915                         "vdur.count-index": count_index,
8916                         "vdur.vdu-id-ref": vdu_id_ref,
8917                     }
8918                     q_filter.update(filter_text)
8919                     db_update = {}
8920                     db_update[
8921                         "vdur.{}.ns-flavor-id".format(vdu_index)
8922                     ] = db_flavor_index
8923                     self.db.set_one(
8924                         "vnfrs",
8925                         q_filter=q_filter,
8926                         update_dict=db_update,
8927                         fail_on_empty=True,
8928                     )
8929             target = {}
8930             target.update(operationParams)
8931             desc = await self.RO.vertical_scale(nsr_id, target)
8932             self.logger.debug("RO return > {}".format(desc))
8933             action_id = desc["action_id"]
8934             await self._wait_ng_ro(
8935                 nsr_id,
8936                 action_id,
8937                 nslcmop_id,
8938                 start_deploy,
8939                 self.timeout.verticalscale,
8940                 operation="verticalscale",
8941             )
8942         except (
8943             NgRoException,
8944             ROclient.ROClientException,
8945             DbException,
8946             LcmException,
8947         ) as e:
8948             self.logger.error("Exit Exception {}".format(e))
8949             exc = e
8950         except asyncio.CancelledError:
8951             self.logger.error("Cancelled Exception while '{}'".format(step))
8952             exc = "Operation was cancelled"
8953         except Exception as e:
8954             exc = traceback.format_exc()
8955             self.logger.critical(
8956                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
8957             )
8958         finally:
8959             self._write_ns_status(
8960                 nsr_id=nsr_id,
8961                 ns_state=None,
8962                 current_operation="IDLE",
8963                 current_operation_id=None,
8964             )
8965             if exc:
8966                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
8967                 nslcmop_operation_state = "FAILED"
8968                 old_db_update[
8969                     "vdur.{}.ns-flavor-id".format(old_vdu_index)
8970                 ] = old_flavor_id
8971             else:
8972                 nslcmop_operation_state = "COMPLETED"
8973                 db_nslcmop_update["detailed-status"] = "Done"
8974                 db_nsr_update["detailed-status"] = "Done"
8975
8976             self._write_op_status(
8977                 op_id=nslcmop_id,
8978                 stage="",
8979                 error_message="",
8980                 operation_state=nslcmop_operation_state,
8981                 other_update=db_nslcmop_update,
8982             )
8983             if old_vdu_index and old_db_update != {}:
8984                 self.logger.critical(
8985                     "Reverting Old Flavor -- : {}".format(old_db_update)
8986                 )
8987                 self.db.set_one(
8988                     "vnfrs",
8989                     q_filter=q_filter,
8990                     update_dict=old_db_update,
8991                     fail_on_empty=True,
8992                 )
8993             if nslcmop_operation_state:
8994                 try:
8995                     msg = {
8996                         "nsr_id": nsr_id,
8997                         "nslcmop_id": nslcmop_id,
8998                         "operationState": nslcmop_operation_state,
8999                     }
9000                     await self.msg.aiowrite("ns", "verticalscaled", msg)
9001                 except Exception as e:
9002                     self.logger.error(
9003                         logging_text + "kafka_write notification Exception {}".format(e)
9004                     )
9005             self.logger.debug(logging_text + "Exit")
9006             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_verticalscale")