osm_lcm/ns.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2018 Telefonica S.A.
   5 #
   6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   7 # not use this file except in compliance with the License. You may obtain
   8 # a copy of the License at
   9 #
  10 #         http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing, software
  13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  15 # License for the specific language governing permissions and limitations
  16 # under the License.
  17 ##
  18
  19 import asyncio
  20 import shutil
  21 from typing import Any, Dict, List
  22 import yaml
  23 import logging
  24 import logging.handlers
  25 import traceback
  26 import json
  27 from jinja2 import (
  28     Environment,
  29     TemplateError,
  30     TemplateNotFound,
  31     StrictUndefined,
  32     UndefinedError,
  33 )
  34
  35 from osm_lcm import ROclient
  36 from osm_lcm.data_utils.nsr import (
  37     get_deployed_kdu,
  38     get_deployed_vca,
  39     get_deployed_vca_list,
  40     get_nsd,
  41 )
  42 from osm_lcm.data_utils.vca import (
  43     DeployedComponent,
  44     DeployedK8sResource,
  45     DeployedVCA,
  46     EELevel,
  47     Relation,
  48     EERelation,
  49     safe_get_ee_relation,
  50 )
  51 from osm_lcm.ng_ro import NgRoClient, NgRoException
  52 from osm_lcm.lcm_utils import (
  53     LcmException,
  54     LcmExceptionNoMgmtIP,
  55     LcmBase,
  56     deep_get,
  57     get_iterable,
  58     populate_dict,
  59     check_juju_bundle_existence,
  60     get_charm_artifact_path,
  61 )
  62 from osm_lcm.data_utils.nsd import (
  63     get_ns_configuration_relation_list,
  64     get_vnf_profile,
  65     get_vnf_profiles,
  66 )
  67 from osm_lcm.data_utils.vnfd import (
  68     get_kdu,
  69     get_kdu_services,
  70     get_relation_list,
  71     get_vdu_list,
  72     get_vdu_profile,
  73     get_ee_sorted_initial_config_primitive_list,
  74     get_ee_sorted_terminate_config_primitive_list,
  75     get_kdu_list,
  76     get_virtual_link_profiles,
  77     get_vdu,
  78     get_configuration,
  79     get_vdu_index,
  80     get_scaling_aspect,
  81     get_number_of_instances,
  82     get_juju_ee_ref,
  83     get_kdu_resource_profile,
  84     find_software_version,
  85 )
  86 from osm_lcm.data_utils.list_utils import find_in_list
  87 from osm_lcm.data_utils.vnfr import (
  88     get_osm_params,
  89     get_vdur_index,
  90     get_kdur,
  91     get_volumes_from_instantiation_params,
  92 )
  93 from osm_lcm.data_utils.dict_utils import parse_yaml_strings
  94 from osm_lcm.data_utils.database.vim_account import VimAccountDB
  95 from n2vc.definitions import RelationEndpoint
  96 from n2vc.k8s_helm_conn import K8sHelmConnector
  97 from n2vc.k8s_helm3_conn import K8sHelm3Connector
  98 from n2vc.k8s_juju_conn import K8sJujuConnector
  99
 100 from osm_common.dbbase import DbException
 101 from osm_common.fsbase import FsException
 102
 103 from osm_lcm.data_utils.database.database import Database
 104 from osm_lcm.data_utils.filesystem.filesystem import Filesystem
 105
 106 from n2vc.n2vc_juju_conn import N2VCJujuConnector
 107 from n2vc.exceptions import N2VCException, N2VCNotFound, K8sException
 108
 109 from osm_lcm.lcm_helm_conn import LCMHelmConn
 110 from osm_lcm.osm_config import OsmConfigBuilder
 111 from osm_lcm.prometheus import parse_job
 112
 113 from copy import copy, deepcopy
 114 from time import time
 115 from uuid import uuid4
 116
 117 from random import randint
 118
 119 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
 120
 121
 122 class NsLcm(LcmBase):
 123     timeout_vca_on_error = (
 124         5 * 60
 125     )  # Time for charm from first time at blocked,error status to mark as failed
 126     timeout_ns_deploy = 2 * 3600  # default global timeout for deployment a ns
 127     timeout_ns_terminate = 1800  # default global timeout for un deployment a ns
 128     timeout_ns_heal = 1800  # default global timeout for un deployment a ns
 129     timeout_charm_delete = 10 * 60
 130     timeout_primitive = 30 * 60  # timeout for primitive execution
 131     timeout_ns_update = 30 * 60  # timeout for ns update
 132     timeout_progress_primitive = (
 133         10 * 60
 134     )  # timeout for some progress in a primitive execution
 135     timeout_migrate = 1800  # default global timeout for migrating vnfs
 136
 137     SUBOPERATION_STATUS_NOT_FOUND = -1
 138     SUBOPERATION_STATUS_NEW = -2
 139     SUBOPERATION_STATUS_SKIP = -3
 140     task_name_deploy_vca = "Deploying VCA"
 141
 142     def __init__(self, msg, lcm_tasks, config, loop):
 143         """
 144         Init, Connect to database, filesystem storage, and messaging
 145         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
 146         :return: None
 147         """
 148         super().__init__(msg=msg, logger=logging.getLogger("lcm.ns"))
 149
 150         self.db = Database().instance.db
 151         self.fs = Filesystem().instance.fs
 152         self.loop = loop
 153         self.lcm_tasks = lcm_tasks
 154         self.timeout = config["timeout"]
 155         self.ro_config = config["ro_config"]
 156         self.ng_ro = config["ro_config"].get("ng")
 157         self.vca_config = config["VCA"].copy()
 158
 159         # create N2VC connector
 160         self.n2vc = N2VCJujuConnector(
 161             log=self.logger,
 162             loop=self.loop,
 163             on_update_db=self._on_update_n2vc_db,
 164             fs=self.fs,
 165             db=self.db,
 166         )
 167
 168         self.conn_helm_ee = LCMHelmConn(
 169             log=self.logger,
 170             loop=self.loop,
 171             vca_config=self.vca_config,
 172             on_update_db=self._on_update_n2vc_db,
 173         )
 174
 175         self.k8sclusterhelm2 = K8sHelmConnector(
 176             kubectl_command=self.vca_config.get("kubectlpath"),
 177             helm_command=self.vca_config.get("helmpath"),
 178             log=self.logger,
 179             on_update_db=None,
 180             fs=self.fs,
 181             db=self.db,
 182         )
 183
 184         self.k8sclusterhelm3 = K8sHelm3Connector(
 185             kubectl_command=self.vca_config.get("kubectlpath"),
 186             helm_command=self.vca_config.get("helm3path"),
 187             fs=self.fs,
 188             log=self.logger,
 189             db=self.db,
 190             on_update_db=None,
 191         )
 192
 193         self.k8sclusterjuju = K8sJujuConnector(
 194             kubectl_command=self.vca_config.get("kubectlpath"),
 195             juju_command=self.vca_config.get("jujupath"),
 196             log=self.logger,
 197             loop=self.loop,
 198             on_update_db=self._on_update_k8s_db,
 199             fs=self.fs,
 200             db=self.db,
 201         )
 202
 203         self.k8scluster_map = {
 204             "helm-chart": self.k8sclusterhelm2,
 205             "helm-chart-v3": self.k8sclusterhelm3,
 206             "chart": self.k8sclusterhelm3,
 207             "juju-bundle": self.k8sclusterjuju,
 208             "juju": self.k8sclusterjuju,
 209         }
 210
 211         self.vca_map = {
 212             "lxc_proxy_charm": self.n2vc,
 213             "native_charm": self.n2vc,
 214             "k8s_proxy_charm": self.n2vc,
 215             "helm": self.conn_helm_ee,
 216             "helm-v3": self.conn_helm_ee,
 217         }
 218
 219         # create RO client
 220         self.RO = NgRoClient(self.loop, **self.ro_config)
 221
 222         self.op_status_map = {
 223             "instantiation": self.RO.status,
 224             "termination": self.RO.status,
 225             "migrate": self.RO.status,
 226             "healing": self.RO.recreate_status,
 227         }
 228
 229     @staticmethod
 230     def increment_ip_mac(ip_mac, vm_index=1):
 231         if not isinstance(ip_mac, str):
 232             return ip_mac
 233         try:
 234             # try with ipv4 look for last dot
 235             i = ip_mac.rfind(".")
 236             if i > 0:
 237                 i += 1
 238                 return "{}{}".format(ip_mac[:i], int(ip_mac[i:]) + vm_index)
 239             # try with ipv6 or mac look for last colon. Operate in hex
 240             i = ip_mac.rfind(":")
 241             if i > 0:
 242                 i += 1
 243                 # format in hex, len can be 2 for mac or 4 for ipv6
 244                 return ("{}{:0" + str(len(ip_mac) - i) + "x}").format(
 245                     ip_mac[:i], int(ip_mac[i:], 16) + vm_index
 246                 )
 247         except Exception:
 248             pass
 249         return None
 250
 251     def _on_update_ro_db(self, nsrs_id, ro_descriptor):
 252
 253         # self.logger.debug('_on_update_ro_db(nsrs_id={}'.format(nsrs_id))
 254
 255         try:
 256             # TODO filter RO descriptor fields...
 257
 258             # write to database
 259             db_dict = dict()
 260             # db_dict['deploymentStatus'] = yaml.dump(ro_descriptor, default_flow_style=False, indent=2)
 261             db_dict["deploymentStatus"] = ro_descriptor
 262             self.update_db_2("nsrs", nsrs_id, db_dict)
 263
 264         except Exception as e:
 265             self.logger.warn(
 266                 "Cannot write database RO deployment for ns={} -> {}".format(nsrs_id, e)
 267             )
 268
 269     async def _on_update_n2vc_db(self, table, filter, path, updated_data, vca_id=None):
 270
 271         # remove last dot from path (if exists)
 272         if path.endswith("."):
 273             path = path[:-1]
 274
 275         # self.logger.debug('_on_update_n2vc_db(table={}, filter={}, path={}, updated_data={}'
 276         #                   .format(table, filter, path, updated_data))
 277         try:
 278
 279             nsr_id = filter.get("_id")
 280
 281             # read ns record from database
 282             nsr = self.db.get_one(table="nsrs", q_filter=filter)
 283             current_ns_status = nsr.get("nsState")
 284
 285             # get vca status for NS
 286             status_dict = await self.n2vc.get_status(
 287                 namespace="." + nsr_id, yaml_format=False, vca_id=vca_id
 288             )
 289
 290             # vcaStatus
 291             db_dict = dict()
 292             db_dict["vcaStatus"] = status_dict
 293             await self.n2vc.update_vca_status(db_dict["vcaStatus"], vca_id=vca_id)
 294
 295             # update configurationStatus for this VCA
 296             try:
 297                 vca_index = int(path[path.rfind(".") + 1 :])
 298
 299                 vca_list = deep_get(
 300                     target_dict=nsr, key_list=("_admin", "deployed", "VCA")
 301                 )
 302                 vca_status = vca_list[vca_index].get("status")
 303
 304                 configuration_status_list = nsr.get("configurationStatus")
 305                 config_status = configuration_status_list[vca_index].get("status")
 306
 307                 if config_status == "BROKEN" and vca_status != "failed":
 308                     db_dict["configurationStatus"][vca_index] = "READY"
 309                 elif config_status != "BROKEN" and vca_status == "failed":
 310                     db_dict["configurationStatus"][vca_index] = "BROKEN"
 311             except Exception as e:
 312                 # not update configurationStatus
 313                 self.logger.debug("Error updating vca_index (ignore): {}".format(e))
 314
 315             # if nsState = 'READY' check if juju is reporting some error => nsState = 'DEGRADED'
 316             # if nsState = 'DEGRADED' check if all is OK
 317             is_degraded = False
 318             if current_ns_status in ("READY", "DEGRADED"):
 319                 error_description = ""
 320                 # check machines
 321                 if status_dict.get("machines"):
 322                     for machine_id in status_dict.get("machines"):
 323                         machine = status_dict.get("machines").get(machine_id)
 324                         # check machine agent-status
 325                         if machine.get("agent-status"):
 326                             s = machine.get("agent-status").get("status")
 327                             if s != "started":
 328                                 is_degraded = True
 329                                 error_description += (
 330                                     "machine {} agent-status={} ; ".format(
 331                                         machine_id, s
 332                                     )
 333                                 )
 334                         # check machine instance status
 335                         if machine.get("instance-status"):
 336                             s = machine.get("instance-status").get("status")
 337                             if s != "running":
 338                                 is_degraded = True
 339                                 error_description += (
 340                                     "machine {} instance-status={} ; ".format(
 341                                         machine_id, s
 342                                     )
 343                                 )
 344                 # check applications
 345                 if status_dict.get("applications"):
 346                     for app_id in status_dict.get("applications"):
 347                         app = status_dict.get("applications").get(app_id)
 348                         # check application status
 349                         if app.get("status"):
 350                             s = app.get("status").get("status")
 351                             if s != "active":
 352                                 is_degraded = True
 353                                 error_description += (
 354                                     "application {} status={} ; ".format(app_id, s)
 355                                 )
 356
 357                 if error_description:
 358                     db_dict["errorDescription"] = error_description
 359                 if current_ns_status == "READY" and is_degraded:
 360                     db_dict["nsState"] = "DEGRADED"
 361                 if current_ns_status == "DEGRADED" and not is_degraded:
 362                     db_dict["nsState"] = "READY"
 363
 364             # write to database
 365             self.update_db_2("nsrs", nsr_id, db_dict)
 366
 367         except (asyncio.CancelledError, asyncio.TimeoutError):
 368             raise
 369         except Exception as e:
 370             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 371
 372     async def _on_update_k8s_db(
 373         self, cluster_uuid, kdu_instance, filter=None, vca_id=None, cluster_type="juju"
 374     ):
 375         """
 376         Updating vca status in NSR record
 377         :param cluster_uuid: UUID of a k8s cluster
 378         :param kdu_instance: The unique name of the KDU instance
 379         :param filter: To get nsr_id
 380         :cluster_type: The cluster type (juju, k8s)
 381         :return: none
 382         """
 383
 384         # self.logger.debug("_on_update_k8s_db(cluster_uuid={}, kdu_instance={}, filter={}"
 385         #                   .format(cluster_uuid, kdu_instance, filter))
 386
 387         nsr_id = filter.get("_id")
 388         try:
 389             vca_status = await self.k8scluster_map[cluster_type].status_kdu(
 390                 cluster_uuid=cluster_uuid,
 391                 kdu_instance=kdu_instance,
 392                 yaml_format=False,
 393                 complete_status=True,
 394                 vca_id=vca_id,
 395             )
 396
 397             # vcaStatus
 398             db_dict = dict()
 399             db_dict["vcaStatus"] = {nsr_id: vca_status}
 400
 401             if cluster_type in ("juju-bundle", "juju"):
 402                 # TODO -> this should be done in a more uniform way, I think in N2VC, in order to update the K8s VCA
 403                 #  status in a similar way between Juju Bundles and Helm Charts on this side
 404                 await self.k8sclusterjuju.update_vca_status(
 405                     db_dict["vcaStatus"],
 406                     kdu_instance,
 407                     vca_id=vca_id,
 408                 )
 409
 410             self.logger.debug(
 411                 f"Obtained VCA status for cluster type '{cluster_type}': {vca_status}"
 412             )
 413
 414             # write to database
 415             self.update_db_2("nsrs", nsr_id, db_dict)
 416         except (asyncio.CancelledError, asyncio.TimeoutError):
 417             raise
 418         except Exception as e:
 419             self.logger.warn("Error updating NS state for ns={}: {}".format(nsr_id, e))
 420
 421     @staticmethod
 422     def _parse_cloud_init(cloud_init_text, additional_params, vnfd_id, vdu_id):
 423         try:
 424             env = Environment(undefined=StrictUndefined)
 425             template = env.from_string(cloud_init_text)
 426             return template.render(additional_params or {})
 427         except UndefinedError as e:
 428             raise LcmException(
 429                 "Variable {} at vnfd[id={}]:vdu[id={}]:cloud-init/cloud-init-"
 430                 "file, must be provided in the instantiation parameters inside the "
 431                 "'additionalParamsForVnf/Vdu' block".format(e, vnfd_id, vdu_id)
 432             )
 433         except (TemplateError, TemplateNotFound) as e:
 434             raise LcmException(
 435                 "Error parsing Jinja2 to cloud-init content at vnfd[id={}]:vdu[id={}]: {}".format(
 436                     vnfd_id, vdu_id, e
 437                 )
 438             )
 439
 440     def _get_vdu_cloud_init_content(self, vdu, vnfd):
 441         cloud_init_content = cloud_init_file = None
 442         try:
 443             if vdu.get("cloud-init-file"):
 444                 base_folder = vnfd["_admin"]["storage"]
 445                 if base_folder["pkg-dir"]:
 446                     cloud_init_file = "{}/{}/cloud_init/{}".format(
 447                         base_folder["folder"],
 448                         base_folder["pkg-dir"],
 449                         vdu["cloud-init-file"],
 450                     )
 451                 else:
 452                     cloud_init_file = "{}/Scripts/cloud_init/{}".format(
 453                         base_folder["folder"],
 454                         vdu["cloud-init-file"],
 455                     )
 456                 with self.fs.file_open(cloud_init_file, "r") as ci_file:
 457                     cloud_init_content = ci_file.read()
 458             elif vdu.get("cloud-init"):
 459                 cloud_init_content = vdu["cloud-init"]
 460
 461             return cloud_init_content
 462         except FsException as e:
 463             raise LcmException(
 464                 "Error reading vnfd[id={}]:vdu[id={}]:cloud-init-file={}: {}".format(
 465                     vnfd["id"], vdu["id"], cloud_init_file, e
 466                 )
 467             )
 468
 469     def _get_vdu_additional_params(self, db_vnfr, vdu_id):
 470         vdur = next(
 471             (vdur for vdur in db_vnfr.get("vdur") if vdu_id == vdur["vdu-id-ref"]), {}
 472         )
 473         additional_params = vdur.get("additionalParams")
 474         return parse_yaml_strings(additional_params)
 475
 476     def vnfd2RO(self, vnfd, new_id=None, additionalParams=None, nsrId=None):
 477         """
 478         Converts creates a new vnfd descriptor for RO base on input OSM IM vnfd
 479         :param vnfd: input vnfd
 480         :param new_id: overrides vnf id if provided
 481         :param additionalParams: Instantiation params for VNFs provided
 482         :param nsrId: Id of the NSR
 483         :return: copy of vnfd
 484         """
 485         vnfd_RO = deepcopy(vnfd)
 486         # remove unused by RO configuration, monitoring, scaling and internal keys
 487         vnfd_RO.pop("_id", None)
 488         vnfd_RO.pop("_admin", None)
 489         vnfd_RO.pop("monitoring-param", None)
 490         vnfd_RO.pop("scaling-group-descriptor", None)
 491         vnfd_RO.pop("kdu", None)
 492         vnfd_RO.pop("k8s-cluster", None)
 493         if new_id:
 494             vnfd_RO["id"] = new_id
 495
 496         # parse cloud-init or cloud-init-file with the provided variables using Jinja2
 497         for vdu in get_iterable(vnfd_RO, "vdu"):
 498             vdu.pop("cloud-init-file", None)
 499             vdu.pop("cloud-init", None)
 500         return vnfd_RO
 501
 502     @staticmethod
 503     def ip_profile_2_RO(ip_profile):
 504         RO_ip_profile = deepcopy(ip_profile)
 505         if "dns-server" in RO_ip_profile:
 506             if isinstance(RO_ip_profile["dns-server"], list):
 507                 RO_ip_profile["dns-address"] = []
 508                 for ds in RO_ip_profile.pop("dns-server"):
 509                     RO_ip_profile["dns-address"].append(ds["address"])
 510             else:
 511                 RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 512         if RO_ip_profile.get("ip-version") == "ipv4":
 513             RO_ip_profile["ip-version"] = "IPv4"
 514         if RO_ip_profile.get("ip-version") == "ipv6":
 515             RO_ip_profile["ip-version"] = "IPv6"
 516         if "dhcp-params" in RO_ip_profile:
 517             RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 518         return RO_ip_profile
 519
 520     def _get_ro_vim_id_for_vim_account(self, vim_account):
 521         db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 522         if db_vim["_admin"]["operationalState"] != "ENABLED":
 523             raise LcmException(
 524                 "VIM={} is not available. operationalState={}".format(
 525                     vim_account, db_vim["_admin"]["operationalState"]
 526                 )
 527             )
 528         RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 529         return RO_vim_id
 530
 531     def get_ro_wim_id_for_wim_account(self, wim_account):
 532         if isinstance(wim_account, str):
 533             db_wim = self.db.get_one("wim_accounts", {"_id": wim_account})
 534             if db_wim["_admin"]["operationalState"] != "ENABLED":
 535                 raise LcmException(
 536                     "WIM={} is not available. operationalState={}".format(
 537                         wim_account, db_wim["_admin"]["operationalState"]
 538                     )
 539                 )
 540             RO_wim_id = db_wim["_admin"]["deployed"]["RO-account"]
 541             return RO_wim_id
 542         else:
 543             return wim_account
 544
 545     def scale_vnfr(self, db_vnfr, vdu_create=None, vdu_delete=None, mark_delete=False):
 546
 547         db_vdu_push_list = []
 548         template_vdur = []
 549         db_update = {"_admin.modified": time()}
 550         if vdu_create:
 551             for vdu_id, vdu_count in vdu_create.items():
 552                 vdur = next(
 553                     (
 554                         vdur
 555                         for vdur in reversed(db_vnfr["vdur"])
 556                         if vdur["vdu-id-ref"] == vdu_id
 557                     ),
 558                     None,
 559                 )
 560                 if not vdur:
 561                     # Read the template saved in the db:
 562                     self.logger.debug(
 563                         "No vdur in the database. Using the vdur-template to scale"
 564                     )
 565                     vdur_template = db_vnfr.get("vdur-template")
 566                     if not vdur_template:
 567                         raise LcmException(
 568                             "Error scaling OUT VNFR for {}. No vnfr or template exists".format(
 569                                 vdu_id
 570                             )
 571                         )
 572                     vdur = vdur_template[0]
 573                     # Delete a template from the database after using it
 574                     self.db.set_one(
 575                         "vnfrs",
 576                         {"_id": db_vnfr["_id"]},
 577                         None,
 578                         pull={"vdur-template": {"_id": vdur["_id"]}},
 579                     )
 580                 for count in range(vdu_count):
 581                     vdur_copy = deepcopy(vdur)
 582                     vdur_copy["status"] = "BUILD"
 583                     vdur_copy["status-detailed"] = None
 584                     vdur_copy["ip-address"] = None
 585                     vdur_copy["_id"] = str(uuid4())
 586                     vdur_copy["count-index"] += count + 1
 587                     vdur_copy["id"] = "{}-{}".format(
 588                         vdur_copy["vdu-id-ref"], vdur_copy["count-index"]
 589                     )
 590                     vdur_copy.pop("vim_info", None)
 591                     for iface in vdur_copy["interfaces"]:
 592                         if iface.get("fixed-ip"):
 593                             iface["ip-address"] = self.increment_ip_mac(
 594                                 iface["ip-address"], count + 1
 595                             )
 596                         else:
 597                             iface.pop("ip-address", None)
 598                         if iface.get("fixed-mac"):
 599                             iface["mac-address"] = self.increment_ip_mac(
 600                                 iface["mac-address"], count + 1
 601                             )
 602                         else:
 603                             iface.pop("mac-address", None)
 604                         if db_vnfr["vdur"]:
 605                             iface.pop(
 606                                 "mgmt_vnf", None
 607                             )  # only first vdu can be managment of vnf
 608                     db_vdu_push_list.append(vdur_copy)
 609                     # self.logger.debug("scale out, adding vdu={}".format(vdur_copy))
 610         if vdu_delete:
 611             if len(db_vnfr["vdur"]) == 1:
 612                 # The scale will move to 0 instances
 613                 self.logger.debug(
 614                     "Scaling to 0 !, creating the template with the last vdur"
 615                 )
 616                 template_vdur = [db_vnfr["vdur"][0]]
 617             for vdu_id, vdu_count in vdu_delete.items():
 618                 if mark_delete:
 619                     indexes_to_delete = [
 620                         iv[0]
 621                         for iv in enumerate(db_vnfr["vdur"])
 622                         if iv[1]["vdu-id-ref"] == vdu_id
 623                     ]
 624                     db_update.update(
 625                         {
 626                             "vdur.{}.status".format(i): "DELETING"
 627                             for i in indexes_to_delete[-vdu_count:]
 628                         }
 629                     )
 630                 else:
 631                     # it must be deleted one by one because common.db does not allow otherwise
 632                     vdus_to_delete = [
 633                         v
 634                         for v in reversed(db_vnfr["vdur"])
 635                         if v["vdu-id-ref"] == vdu_id
 636                     ]
 637                     for vdu in vdus_to_delete[:vdu_count]:
 638                         self.db.set_one(
 639                             "vnfrs",
 640                             {"_id": db_vnfr["_id"]},
 641                             None,
 642                             pull={"vdur": {"_id": vdu["_id"]}},
 643                         )
 644         db_push = {}
 645         if db_vdu_push_list:
 646             db_push["vdur"] = db_vdu_push_list
 647         if template_vdur:
 648             db_push["vdur-template"] = template_vdur
 649         if not db_push:
 650             db_push = None
 651         db_vnfr["vdur-template"] = template_vdur
 652         self.db.set_one("vnfrs", {"_id": db_vnfr["_id"]}, db_update, push_list=db_push)
 653         # modify passed dictionary db_vnfr
 654         db_vnfr_ = self.db.get_one("vnfrs", {"_id": db_vnfr["_id"]})
 655         db_vnfr["vdur"] = db_vnfr_["vdur"]
 656
 657     def ns_update_nsr(self, ns_update_nsr, db_nsr, nsr_desc_RO):
 658         """
 659         Updates database nsr with the RO info for the created vld
 660         :param ns_update_nsr: dictionary to be filled with the updated info
 661         :param db_nsr: content of db_nsr. This is also modified
 662         :param nsr_desc_RO: nsr descriptor from RO
 663         :return: Nothing, LcmException is raised on errors
 664         """
 665
 666         for vld_index, vld in enumerate(get_iterable(db_nsr, "vld")):
 667             for net_RO in get_iterable(nsr_desc_RO, "nets"):
 668                 if vld["id"] != net_RO.get("ns_net_osm_id"):
 669                     continue
 670                 vld["vim-id"] = net_RO.get("vim_net_id")
 671                 vld["name"] = net_RO.get("vim_name")
 672                 vld["status"] = net_RO.get("status")
 673                 vld["status-detailed"] = net_RO.get("error_msg")
 674                 ns_update_nsr["vld.{}".format(vld_index)] = vld
 675                 break
 676             else:
 677                 raise LcmException(
 678                     "ns_update_nsr: Not found vld={} at RO info".format(vld["id"])
 679                 )
 680
 681     def set_vnfr_at_error(self, db_vnfrs, error_text):
 682         try:
 683             for db_vnfr in db_vnfrs.values():
 684                 vnfr_update = {"status": "ERROR"}
 685                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 686                     if "status" not in vdur:
 687                         vdur["status"] = "ERROR"
 688                         vnfr_update["vdur.{}.status".format(vdu_index)] = "ERROR"
 689                         if error_text:
 690                             vdur["status-detailed"] = str(error_text)
 691                             vnfr_update[
 692                                 "vdur.{}.status-detailed".format(vdu_index)
 693                             ] = "ERROR"
 694                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 695         except DbException as e:
 696             self.logger.error("Cannot update vnf. {}".format(e))
 697
 698     def ns_update_vnfr(self, db_vnfrs, nsr_desc_RO):
 699         """
 700         Updates database vnfr with the RO info, e.g. ip_address, vim_id... Descriptor db_vnfrs is also updated
 701         :param db_vnfrs: dictionary with member-vnf-index: vnfr-content
 702         :param nsr_desc_RO: nsr descriptor from RO
 703         :return: Nothing, LcmException is raised on errors
 704         """
 705         for vnf_index, db_vnfr in db_vnfrs.items():
 706             for vnf_RO in nsr_desc_RO["vnfs"]:
 707                 if vnf_RO["member_vnf_index"] != vnf_index:
 708                     continue
 709                 vnfr_update = {}
 710                 if vnf_RO.get("ip_address"):
 711                     db_vnfr["ip-address"] = vnfr_update["ip-address"] = vnf_RO[
 712                         "ip_address"
 713                     ].split(";")[0]
 714                 elif not db_vnfr.get("ip-address"):
 715                     if db_vnfr.get("vdur"):  # if not VDUs, there is not ip_address
 716                         raise LcmExceptionNoMgmtIP(
 717                             "ns member_vnf_index '{}' has no IP address".format(
 718                                 vnf_index
 719                             )
 720                         )
 721
 722                 for vdu_index, vdur in enumerate(get_iterable(db_vnfr, "vdur")):
 723                     vdur_RO_count_index = 0
 724                     if vdur.get("pdu-type"):
 725                         continue
 726                     for vdur_RO in get_iterable(vnf_RO, "vms"):
 727                         if vdur["vdu-id-ref"] != vdur_RO["vdu_osm_id"]:
 728                             continue
 729                         if vdur["count-index"] != vdur_RO_count_index:
 730                             vdur_RO_count_index += 1
 731                             continue
 732                         vdur["vim-id"] = vdur_RO.get("vim_vm_id")
 733                         if vdur_RO.get("ip_address"):
 734                             vdur["ip-address"] = vdur_RO["ip_address"].split(";")[0]
 735                         else:
 736                             vdur["ip-address"] = None
 737                         vdur["vdu-id-ref"] = vdur_RO.get("vdu_osm_id")
 738                         vdur["name"] = vdur_RO.get("vim_name")
 739                         vdur["status"] = vdur_RO.get("status")
 740                         vdur["status-detailed"] = vdur_RO.get("error_msg")
 741                         for ifacer in get_iterable(vdur, "interfaces"):
 742                             for interface_RO in get_iterable(vdur_RO, "interfaces"):
 743                                 if ifacer["name"] == interface_RO.get("internal_name"):
 744                                     ifacer["ip-address"] = interface_RO.get(
 745                                         "ip_address"
 746                                     )
 747                                     ifacer["mac-address"] = interface_RO.get(
 748                                         "mac_address"
 749                                     )
 750                                     break
 751                             else:
 752                                 raise LcmException(
 753                                     "ns_update_vnfr: Not found member_vnf_index={} vdur={} interface={} "
 754                                     "from VIM info".format(
 755                                         vnf_index, vdur["vdu-id-ref"], ifacer["name"]
 756                                     )
 757                                 )
 758                         vnfr_update["vdur.{}".format(vdu_index)] = vdur
 759                         break
 760                     else:
 761                         raise LcmException(
 762                             "ns_update_vnfr: Not found member_vnf_index={} vdur={} count_index={} from "
 763                             "VIM info".format(
 764                                 vnf_index, vdur["vdu-id-ref"], vdur["count-index"]
 765                             )
 766                         )
 767
 768                 for vld_index, vld in enumerate(get_iterable(db_vnfr, "vld")):
 769                     for net_RO in get_iterable(nsr_desc_RO, "nets"):
 770                         if vld["id"] != net_RO.get("vnf_net_osm_id"):
 771                             continue
 772                         vld["vim-id"] = net_RO.get("vim_net_id")
 773                         vld["name"] = net_RO.get("vim_name")
 774                         vld["status"] = net_RO.get("status")
 775                         vld["status-detailed"] = net_RO.get("error_msg")
 776                         vnfr_update["vld.{}".format(vld_index)] = vld
 777                         break
 778                     else:
 779                         raise LcmException(
 780                             "ns_update_vnfr: Not found member_vnf_index={} vld={} from VIM info".format(
 781                                 vnf_index, vld["id"]
 782                             )
 783                         )
 784
 785                 self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
 786                 break
 787
 788             else:
 789                 raise LcmException(
 790                     "ns_update_vnfr: Not found member_vnf_index={} from VIM info".format(
 791                         vnf_index
 792                     )
 793                 )
 794
 795     def _get_ns_config_info(self, nsr_id):
 796         """
 797         Generates a mapping between vnf,vdu elements and the N2VC id
 798         :param nsr_id: id of nsr to get last  database _admin.deployed.VCA that contains this list
 799         :return: a dictionary with {osm-config-mapping: {}} where its element contains:
 800             "<member-vnf-index>": <N2VC-id>  for a vnf configuration, or
 801             "<member-vnf-index>.<vdu.id>.<vdu replica(0, 1,..)>": <N2VC-id>  for a vdu configuration
 802         """
 803         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
 804         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
 805         mapping = {}
 806         ns_config_info = {"osm-config-mapping": mapping}
 807         for vca in vca_deployed_list:
 808             if not vca["member-vnf-index"]:
 809                 continue
 810             if not vca["vdu_id"]:
 811                 mapping[vca["member-vnf-index"]] = vca["application"]
 812             else:
 813                 mapping[
 814                     "{}.{}.{}".format(
 815                         vca["member-vnf-index"], vca["vdu_id"], vca["vdu_count_index"]
 816                     )
 817                 ] = vca["application"]
 818         return ns_config_info
 819
 820     async def _instantiate_ng_ro(
 821         self,
 822         logging_text,
 823         nsr_id,
 824         nsd,
 825         db_nsr,
 826         db_nslcmop,
 827         db_vnfrs,
 828         db_vnfds,
 829         n2vc_key_list,
 830         stage,
 831         start_deploy,
 832         timeout_ns_deploy,
 833     ):
 834
 835         db_vims = {}
 836
 837         def get_vim_account(vim_account_id):
 838             nonlocal db_vims
 839             if vim_account_id in db_vims:
 840                 return db_vims[vim_account_id]
 841             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
 842             db_vims[vim_account_id] = db_vim
 843             return db_vim
 844
 845         # modify target_vld info with instantiation parameters
 846         def parse_vld_instantiation_params(
 847             target_vim, target_vld, vld_params, target_sdn
 848         ):
 849             if vld_params.get("ip-profile"):
 850                 target_vld["vim_info"][target_vim]["ip_profile"] = vld_params[
 851                     "ip-profile"
 852                 ]
 853             if vld_params.get("provider-network"):
 854                 target_vld["vim_info"][target_vim]["provider_network"] = vld_params[
 855                     "provider-network"
 856                 ]
 857                 if "sdn-ports" in vld_params["provider-network"] and target_sdn:
 858                     target_vld["vim_info"][target_sdn]["sdn-ports"] = vld_params[
 859                         "provider-network"
 860                     ]["sdn-ports"]
 861             if vld_params.get("wimAccountId"):
 862                 target_wim = "wim:{}".format(vld_params["wimAccountId"])
 863                 target_vld["vim_info"][target_wim] = {}
 864             for param in ("vim-network-name", "vim-network-id"):
 865                 if vld_params.get(param):
 866                     if isinstance(vld_params[param], dict):
 867                         for vim, vim_net in vld_params[param].items():
 868                             other_target_vim = "vim:" + vim
 869                             populate_dict(
 870                                 target_vld["vim_info"],
 871                                 (other_target_vim, param.replace("-", "_")),
 872                                 vim_net,
 873                             )
 874                     else:  # isinstance str
 875                         target_vld["vim_info"][target_vim][
 876                             param.replace("-", "_")
 877                         ] = vld_params[param]
 878             if vld_params.get("common_id"):
 879                 target_vld["common_id"] = vld_params.get("common_id")
 880
 881         # modify target["ns"]["vld"] with instantiation parameters to override vnf vim-account
 882         def update_ns_vld_target(target, ns_params):
 883             for vnf_params in ns_params.get("vnf", ()):
 884                 if vnf_params.get("vimAccountId"):
 885                     target_vnf = next(
 886                         (
 887                             vnfr
 888                             for vnfr in db_vnfrs.values()
 889                             if vnf_params["member-vnf-index"]
 890                             == vnfr["member-vnf-index-ref"]
 891                         ),
 892                         None,
 893                     )
 894                     vdur = next((vdur for vdur in target_vnf.get("vdur", ())), None)
 895                     for a_index, a_vld in enumerate(target["ns"]["vld"]):
 896                         target_vld = find_in_list(
 897                             get_iterable(vdur, "interfaces"),
 898                             lambda iface: iface.get("ns-vld-id") == a_vld["name"],
 899                         )
 900                         if target_vld:
 901                             if vnf_params.get("vimAccountId") not in a_vld.get(
 902                                 "vim_info", {}
 903                             ):
 904                                 target["ns"]["vld"][a_index].get("vim_info").update(
 905                                     {
 906                                         "vim:{}".format(vnf_params["vimAccountId"]): {
 907                                             "vim_network_name": ""
 908                                         }
 909                                     }
 910                                 )
 911
 912         nslcmop_id = db_nslcmop["_id"]
 913         target = {
 914             "name": db_nsr["name"],
 915             "ns": {"vld": []},
 916             "vnf": [],
 917             "image": deepcopy(db_nsr["image"]),
 918             "flavor": deepcopy(db_nsr["flavor"]),
 919             "action_id": nslcmop_id,
 920             "cloud_init_content": {},
 921         }
 922         for image in target["image"]:
 923             image["vim_info"] = {}
 924         for flavor in target["flavor"]:
 925             flavor["vim_info"] = {}
 926         if db_nsr.get("affinity-or-anti-affinity-group"):
 927             target["affinity-or-anti-affinity-group"] = deepcopy(
 928                 db_nsr["affinity-or-anti-affinity-group"]
 929             )
 930             for affinity_or_anti_affinity_group in target[
 931                 "affinity-or-anti-affinity-group"
 932             ]:
 933                 affinity_or_anti_affinity_group["vim_info"] = {}
 934
 935         if db_nslcmop.get("lcmOperationType") != "instantiate":
 936             # get parameters of instantiation:
 937             db_nslcmop_instantiate = self.db.get_list(
 938                 "nslcmops",
 939                 {
 940                     "nsInstanceId": db_nslcmop["nsInstanceId"],
 941                     "lcmOperationType": "instantiate",
 942                 },
 943             )[-1]
 944             ns_params = db_nslcmop_instantiate.get("operationParams")
 945         else:
 946             ns_params = db_nslcmop.get("operationParams")
 947         ssh_keys_instantiation = ns_params.get("ssh_keys") or []
 948         ssh_keys_all = ssh_keys_instantiation + (n2vc_key_list or [])
 949
 950         cp2target = {}
 951         for vld_index, vld in enumerate(db_nsr.get("vld")):
 952             target_vim = "vim:{}".format(ns_params["vimAccountId"])
 953             target_vld = {
 954                 "id": vld["id"],
 955                 "name": vld["name"],
 956                 "mgmt-network": vld.get("mgmt-network", False),
 957                 "type": vld.get("type"),
 958                 "vim_info": {
 959                     target_vim: {
 960                         "vim_network_name": vld.get("vim-network-name"),
 961                         "vim_account_id": ns_params["vimAccountId"],
 962                     }
 963                 },
 964             }
 965             # check if this network needs SDN assist
 966             if vld.get("pci-interfaces"):
 967                 db_vim = get_vim_account(ns_params["vimAccountId"])
 968                 sdnc_id = db_vim["config"].get("sdn-controller")
 969                 if sdnc_id:
 970                     sdn_vld = "nsrs:{}:vld.{}".format(nsr_id, vld["id"])
 971                     target_sdn = "sdn:{}".format(sdnc_id)
 972                     target_vld["vim_info"][target_sdn] = {
 973                         "sdn": True,
 974                         "target_vim": target_vim,
 975                         "vlds": [sdn_vld],
 976                         "type": vld.get("type"),
 977                     }
 978
 979             nsd_vnf_profiles = get_vnf_profiles(nsd)
 980             for nsd_vnf_profile in nsd_vnf_profiles:
 981                 for cp in nsd_vnf_profile["virtual-link-connectivity"]:
 982                     if cp["virtual-link-profile-id"] == vld["id"]:
 983                         cp2target[
 984                             "member_vnf:{}.{}".format(
 985                                 cp["constituent-cpd-id"][0][
 986                                     "constituent-base-element-id"
 987                                 ],
 988                                 cp["constituent-cpd-id"][0]["constituent-cpd-id"],
 989                             )
 990                         ] = "nsrs:{}:vld.{}".format(nsr_id, vld_index)
 991
 992             # check at nsd descriptor, if there is an ip-profile
 993             vld_params = {}
 994             nsd_vlp = find_in_list(
 995                 get_virtual_link_profiles(nsd),
 996                 lambda a_link_profile: a_link_profile["virtual-link-desc-id"]
 997                 == vld["id"],
 998             )
 999             if (
1000                 nsd_vlp
1001                 and nsd_vlp.get("virtual-link-protocol-data")
1002                 and nsd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
1003             ):
1004                 ip_profile_source_data = nsd_vlp["virtual-link-protocol-data"][
1005                     "l3-protocol-data"
1006                 ]
1007                 ip_profile_dest_data = {}
1008                 if "ip-version" in ip_profile_source_data:
1009                     ip_profile_dest_data["ip-version"] = ip_profile_source_data[
1010                         "ip-version"
1011                     ]
1012                 if "cidr" in ip_profile_source_data:
1013                     ip_profile_dest_data["subnet-address"] = ip_profile_source_data[
1014                         "cidr"
1015                     ]
1016                 if "gateway-ip" in ip_profile_source_data:
1017                     ip_profile_dest_data["gateway-address"] = ip_profile_source_data[
1018                         "gateway-ip"
1019                     ]
1020                 if "dhcp-enabled" in ip_profile_source_data:
1021                     ip_profile_dest_data["dhcp-params"] = {
1022                         "enabled": ip_profile_source_data["dhcp-enabled"]
1023                     }
1024                 vld_params["ip-profile"] = ip_profile_dest_data
1025
1026             # update vld_params with instantiation params
1027             vld_instantiation_params = find_in_list(
1028                 get_iterable(ns_params, "vld"),
1029                 lambda a_vld: a_vld["name"] in (vld["name"], vld["id"]),
1030             )
1031             if vld_instantiation_params:
1032                 vld_params.update(vld_instantiation_params)
1033             parse_vld_instantiation_params(target_vim, target_vld, vld_params, None)
1034             target["ns"]["vld"].append(target_vld)
1035         # Update the target ns_vld if vnf vim_account is overriden by instantiation params
1036         update_ns_vld_target(target, ns_params)
1037
1038         for vnfr in db_vnfrs.values():
1039             vnfd = find_in_list(
1040                 db_vnfds, lambda db_vnf: db_vnf["id"] == vnfr["vnfd-ref"]
1041             )
1042             vnf_params = find_in_list(
1043                 get_iterable(ns_params, "vnf"),
1044                 lambda a_vnf: a_vnf["member-vnf-index"] == vnfr["member-vnf-index-ref"],
1045             )
1046             target_vnf = deepcopy(vnfr)
1047             target_vim = "vim:{}".format(vnfr["vim-account-id"])
1048             for vld in target_vnf.get("vld", ()):
1049                 # check if connected to a ns.vld, to fill target'
1050                 vnf_cp = find_in_list(
1051                     vnfd.get("int-virtual-link-desc", ()),
1052                     lambda cpd: cpd.get("id") == vld["id"],
1053                 )
1054                 if vnf_cp:
1055                     ns_cp = "member_vnf:{}.{}".format(
1056                         vnfr["member-vnf-index-ref"], vnf_cp["id"]
1057                     )
1058                     if cp2target.get(ns_cp):
1059                         vld["target"] = cp2target[ns_cp]
1060
1061                 vld["vim_info"] = {
1062                     target_vim: {"vim_network_name": vld.get("vim-network-name")}
1063                 }
1064                 # check if this network needs SDN assist
1065                 target_sdn = None
1066                 if vld.get("pci-interfaces"):
1067                     db_vim = get_vim_account(vnfr["vim-account-id"])
1068                     sdnc_id = db_vim["config"].get("sdn-controller")
1069                     if sdnc_id:
1070                         sdn_vld = "vnfrs:{}:vld.{}".format(target_vnf["_id"], vld["id"])
1071                         target_sdn = "sdn:{}".format(sdnc_id)
1072                         vld["vim_info"][target_sdn] = {
1073                             "sdn": True,
1074                             "target_vim": target_vim,
1075                             "vlds": [sdn_vld],
1076                             "type": vld.get("type"),
1077                         }
1078
1079                 # check at vnfd descriptor, if there is an ip-profile
1080                 vld_params = {}
1081                 vnfd_vlp = find_in_list(
1082                     get_virtual_link_profiles(vnfd),
1083                     lambda a_link_profile: a_link_profile["id"] == vld["id"],
1084                 )
1085                 if (
1086                     vnfd_vlp
1087                     and vnfd_vlp.get("virtual-link-protocol-data")
1088                     and vnfd_vlp["virtual-link-protocol-data"].get("l3-protocol-data")
1089                 ):
1090                     ip_profile_source_data = vnfd_vlp["virtual-link-protocol-data"][
1091                         "l3-protocol-data"
1092                     ]
1093                     ip_profile_dest_data = {}
1094                     if "ip-version" in ip_profile_source_data:
1095                         ip_profile_dest_data["ip-version"] = ip_profile_source_data[
1096                             "ip-version"
1097                         ]
1098                     if "cidr" in ip_profile_source_data:
1099                         ip_profile_dest_data["subnet-address"] = ip_profile_source_data[
1100                             "cidr"
1101                         ]
1102                     if "gateway-ip" in ip_profile_source_data:
1103                         ip_profile_dest_data[
1104                             "gateway-address"
1105                         ] = ip_profile_source_data["gateway-ip"]
1106                     if "dhcp-enabled" in ip_profile_source_data:
1107                         ip_profile_dest_data["dhcp-params"] = {
1108                             "enabled": ip_profile_source_data["dhcp-enabled"]
1109                         }
1110
1111                     vld_params["ip-profile"] = ip_profile_dest_data
1112                 # update vld_params with instantiation params
1113                 if vnf_params:
1114                     vld_instantiation_params = find_in_list(
1115                         get_iterable(vnf_params, "internal-vld"),
1116                         lambda i_vld: i_vld["name"] == vld["id"],
1117                     )
1118                     if vld_instantiation_params:
1119                         vld_params.update(vld_instantiation_params)
1120                 parse_vld_instantiation_params(target_vim, vld, vld_params, target_sdn)
1121
1122             vdur_list = []
1123             for vdur in target_vnf.get("vdur", ()):
1124                 if vdur.get("status") == "DELETING" or vdur.get("pdu-type"):
1125                     continue  # This vdu must not be created
1126                 vdur["vim_info"] = {"vim_account_id": vnfr["vim-account-id"]}
1127
1128                 self.logger.debug("NS > ssh_keys > {}".format(ssh_keys_all))
1129
1130                 if ssh_keys_all:
1131                     vdu_configuration = get_configuration(vnfd, vdur["vdu-id-ref"])
1132                     vnf_configuration = get_configuration(vnfd, vnfd["id"])
1133                     if (
1134                         vdu_configuration
1135                         and vdu_configuration.get("config-access")
1136                         and vdu_configuration.get("config-access").get("ssh-access")
1137                     ):
1138                         vdur["ssh-keys"] = ssh_keys_all
1139                         vdur["ssh-access-required"] = vdu_configuration[
1140                             "config-access"
1141                         ]["ssh-access"]["required"]
1142                     elif (
1143                         vnf_configuration
1144                         and vnf_configuration.get("config-access")
1145                         and vnf_configuration.get("config-access").get("ssh-access")
1146                         and any(iface.get("mgmt-vnf") for iface in vdur["interfaces"])
1147                     ):
1148                         vdur["ssh-keys"] = ssh_keys_all
1149                         vdur["ssh-access-required"] = vnf_configuration[
1150                             "config-access"
1151                         ]["ssh-access"]["required"]
1152                     elif ssh_keys_instantiation and find_in_list(
1153                         vdur["interfaces"], lambda iface: iface.get("mgmt-vnf")
1154                     ):
1155                         vdur["ssh-keys"] = ssh_keys_instantiation
1156
1157                 self.logger.debug("NS > vdur > {}".format(vdur))
1158
1159                 vdud = get_vdu(vnfd, vdur["vdu-id-ref"])
1160                 # cloud-init
1161                 if vdud.get("cloud-init-file"):
1162                     vdur["cloud-init"] = "{}:file:{}".format(
1163                         vnfd["_id"], vdud.get("cloud-init-file")
1164                     )
1165                     # read file and put content at target.cloul_init_content. Avoid ng_ro to use shared package system
1166                     if vdur["cloud-init"] not in target["cloud_init_content"]:
1167                         base_folder = vnfd["_admin"]["storage"]
1168                         if base_folder["pkg-dir"]:
1169                             cloud_init_file = "{}/{}/cloud_init/{}".format(
1170                                 base_folder["folder"],
1171                                 base_folder["pkg-dir"],
1172                                 vdud.get("cloud-init-file"),
1173                             )
1174                         else:
1175                             cloud_init_file = "{}/Scripts/cloud_init/{}".format(
1176                                 base_folder["folder"],
1177                                 vdud.get("cloud-init-file"),
1178                             )
1179                         with self.fs.file_open(cloud_init_file, "r") as ci_file:
1180                             target["cloud_init_content"][
1181                                 vdur["cloud-init"]
1182                             ] = ci_file.read()
1183                 elif vdud.get("cloud-init"):
1184                     vdur["cloud-init"] = "{}:vdu:{}".format(
1185                         vnfd["_id"], get_vdu_index(vnfd, vdur["vdu-id-ref"])
1186                     )
1187                     # put content at target.cloul_init_content. Avoid ng_ro read vnfd descriptor
1188                     target["cloud_init_content"][vdur["cloud-init"]] = vdud[
1189                         "cloud-init"
1190                     ]
1191                 vdur["additionalParams"] = vdur.get("additionalParams") or {}
1192                 deploy_params_vdu = self._format_additional_params(
1193                     vdur.get("additionalParams") or {}
1194                 )
1195                 deploy_params_vdu["OSM"] = get_osm_params(
1196                     vnfr, vdur["vdu-id-ref"], vdur["count-index"]
1197                 )
1198                 vdur["additionalParams"] = deploy_params_vdu
1199
1200                 # flavor
1201                 ns_flavor = target["flavor"][int(vdur["ns-flavor-id"])]
1202                 if target_vim not in ns_flavor["vim_info"]:
1203                     ns_flavor["vim_info"][target_vim] = {}
1204
1205                 # deal with images
1206                 # in case alternative images are provided we must check if they should be applied
1207                 # for the vim_type, modify the vim_type taking into account
1208                 ns_image_id = int(vdur["ns-image-id"])
1209                 if vdur.get("alt-image-ids"):
1210                     db_vim = get_vim_account(vnfr["vim-account-id"])
1211                     vim_type = db_vim["vim_type"]
1212                     for alt_image_id in vdur.get("alt-image-ids"):
1213                         ns_alt_image = target["image"][int(alt_image_id)]
1214                         if vim_type == ns_alt_image.get("vim-type"):
1215                             # must use alternative image
1216                             self.logger.debug(
1217                                 "use alternative image id: {}".format(alt_image_id)
1218                             )
1219                             ns_image_id = alt_image_id
1220                             vdur["ns-image-id"] = ns_image_id
1221                             break
1222                 ns_image = target["image"][int(ns_image_id)]
1223                 if target_vim not in ns_image["vim_info"]:
1224                     ns_image["vim_info"][target_vim] = {}
1225
1226                 # Affinity groups
1227                 if vdur.get("affinity-or-anti-affinity-group-id"):
1228                     for ags_id in vdur["affinity-or-anti-affinity-group-id"]:
1229                         ns_ags = target["affinity-or-anti-affinity-group"][int(ags_id)]
1230                         if target_vim not in ns_ags["vim_info"]:
1231                             ns_ags["vim_info"][target_vim] = {}
1232
1233                 vdur["vim_info"] = {target_vim: {}}
1234                 # instantiation parameters
1235                 if vnf_params:
1236                     vdu_instantiation_params = find_in_list(
1237                         get_iterable(vnf_params, "vdu"),
1238                         lambda i_vdu: i_vdu["id"] == vdud["id"],
1239                     )
1240                     if vdu_instantiation_params:
1241                         # Parse the vdu_volumes from the instantiation params
1242                         vdu_volumes = get_volumes_from_instantiation_params(
1243                             vdu_instantiation_params, vdud
1244                         )
1245                         vdur["additionalParams"]["OSM"]["vdu_volumes"] = vdu_volumes
1246                 vdur_list.append(vdur)
1247             target_vnf["vdur"] = vdur_list
1248             target["vnf"].append(target_vnf)
1249
1250         self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
1251         desc = await self.RO.deploy(nsr_id, target)
1252         self.logger.debug("RO return > {}".format(desc))
1253         action_id = desc["action_id"]
1254         await self._wait_ng_ro(
1255             nsr_id, action_id, nslcmop_id, start_deploy, timeout_ns_deploy, stage,
1256             operation="instantiation"
1257         )
1258
1259         # Updating NSR
1260         db_nsr_update = {
1261             "_admin.deployed.RO.operational-status": "running",
1262             "detailed-status": " ".join(stage),
1263         }
1264         # db_nsr["_admin.deployed.RO.detailed-status"] = "Deployed at VIM"
1265         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1266         self._write_op_status(nslcmop_id, stage)
1267         self.logger.debug(
1268             logging_text + "ns deployed at RO. RO_id={}".format(action_id)
1269         )
1270         return
1271
1272     async def _wait_ng_ro(
1273         self,
1274         nsr_id,
1275         action_id,
1276         nslcmop_id=None,
1277         start_time=None,
1278         timeout=600,
1279         stage=None,
1280         operation=None,
1281     ):
1282         detailed_status_old = None
1283         db_nsr_update = {}
1284         start_time = start_time or time()
1285         while time() <= start_time + timeout:
1286             desc_status = await self.op_status_map[operation](nsr_id, action_id)
1287             self.logger.debug("Wait NG RO > {}".format(desc_status))
1288             if desc_status["status"] == "FAILED":
1289                 raise NgRoException(desc_status["details"])
1290             elif desc_status["status"] == "BUILD":
1291                 if stage:
1292                     stage[2] = "VIM: ({})".format(desc_status["details"])
1293             elif desc_status["status"] == "DONE":
1294                 if stage:
1295                     stage[2] = "Deployed at VIM"
1296                 break
1297             else:
1298                 assert False, "ROclient.check_ns_status returns unknown {}".format(
1299                     desc_status["status"]
1300                 )
1301             if stage and nslcmop_id and stage[2] != detailed_status_old:
1302                 detailed_status_old = stage[2]
1303                 db_nsr_update["detailed-status"] = " ".join(stage)
1304                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
1305                 self._write_op_status(nslcmop_id, stage)
1306             await asyncio.sleep(15, loop=self.loop)
1307         else:  # timeout_ns_deploy
1308             raise NgRoException("Timeout waiting ns to deploy")
1309
1310     async def _terminate_ng_ro(
1311         self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
1312     ):
1313         db_nsr_update = {}
1314         failed_detail = []
1315         action_id = None
1316         start_deploy = time()
1317         try:
1318             target = {
1319                 "ns": {"vld": []},
1320                 "vnf": [],
1321                 "image": [],
1322                 "flavor": [],
1323                 "action_id": nslcmop_id,
1324             }
1325             desc = await self.RO.deploy(nsr_id, target)
1326             action_id = desc["action_id"]
1327             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = action_id
1328             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETING"
1329             self.logger.debug(
1330                 logging_text
1331                 + "ns terminate action at RO. action_id={}".format(action_id)
1332             )
1333
1334             # wait until done
1335             delete_timeout = 20 * 60  # 20 minutes
1336             await self._wait_ng_ro(
1337                 nsr_id, action_id, nslcmop_id, start_deploy, delete_timeout, stage,
1338                 operation="termination"
1339             )
1340
1341             db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1342             db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1343             # delete all nsr
1344             await self.RO.delete(nsr_id)
1345         except Exception as e:
1346             if isinstance(e, NgRoException) and e.http_code == 404:  # not found
1347                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
1348                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
1349                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
1350                 self.logger.debug(
1351                     logging_text + "RO_action_id={} already deleted".format(action_id)
1352                 )
1353             elif isinstance(e, NgRoException) and e.http_code == 409:  # conflict
1354                 failed_detail.append("delete conflict: {}".format(e))
1355                 self.logger.debug(
1356                     logging_text
1357                     + "RO_action_id={} delete conflict: {}".format(action_id, e)
1358                 )
1359             else:
1360                 failed_detail.append("delete error: {}".format(e))
1361                 self.logger.error(
1362                     logging_text
1363                     + "RO_action_id={} delete error: {}".format(action_id, e)
1364                 )
1365
1366         if failed_detail:
1367             stage[2] = "Error deleting from VIM"
1368         else:
1369             stage[2] = "Deleted from VIM"
1370         db_nsr_update["detailed-status"] = " ".join(stage)
1371         self.update_db_2("nsrs", nsr_id, db_nsr_update)
1372         self._write_op_status(nslcmop_id, stage)
1373
1374         if failed_detail:
1375             raise LcmException("; ".join(failed_detail))
1376         return
1377
1378     async def instantiate_RO(
1379         self,
1380         logging_text,
1381         nsr_id,
1382         nsd,
1383         db_nsr,
1384         db_nslcmop,
1385         db_vnfrs,
1386         db_vnfds,
1387         n2vc_key_list,
1388         stage,
1389     ):
1390         """
1391         Instantiate at RO
1392         :param logging_text: preffix text to use at logging
1393         :param nsr_id: nsr identity
1394         :param nsd: database content of ns descriptor
1395         :param db_nsr: database content of ns record
1396         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
1397         :param db_vnfrs:
1398         :param db_vnfds: database content of vnfds, indexed by id (not _id). {id: {vnfd_object}, ...}
1399         :param n2vc_key_list: ssh-public-key list to be inserted to management vdus via cloud-init
1400         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
1401         :return: None or exception
1402         """
1403         try:
1404             start_deploy = time()
1405             ns_params = db_nslcmop.get("operationParams")
1406             if ns_params and ns_params.get("timeout_ns_deploy"):
1407                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
1408             else:
1409                 timeout_ns_deploy = self.timeout.get(
1410                     "ns_deploy", self.timeout_ns_deploy
1411                 )
1412
1413             # Check for and optionally request placement optimization. Database will be updated if placement activated
1414             stage[2] = "Waiting for Placement."
1415             if await self._do_placement(logging_text, db_nslcmop, db_vnfrs):
1416                 # in case of placement change ns_params[vimAcountId) if not present at any vnfrs
1417                 for vnfr in db_vnfrs.values():
1418                     if ns_params["vimAccountId"] == vnfr["vim-account-id"]:
1419                         break
1420                 else:
1421                     ns_params["vimAccountId"] == vnfr["vim-account-id"]
1422
1423             return await self._instantiate_ng_ro(
1424                 logging_text,
1425                 nsr_id,
1426                 nsd,
1427                 db_nsr,
1428                 db_nslcmop,
1429                 db_vnfrs,
1430                 db_vnfds,
1431                 n2vc_key_list,
1432                 stage,
1433                 start_deploy,
1434                 timeout_ns_deploy,
1435             )
1436         except Exception as e:
1437             stage[2] = "ERROR deploying at VIM"
1438             self.set_vnfr_at_error(db_vnfrs, str(e))
1439             self.logger.error(
1440                 "Error deploying at VIM {}".format(e),
1441                 exc_info=not isinstance(
1442                     e,
1443                     (
1444                         ROclient.ROClientException,
1445                         LcmException,
1446                         DbException,
1447                         NgRoException,
1448                     ),
1449                 ),
1450             )
1451             raise
1452
1453     async def wait_kdu_up(self, logging_text, nsr_id, vnfr_id, kdu_name):
1454         """
1455         Wait for kdu to be up, get ip address
1456         :param logging_text: prefix use for logging
1457         :param nsr_id:
1458         :param vnfr_id:
1459         :param kdu_name:
1460         :return: IP address, K8s services
1461         """
1462
1463         # self.logger.debug(logging_text + "Starting wait_kdu_up")
1464         nb_tries = 0
1465
1466         while nb_tries < 360:
1467             db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1468             kdur = next(
1469                 (
1470                     x
1471                     for x in get_iterable(db_vnfr, "kdur")
1472                     if x.get("kdu-name") == kdu_name
1473                 ),
1474                 None,
1475             )
1476             if not kdur:
1477                 raise LcmException(
1478                     "Not found vnfr_id={}, kdu_name={}".format(vnfr_id, kdu_name)
1479                 )
1480             if kdur.get("status"):
1481                 if kdur["status"] in ("READY", "ENABLED"):
1482                     return kdur.get("ip-address"), kdur.get("services")
1483                 else:
1484                     raise LcmException(
1485                         "target KDU={} is in error state".format(kdu_name)
1486                     )
1487
1488             await asyncio.sleep(10, loop=self.loop)
1489             nb_tries += 1
1490         raise LcmException("Timeout waiting KDU={} instantiated".format(kdu_name))
1491
1492     async def wait_vm_up_insert_key_ro(
1493         self, logging_text, nsr_id, vnfr_id, vdu_id, vdu_index, pub_key=None, user=None
1494     ):
1495         """
1496         Wait for ip addres at RO, and optionally, insert public key in virtual machine
1497         :param logging_text: prefix use for logging
1498         :param nsr_id:
1499         :param vnfr_id:
1500         :param vdu_id:
1501         :param vdu_index:
1502         :param pub_key: public ssh key to inject, None to skip
1503         :param user: user to apply the public ssh key
1504         :return: IP address
1505         """
1506
1507         self.logger.debug(logging_text + "Starting wait_vm_up_insert_key_ro")
1508         ro_nsr_id = None
1509         ip_address = None
1510         nb_tries = 0
1511         target_vdu_id = None
1512         ro_retries = 0
1513
1514         while True:
1515
1516             ro_retries += 1
1517             if ro_retries >= 360:  # 1 hour
1518                 raise LcmException(
1519                     "Not found _admin.deployed.RO.nsr_id for nsr_id: {}".format(nsr_id)
1520                 )
1521
1522             await asyncio.sleep(10, loop=self.loop)
1523
1524             # get ip address
1525             if not target_vdu_id:
1526                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnfr_id})
1527
1528                 if not vdu_id:  # for the VNF case
1529                     if db_vnfr.get("status") == "ERROR":
1530                         raise LcmException(
1531                             "Cannot inject ssh-key because target VNF is in error state"
1532                         )
1533                     ip_address = db_vnfr.get("ip-address")
1534                     if not ip_address:
1535                         continue
1536                     vdur = next(
1537                         (
1538                             x
1539                             for x in get_iterable(db_vnfr, "vdur")
1540                             if x.get("ip-address") == ip_address
1541                         ),
1542                         None,
1543                     )
1544                 else:  # VDU case
1545                     vdur = next(
1546                         (
1547                             x
1548                             for x in get_iterable(db_vnfr, "vdur")
1549                             if x.get("vdu-id-ref") == vdu_id
1550                             and x.get("count-index") == vdu_index
1551                         ),
1552                         None,
1553                     )
1554
1555                 if (
1556                     not vdur and len(db_vnfr.get("vdur", ())) == 1
1557                 ):  # If only one, this should be the target vdu
1558                     vdur = db_vnfr["vdur"][0]
1559                 if not vdur:
1560                     raise LcmException(
1561                         "Not found vnfr_id={}, vdu_id={}, vdu_index={}".format(
1562                             vnfr_id, vdu_id, vdu_index
1563                         )
1564                     )
1565                 # New generation RO stores information at "vim_info"
1566                 ng_ro_status = None
1567                 target_vim = None
1568                 if vdur.get("vim_info"):
1569                     target_vim = next(
1570                         t for t in vdur["vim_info"]
1571                     )  # there should be only one key
1572                     ng_ro_status = vdur["vim_info"][target_vim].get("vim_status")
1573                 if (
1574                     vdur.get("pdu-type")
1575                     or vdur.get("status") == "ACTIVE"
1576                     or ng_ro_status == "ACTIVE"
1577                 ):
1578                     ip_address = vdur.get("ip-address")
1579                     if not ip_address:
1580                         continue
1581                     target_vdu_id = vdur["vdu-id-ref"]
1582                 elif vdur.get("status") == "ERROR" or ng_ro_status == "ERROR":
1583                     raise LcmException(
1584                         "Cannot inject ssh-key because target VM is in error state"
1585                     )
1586
1587             if not target_vdu_id:
1588                 continue
1589
1590             # inject public key into machine
1591             if pub_key and user:
1592                 self.logger.debug(logging_text + "Inserting RO key")
1593                 self.logger.debug("SSH > PubKey > {}".format(pub_key))
1594                 if vdur.get("pdu-type"):
1595                     self.logger.error(logging_text + "Cannot inject ssh-ky to a PDU")
1596                     return ip_address
1597                 try:
1598                     ro_vm_id = "{}-{}".format(
1599                         db_vnfr["member-vnf-index-ref"], target_vdu_id
1600                     )  # TODO add vdu_index
1601                     if self.ng_ro:
1602                         target = {
1603                             "action": {
1604                                 "action": "inject_ssh_key",
1605                                 "key": pub_key,
1606                                 "user": user,
1607                             },
1608                             "vnf": [{"_id": vnfr_id, "vdur": [{"id": vdur["id"]}]}],
1609                         }
1610                         desc = await self.RO.deploy(nsr_id, target)
1611                         action_id = desc["action_id"]
1612                         await self._wait_ng_ro(nsr_id, action_id, timeout=600, operation="instantiation")
1613                         break
1614                     else:
1615                         # wait until NS is deployed at RO
1616                         if not ro_nsr_id:
1617                             db_nsrs = self.db.get_one("nsrs", {"_id": nsr_id})
1618                             ro_nsr_id = deep_get(
1619                                 db_nsrs, ("_admin", "deployed", "RO", "nsr_id")
1620                             )
1621                         if not ro_nsr_id:
1622                             continue
1623                         result_dict = await self.RO.create_action(
1624                             item="ns",
1625                             item_id_name=ro_nsr_id,
1626                             descriptor={
1627                                 "add_public_key": pub_key,
1628                                 "vms": [ro_vm_id],
1629                                 "user": user,
1630                             },
1631                         )
1632                         # result_dict contains the format {VM-id: {vim_result: 200, description: text}}
1633                         if not result_dict or not isinstance(result_dict, dict):
1634                             raise LcmException(
1635                                 "Unknown response from RO when injecting key"
1636                             )
1637                         for result in result_dict.values():
1638                             if result.get("vim_result") == 200:
1639                                 break
1640                             else:
1641                                 raise ROclient.ROClientException(
1642                                     "error injecting key: {}".format(
1643                                         result.get("description")
1644                                     )
1645                                 )
1646                         break
1647                 except NgRoException as e:
1648                     raise LcmException(
1649                         "Reaching max tries injecting key. Error: {}".format(e)
1650                     )
1651                 except ROclient.ROClientException as e:
1652                     if not nb_tries:
1653                         self.logger.debug(
1654                             logging_text
1655                             + "error injecting key: {}. Retrying until {} seconds".format(
1656                                 e, 20 * 10
1657                             )
1658                         )
1659                     nb_tries += 1
1660                     if nb_tries >= 20:
1661                         raise LcmException(
1662                             "Reaching max tries injecting key. Error: {}".format(e)
1663                         )
1664             else:
1665                 break
1666
1667         return ip_address
1668
1669     async def _wait_dependent_n2vc(self, nsr_id, vca_deployed_list, vca_index):
1670         """
1671         Wait until dependent VCA deployments have been finished. NS wait for VNFs and VDUs. VNFs for VDUs
1672         """
1673         my_vca = vca_deployed_list[vca_index]
1674         if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
1675             # vdu or kdu: no dependencies
1676             return
1677         timeout = 300
1678         while timeout >= 0:
1679             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
1680             vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1681             configuration_status_list = db_nsr["configurationStatus"]
1682             for index, vca_deployed in enumerate(configuration_status_list):
1683                 if index == vca_index:
1684                     # myself
1685                     continue
1686                 if not my_vca.get("member-vnf-index") or (
1687                     vca_deployed.get("member-vnf-index")
1688                     == my_vca.get("member-vnf-index")
1689                 ):
1690                     internal_status = configuration_status_list[index].get("status")
1691                     if internal_status == "READY":
1692                         continue
1693                     elif internal_status == "BROKEN":
1694                         raise LcmException(
1695                             "Configuration aborted because dependent charm/s has failed"
1696                         )
1697                     else:
1698                         break
1699             else:
1700                 # no dependencies, return
1701                 return
1702             await asyncio.sleep(10)
1703             timeout -= 1
1704
1705         raise LcmException("Configuration aborted because dependent charm/s timeout")
1706
1707     def get_vca_id(self, db_vnfr: dict, db_nsr: dict):
1708         vca_id = None
1709         if db_vnfr:
1710             vca_id = deep_get(db_vnfr, ("vca-id",))
1711         elif db_nsr:
1712             vim_account_id = deep_get(db_nsr, ("instantiate_params", "vimAccountId"))
1713             vca_id = VimAccountDB.get_vim_account_with_id(vim_account_id).get("vca")
1714         return vca_id
1715
1716     async def instantiate_N2VC(
1717         self,
1718         logging_text,
1719         vca_index,
1720         nsi_id,
1721         db_nsr,
1722         db_vnfr,
1723         vdu_id,
1724         kdu_name,
1725         vdu_index,
1726         config_descriptor,
1727         deploy_params,
1728         base_folder,
1729         nslcmop_id,
1730         stage,
1731         vca_type,
1732         vca_name,
1733         ee_config_descriptor,
1734     ):
1735         nsr_id = db_nsr["_id"]
1736         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
1737         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
1738         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
1739         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
1740         db_dict = {
1741             "collection": "nsrs",
1742             "filter": {"_id": nsr_id},
1743             "path": db_update_entry,
1744         }
1745         step = ""
1746         try:
1747
1748             element_type = "NS"
1749             element_under_configuration = nsr_id
1750
1751             vnfr_id = None
1752             if db_vnfr:
1753                 vnfr_id = db_vnfr["_id"]
1754                 osm_config["osm"]["vnf_id"] = vnfr_id
1755
1756             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
1757
1758             if vca_type == "native_charm":
1759                 index_number = 0
1760             else:
1761                 index_number = vdu_index or 0
1762
1763             if vnfr_id:
1764                 element_type = "VNF"
1765                 element_under_configuration = vnfr_id
1766                 namespace += ".{}-{}".format(vnfr_id, index_number)
1767                 if vdu_id:
1768                     namespace += ".{}-{}".format(vdu_id, index_number)
1769                     element_type = "VDU"
1770                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
1771                     osm_config["osm"]["vdu_id"] = vdu_id
1772                 elif kdu_name:
1773                     namespace += ".{}".format(kdu_name)
1774                     element_type = "KDU"
1775                     element_under_configuration = kdu_name
1776                     osm_config["osm"]["kdu_name"] = kdu_name
1777
1778             # Get artifact path
1779             if base_folder["pkg-dir"]:
1780                 artifact_path = "{}/{}/{}/{}".format(
1781                     base_folder["folder"],
1782                     base_folder["pkg-dir"],
1783                     "charms"
1784                     if vca_type
1785                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1786                     else "helm-charts",
1787                     vca_name,
1788                 )
1789             else:
1790                 artifact_path = "{}/Scripts/{}/{}/".format(
1791                     base_folder["folder"],
1792                     "charms"
1793                     if vca_type
1794                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
1795                     else "helm-charts",
1796                     vca_name,
1797                 )
1798
1799             self.logger.debug("Artifact path > {}".format(artifact_path))
1800
1801             # get initial_config_primitive_list that applies to this element
1802             initial_config_primitive_list = config_descriptor.get(
1803                 "initial-config-primitive"
1804             )
1805
1806             self.logger.debug(
1807                 "Initial config primitive list > {}".format(
1808                     initial_config_primitive_list
1809                 )
1810             )
1811
1812             # add config if not present for NS charm
1813             ee_descriptor_id = ee_config_descriptor.get("id")
1814             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
1815             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
1816                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
1817             )
1818
1819             self.logger.debug(
1820                 "Initial config primitive list #2 > {}".format(
1821                     initial_config_primitive_list
1822                 )
1823             )
1824             # n2vc_redesign STEP 3.1
1825             # find old ee_id if exists
1826             ee_id = vca_deployed.get("ee_id")
1827
1828             vca_id = self.get_vca_id(db_vnfr, db_nsr)
1829             # create or register execution environment in VCA
1830             if vca_type in ("lxc_proxy_charm", "k8s_proxy_charm", "helm", "helm-v3"):
1831
1832                 self._write_configuration_status(
1833                     nsr_id=nsr_id,
1834                     vca_index=vca_index,
1835                     status="CREATING",
1836                     element_under_configuration=element_under_configuration,
1837                     element_type=element_type,
1838                 )
1839
1840                 step = "create execution environment"
1841                 self.logger.debug(logging_text + step)
1842
1843                 ee_id = None
1844                 credentials = None
1845                 if vca_type == "k8s_proxy_charm":
1846                     ee_id = await self.vca_map[vca_type].install_k8s_proxy_charm(
1847                         charm_name=artifact_path[artifact_path.rfind("/") + 1 :],
1848                         namespace=namespace,
1849                         artifact_path=artifact_path,
1850                         db_dict=db_dict,
1851                         vca_id=vca_id,
1852                     )
1853                 elif vca_type == "helm" or vca_type == "helm-v3":
1854                     ee_id, credentials = await self.vca_map[
1855                         vca_type
1856                     ].create_execution_environment(
1857                         namespace=namespace,
1858                         reuse_ee_id=ee_id,
1859                         db_dict=db_dict,
1860                         config=osm_config,
1861                         artifact_path=artifact_path,
1862                         vca_type=vca_type,
1863                     )
1864                 else:
1865                     ee_id, credentials = await self.vca_map[
1866                         vca_type
1867                     ].create_execution_environment(
1868                         namespace=namespace,
1869                         reuse_ee_id=ee_id,
1870                         db_dict=db_dict,
1871                         vca_id=vca_id,
1872                     )
1873
1874             elif vca_type == "native_charm":
1875                 step = "Waiting to VM being up and getting IP address"
1876                 self.logger.debug(logging_text + step)
1877                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
1878                     logging_text,
1879                     nsr_id,
1880                     vnfr_id,
1881                     vdu_id,
1882                     vdu_index,
1883                     user=None,
1884                     pub_key=None,
1885                 )
1886                 credentials = {"hostname": rw_mgmt_ip}
1887                 # get username
1888                 username = deep_get(
1889                     config_descriptor, ("config-access", "ssh-access", "default-user")
1890                 )
1891                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
1892                 #  merged. Meanwhile let's get username from initial-config-primitive
1893                 if not username and initial_config_primitive_list:
1894                     for config_primitive in initial_config_primitive_list:
1895                         for param in config_primitive.get("parameter", ()):
1896                             if param["name"] == "ssh-username":
1897                                 username = param["value"]
1898                                 break
1899                 if not username:
1900                     raise LcmException(
1901                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
1902                         "'config-access.ssh-access.default-user'"
1903                     )
1904                 credentials["username"] = username
1905                 # n2vc_redesign STEP 3.2
1906
1907                 self._write_configuration_status(
1908                     nsr_id=nsr_id,
1909                     vca_index=vca_index,
1910                     status="REGISTERING",
1911                     element_under_configuration=element_under_configuration,
1912                     element_type=element_type,
1913                 )
1914
1915                 step = "register execution environment {}".format(credentials)
1916                 self.logger.debug(logging_text + step)
1917                 ee_id = await self.vca_map[vca_type].register_execution_environment(
1918                     credentials=credentials,
1919                     namespace=namespace,
1920                     db_dict=db_dict,
1921                     vca_id=vca_id,
1922                 )
1923
1924             # for compatibility with MON/POL modules, the need model and application name at database
1925             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
1926             ee_id_parts = ee_id.split(".")
1927             db_nsr_update = {db_update_entry + "ee_id": ee_id}
1928             if len(ee_id_parts) >= 2:
1929                 model_name = ee_id_parts[0]
1930                 application_name = ee_id_parts[1]
1931                 db_nsr_update[db_update_entry + "model"] = model_name
1932                 db_nsr_update[db_update_entry + "application"] = application_name
1933
1934             # n2vc_redesign STEP 3.3
1935             step = "Install configuration Software"
1936
1937             self._write_configuration_status(
1938                 nsr_id=nsr_id,
1939                 vca_index=vca_index,
1940                 status="INSTALLING SW",
1941                 element_under_configuration=element_under_configuration,
1942                 element_type=element_type,
1943                 other_update=db_nsr_update,
1944             )
1945
1946             # TODO check if already done
1947             self.logger.debug(logging_text + step)
1948             config = None
1949             if vca_type == "native_charm":
1950                 config_primitive = next(
1951                     (p for p in initial_config_primitive_list if p["name"] == "config"),
1952                     None,
1953                 )
1954                 if config_primitive:
1955                     config = self._map_primitive_params(
1956                         config_primitive, {}, deploy_params
1957                     )
1958             num_units = 1
1959             if vca_type == "lxc_proxy_charm":
1960                 if element_type == "NS":
1961                     num_units = db_nsr.get("config-units") or 1
1962                 elif element_type == "VNF":
1963                     num_units = db_vnfr.get("config-units") or 1
1964                 elif element_type == "VDU":
1965                     for v in db_vnfr["vdur"]:
1966                         if vdu_id == v["vdu-id-ref"]:
1967                             num_units = v.get("config-units") or 1
1968                             break
1969             if vca_type != "k8s_proxy_charm":
1970                 await self.vca_map[vca_type].install_configuration_sw(
1971                     ee_id=ee_id,
1972                     artifact_path=artifact_path,
1973                     db_dict=db_dict,
1974                     config=config,
1975                     num_units=num_units,
1976                     vca_id=vca_id,
1977                     vca_type=vca_type,
1978                 )
1979
1980             # write in db flag of configuration_sw already installed
1981             self.update_db_2(
1982                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
1983             )
1984
1985             # add relations for this VCA (wait for other peers related with this VCA)
1986             await self._add_vca_relations(
1987                 logging_text=logging_text,
1988                 nsr_id=nsr_id,
1989                 vca_type=vca_type,
1990                 vca_index=vca_index,
1991             )
1992
1993             # if SSH access is required, then get execution environment SSH public
1994             # if native charm we have waited already to VM be UP
1995             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm", "helm-v3"):
1996                 pub_key = None
1997                 user = None
1998                 # self.logger.debug("get ssh key block")
1999                 if deep_get(
2000                     config_descriptor, ("config-access", "ssh-access", "required")
2001                 ):
2002                     # self.logger.debug("ssh key needed")
2003                     # Needed to inject a ssh key
2004                     user = deep_get(
2005                         config_descriptor,
2006                         ("config-access", "ssh-access", "default-user"),
2007                     )
2008                     step = "Install configuration Software, getting public ssh key"
2009                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
2010                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
2011                     )
2012
2013                     step = "Insert public key into VM user={} ssh_key={}".format(
2014                         user, pub_key
2015                     )
2016                 else:
2017                     # self.logger.debug("no need to get ssh key")
2018                     step = "Waiting to VM being up and getting IP address"
2019                 self.logger.debug(logging_text + step)
2020
2021                 # default rw_mgmt_ip to None, avoiding the non definition of the variable
2022                 rw_mgmt_ip = None
2023
2024                 # n2vc_redesign STEP 5.1
2025                 # wait for RO (ip-address) Insert pub_key into VM
2026                 if vnfr_id:
2027                     if kdu_name:
2028                         rw_mgmt_ip, services = await self.wait_kdu_up(
2029                             logging_text, nsr_id, vnfr_id, kdu_name
2030                         )
2031                         vnfd = self.db.get_one(
2032                             "vnfds_revisions",
2033                             {"_id": f'{db_vnfr["vnfd-id"]}:{db_vnfr["revision"]}'},
2034                         )
2035                         kdu = get_kdu(vnfd, kdu_name)
2036                         kdu_services = [
2037                             service["name"] for service in get_kdu_services(kdu)
2038                         ]
2039                         exposed_services = []
2040                         for service in services:
2041                             if any(s in service["name"] for s in kdu_services):
2042                                 exposed_services.append(service)
2043                         await self.vca_map[vca_type].exec_primitive(
2044                             ee_id=ee_id,
2045                             primitive_name="config",
2046                             params_dict={
2047                                 "osm-config": json.dumps(
2048                                     OsmConfigBuilder(
2049                                         k8s={"services": exposed_services}
2050                                     ).build()
2051                                 )
2052                             },
2053                             vca_id=vca_id,
2054                         )
2055
2056                     # This verification is needed in order to avoid trying to add a public key
2057                     # to a VM, when the VNF is a KNF (in the edge case where the user creates a VCA
2058                     # for a KNF and not for its KDUs, the previous verification gives False, and the code
2059                     # jumps to this block, meaning that there is the need to verify if the VNF is actually a VNF
2060                     # or it is a KNF)
2061                     elif db_vnfr.get('vdur'):
2062                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
2063                             logging_text,
2064                             nsr_id,
2065                             vnfr_id,
2066                             vdu_id,
2067                             vdu_index,
2068                             user=user,
2069                             pub_key=pub_key,
2070                         )
2071
2072                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
2073
2074             # store rw_mgmt_ip in deploy params for later replacement
2075             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
2076
2077             # n2vc_redesign STEP 6  Execute initial config primitive
2078             step = "execute initial config primitive"
2079
2080             # wait for dependent primitives execution (NS -> VNF -> VDU)
2081             if initial_config_primitive_list:
2082                 await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
2083
2084             # stage, in function of element type: vdu, kdu, vnf or ns
2085             my_vca = vca_deployed_list[vca_index]
2086             if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
2087                 # VDU or KDU
2088                 stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
2089             elif my_vca.get("member-vnf-index"):
2090                 # VNF
2091                 stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
2092             else:
2093                 # NS
2094                 stage[0] = "Stage 5/5: running Day-1 primitives for NS."
2095
2096             self._write_configuration_status(
2097                 nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
2098             )
2099
2100             self._write_op_status(op_id=nslcmop_id, stage=stage)
2101
2102             check_if_terminated_needed = True
2103             for initial_config_primitive in initial_config_primitive_list:
2104                 # adding information on the vca_deployed if it is a NS execution environment
2105                 if not vca_deployed["member-vnf-index"]:
2106                     deploy_params["ns_config_info"] = json.dumps(
2107                         self._get_ns_config_info(nsr_id)
2108                     )
2109                 # TODO check if already done
2110                 primitive_params_ = self._map_primitive_params(
2111                     initial_config_primitive, {}, deploy_params
2112                 )
2113
2114                 step = "execute primitive '{}' params '{}'".format(
2115                     initial_config_primitive["name"], primitive_params_
2116                 )
2117                 self.logger.debug(logging_text + step)
2118                 await self.vca_map[vca_type].exec_primitive(
2119                     ee_id=ee_id,
2120                     primitive_name=initial_config_primitive["name"],
2121                     params_dict=primitive_params_,
2122                     db_dict=db_dict,
2123                     vca_id=vca_id,
2124                     vca_type=vca_type,
2125                 )
2126                 # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
2127                 if check_if_terminated_needed:
2128                     if config_descriptor.get("terminate-config-primitive"):
2129                         self.update_db_2(
2130                             "nsrs", nsr_id, {db_update_entry + "needed_terminate": True}
2131                         )
2132                     check_if_terminated_needed = False
2133
2134                 # TODO register in database that primitive is done
2135
2136             # STEP 7 Configure metrics
2137             if vca_type == "helm" or vca_type == "helm-v3":
2138                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
2139                     ee_id=ee_id,
2140                     artifact_path=artifact_path,
2141                     ee_config_descriptor=ee_config_descriptor,
2142                     vnfr_id=vnfr_id,
2143                     nsr_id=nsr_id,
2144                     target_ip=rw_mgmt_ip,
2145                 )
2146                 if prometheus_jobs:
2147                     self.update_db_2(
2148                         "nsrs",
2149                         nsr_id,
2150                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
2151                     )
2152
2153                     for job in prometheus_jobs:
2154                         self.db.set_one(
2155                             "prometheus_jobs",
2156                             {"job_name": job["job_name"]},
2157                             job,
2158                             upsert=True,
2159                             fail_on_empty=False,
2160                         )
2161
2162             step = "instantiated at VCA"
2163             self.logger.debug(logging_text + step)
2164
2165             self._write_configuration_status(
2166                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
2167             )
2168
2169         except Exception as e:  # TODO not use Exception but N2VC exception
2170             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
2171             if not isinstance(
2172                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
2173             ):
2174                 self.logger.error(
2175                     "Exception while {} : {}".format(step, e), exc_info=True
2176                 )
2177             self._write_configuration_status(
2178                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
2179             )
2180             raise LcmException("{} {}".format(step, e)) from e
2181
2182     def _write_ns_status(
2183         self,
2184         nsr_id: str,
2185         ns_state: str,
2186         current_operation: str,
2187         current_operation_id: str,
2188         error_description: str = None,
2189         error_detail: str = None,
2190         other_update: dict = None,
2191     ):
2192         """
2193         Update db_nsr fields.
2194         :param nsr_id:
2195         :param ns_state:
2196         :param current_operation:
2197         :param current_operation_id:
2198         :param error_description:
2199         :param error_detail:
2200         :param other_update: Other required changes at database if provided, will be cleared
2201         :return:
2202         """
2203         try:
2204             db_dict = other_update or {}
2205             db_dict[
2206                 "_admin.nslcmop"
2207             ] = current_operation_id  # for backward compatibility
2208             db_dict["_admin.current-operation"] = current_operation_id
2209             db_dict["_admin.operation-type"] = (
2210                 current_operation if current_operation != "IDLE" else None
2211             )
2212             db_dict["currentOperation"] = current_operation
2213             db_dict["currentOperationID"] = current_operation_id
2214             db_dict["errorDescription"] = error_description
2215             db_dict["errorDetail"] = error_detail
2216
2217             if ns_state:
2218                 db_dict["nsState"] = ns_state
2219             self.update_db_2("nsrs", nsr_id, db_dict)
2220         except DbException as e:
2221             self.logger.warn("Error writing NS status, ns={}: {}".format(nsr_id, e))
2222
2223     def _write_op_status(
2224         self,
2225         op_id: str,
2226         stage: list = None,
2227         error_message: str = None,
2228         queuePosition: int = 0,
2229         operation_state: str = None,
2230         other_update: dict = None,
2231     ):
2232         try:
2233             db_dict = other_update or {}
2234             db_dict["queuePosition"] = queuePosition
2235             if isinstance(stage, list):
2236                 db_dict["stage"] = stage[0]
2237                 db_dict["detailed-status"] = " ".join(stage)
2238             elif stage is not None:
2239                 db_dict["stage"] = str(stage)
2240
2241             if error_message is not None:
2242                 db_dict["errorMessage"] = error_message
2243             if operation_state is not None:
2244                 db_dict["operationState"] = operation_state
2245                 db_dict["statusEnteredTime"] = time()
2246             self.update_db_2("nslcmops", op_id, db_dict)
2247         except DbException as e:
2248             self.logger.warn(
2249                 "Error writing OPERATION status for op_id: {} -> {}".format(op_id, e)
2250             )
2251
2252     def _write_all_config_status(self, db_nsr: dict, status: str):
2253         try:
2254             nsr_id = db_nsr["_id"]
2255             # configurationStatus
2256             config_status = db_nsr.get("configurationStatus")
2257             if config_status:
2258                 db_nsr_update = {
2259                     "configurationStatus.{}.status".format(index): status
2260                     for index, v in enumerate(config_status)
2261                     if v
2262                 }
2263                 # update status
2264                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
2265
2266         except DbException as e:
2267             self.logger.warn(
2268                 "Error writing all configuration status, ns={}: {}".format(nsr_id, e)
2269             )
2270
2271     def _write_configuration_status(
2272         self,
2273         nsr_id: str,
2274         vca_index: int,
2275         status: str = None,
2276         element_under_configuration: str = None,
2277         element_type: str = None,
2278         other_update: dict = None,
2279     ):
2280
2281         # self.logger.debug('_write_configuration_status(): vca_index={}, status={}'
2282         #                   .format(vca_index, status))
2283
2284         try:
2285             db_path = "configurationStatus.{}.".format(vca_index)
2286             db_dict = other_update or {}
2287             if status:
2288                 db_dict[db_path + "status"] = status
2289             if element_under_configuration:
2290                 db_dict[
2291                     db_path + "elementUnderConfiguration"
2292                 ] = element_under_configuration
2293             if element_type:
2294                 db_dict[db_path + "elementType"] = element_type
2295             self.update_db_2("nsrs", nsr_id, db_dict)
2296         except DbException as e:
2297             self.logger.warn(
2298                 "Error writing configuration status={}, ns={}, vca_index={}: {}".format(
2299                     status, nsr_id, vca_index, e
2300                 )
2301             )
2302
2303     async def _do_placement(self, logging_text, db_nslcmop, db_vnfrs):
2304         """
2305         Check and computes the placement, (vim account where to deploy). If it is decided by an external tool, it
2306         sends the request via kafka and wait until the result is wrote at database (nslcmops _admin.plca).
2307         Database is used because the result can be obtained from a different LCM worker in case of HA.
2308         :param logging_text: contains the prefix for logging, with the ns and nslcmop identifiers
2309         :param db_nslcmop: database content of nslcmop
2310         :param db_vnfrs: database content of vnfrs, indexed by member-vnf-index.
2311         :return: True if some modification is done. Modifies database vnfrs and parameter db_vnfr with the
2312             computed 'vim-account-id'
2313         """
2314         modified = False
2315         nslcmop_id = db_nslcmop["_id"]
2316         placement_engine = deep_get(db_nslcmop, ("operationParams", "placement-engine"))
2317         if placement_engine == "PLA":
2318             self.logger.debug(
2319                 logging_text + "Invoke and wait for placement optimization"
2320             )
2321             await self.msg.aiowrite(
2322                 "pla", "get_placement", {"nslcmopId": nslcmop_id}, loop=self.loop
2323             )
2324             db_poll_interval = 5
2325             wait = db_poll_interval * 10
2326             pla_result = None
2327             while not pla_result and wait >= 0:
2328                 await asyncio.sleep(db_poll_interval)
2329                 wait -= db_poll_interval
2330                 db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2331                 pla_result = deep_get(db_nslcmop, ("_admin", "pla"))
2332
2333             if not pla_result:
2334                 raise LcmException(
2335                     "Placement timeout for nslcmopId={}".format(nslcmop_id)
2336                 )
2337
2338             for pla_vnf in pla_result["vnf"]:
2339                 vnfr = db_vnfrs.get(pla_vnf["member-vnf-index"])
2340                 if not pla_vnf.get("vimAccountId") or not vnfr:
2341                     continue
2342                 modified = True
2343                 self.db.set_one(
2344                     "vnfrs",
2345                     {"_id": vnfr["_id"]},
2346                     {"vim-account-id": pla_vnf["vimAccountId"]},
2347                 )
2348                 # Modifies db_vnfrs
2349                 vnfr["vim-account-id"] = pla_vnf["vimAccountId"]
2350         return modified
2351
2352     def update_nsrs_with_pla_result(self, params):
2353         try:
2354             nslcmop_id = deep_get(params, ("placement", "nslcmopId"))
2355             self.update_db_2(
2356                 "nslcmops", nslcmop_id, {"_admin.pla": params.get("placement")}
2357             )
2358         except Exception as e:
2359             self.logger.warn("Update failed for nslcmop_id={}:{}".format(nslcmop_id, e))
2360
2361     async def instantiate(self, nsr_id, nslcmop_id):
2362         """
2363
2364         :param nsr_id: ns instance to deploy
2365         :param nslcmop_id: operation to run
2366         :return:
2367         """
2368
2369         # Try to lock HA task here
2370         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
2371         if not task_is_locked_by_me:
2372             self.logger.debug(
2373                 "instantiate() task is not locked by me, ns={}".format(nsr_id)
2374             )
2375             return
2376
2377         logging_text = "Task ns={} instantiate={} ".format(nsr_id, nslcmop_id)
2378         self.logger.debug(logging_text + "Enter")
2379
2380         # get all needed from database
2381
2382         # database nsrs record
2383         db_nsr = None
2384
2385         # database nslcmops record
2386         db_nslcmop = None
2387
2388         # update operation on nsrs
2389         db_nsr_update = {}
2390         # update operation on nslcmops
2391         db_nslcmop_update = {}
2392
2393         nslcmop_operation_state = None
2394         db_vnfrs = {}  # vnf's info indexed by member-index
2395         # n2vc_info = {}
2396         tasks_dict_info = {}  # from task to info text
2397         exc = None
2398         error_list = []
2399         stage = [
2400             "Stage 1/5: preparation of the environment.",
2401             "Waiting for previous operations to terminate.",
2402             "",
2403         ]
2404         # ^ stage, step, VIM progress
2405         try:
2406             # wait for any previous tasks in process
2407             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
2408
2409             # STEP 0: Reading database (nslcmops, nsrs, nsds, vnfrs, vnfds)
2410             stage[1] = "Reading from database."
2411             # nsState="BUILDING", currentOperation="INSTANTIATING", currentOperationID=nslcmop_id
2412             db_nsr_update["detailed-status"] = "creating"
2413             db_nsr_update["operational-status"] = "init"
2414             self._write_ns_status(
2415                 nsr_id=nsr_id,
2416                 ns_state="BUILDING",
2417                 current_operation="INSTANTIATING",
2418                 current_operation_id=nslcmop_id,
2419                 other_update=db_nsr_update,
2420             )
2421             self._write_op_status(op_id=nslcmop_id, stage=stage, queuePosition=0)
2422
2423             # read from db: operation
2424             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
2425             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
2426             if db_nslcmop["operationParams"].get("additionalParamsForVnf"):
2427                 db_nslcmop["operationParams"]["additionalParamsForVnf"] = json.loads(
2428                     db_nslcmop["operationParams"]["additionalParamsForVnf"]
2429                 )
2430             ns_params = db_nslcmop.get("operationParams")
2431             if ns_params and ns_params.get("timeout_ns_deploy"):
2432                 timeout_ns_deploy = ns_params["timeout_ns_deploy"]
2433             else:
2434                 timeout_ns_deploy = self.timeout.get(
2435                     "ns_deploy", self.timeout_ns_deploy
2436                 )
2437
2438             # read from db: ns
2439             stage[1] = "Getting nsr={} from db.".format(nsr_id)
2440             self.logger.debug(logging_text + stage[1])
2441             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
2442             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
2443             self.logger.debug(logging_text + stage[1])
2444             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
2445             self.fs.sync(db_nsr["nsd-id"])
2446             db_nsr["nsd"] = nsd
2447             # nsr_name = db_nsr["name"]   # TODO short-name??
2448
2449             # read from db: vnf's of this ns
2450             stage[1] = "Getting vnfrs from db."
2451             self.logger.debug(logging_text + stage[1])
2452             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
2453
2454             # read from db: vnfd's for every vnf
2455             db_vnfds = []  # every vnfd data
2456
2457             # for each vnf in ns, read vnfd
2458             for vnfr in db_vnfrs_list:
2459                 if vnfr.get("kdur"):
2460                     kdur_list = []
2461                     for kdur in vnfr["kdur"]:
2462                         if kdur.get("additionalParams"):
2463                             kdur["additionalParams"] = json.loads(
2464                                 kdur["additionalParams"]
2465                             )
2466                         kdur_list.append(kdur)
2467                     vnfr["kdur"] = kdur_list
2468
2469                 db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
2470                 vnfd_id = vnfr["vnfd-id"]
2471                 vnfd_ref = vnfr["vnfd-ref"]
2472                 self.fs.sync(vnfd_id)
2473
2474                 # if we haven't this vnfd, read it from db
2475                 if vnfd_id not in db_vnfds:
2476                     # read from db
2477                     stage[1] = "Getting vnfd={} id='{}' from db.".format(
2478                         vnfd_id, vnfd_ref
2479                     )
2480                     self.logger.debug(logging_text + stage[1])
2481                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
2482
2483                     # store vnfd
2484                     db_vnfds.append(vnfd)
2485
2486             # Get or generates the _admin.deployed.VCA list
2487             vca_deployed_list = None
2488             if db_nsr["_admin"].get("deployed"):
2489                 vca_deployed_list = db_nsr["_admin"]["deployed"].get("VCA")
2490             if vca_deployed_list is None:
2491                 vca_deployed_list = []
2492                 configuration_status_list = []
2493                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2494                 db_nsr_update["configurationStatus"] = configuration_status_list
2495                 # add _admin.deployed.VCA to db_nsr dictionary, value=vca_deployed_list
2496                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2497             elif isinstance(vca_deployed_list, dict):
2498                 # maintain backward compatibility. Change a dict to list at database
2499                 vca_deployed_list = list(vca_deployed_list.values())
2500                 db_nsr_update["_admin.deployed.VCA"] = vca_deployed_list
2501                 populate_dict(db_nsr, ("_admin", "deployed", "VCA"), vca_deployed_list)
2502
2503             if not isinstance(
2504                 deep_get(db_nsr, ("_admin", "deployed", "RO", "vnfd")), list
2505             ):
2506                 populate_dict(db_nsr, ("_admin", "deployed", "RO", "vnfd"), [])
2507                 db_nsr_update["_admin.deployed.RO.vnfd"] = []
2508
2509             # set state to INSTANTIATED. When instantiated NBI will not delete directly
2510             db_nsr_update["_admin.nsState"] = "INSTANTIATED"
2511             self.update_db_2("nsrs", nsr_id, db_nsr_update)
2512             self.db.set_list(
2513                 "vnfrs", {"nsr-id-ref": nsr_id}, {"_admin.nsState": "INSTANTIATED"}
2514             )
2515
2516             # n2vc_redesign STEP 2 Deploy Network Scenario
2517             stage[0] = "Stage 2/5: deployment of KDUs, VMs and execution environments."
2518             self._write_op_status(op_id=nslcmop_id, stage=stage)
2519
2520             stage[1] = "Deploying KDUs."
2521             # self.logger.debug(logging_text + "Before deploy_kdus")
2522             # Call to deploy_kdus in case exists the "vdu:kdu" param
2523             await self.deploy_kdus(
2524                 logging_text=logging_text,
2525                 nsr_id=nsr_id,
2526                 nslcmop_id=nslcmop_id,
2527                 db_vnfrs=db_vnfrs,
2528                 db_vnfds=db_vnfds,
2529                 task_instantiation_info=tasks_dict_info,
2530             )
2531
2532             stage[1] = "Getting VCA public key."
2533             # n2vc_redesign STEP 1 Get VCA public ssh-key
2534             # feature 1429. Add n2vc public key to needed VMs
2535             n2vc_key = self.n2vc.get_public_key()
2536             n2vc_key_list = [n2vc_key]
2537             if self.vca_config.get("public_key"):
2538                 n2vc_key_list.append(self.vca_config["public_key"])
2539
2540             stage[1] = "Deploying NS at VIM."
2541             task_ro = asyncio.ensure_future(
2542                 self.instantiate_RO(
2543                     logging_text=logging_text,
2544                     nsr_id=nsr_id,
2545                     nsd=nsd,
2546                     db_nsr=db_nsr,
2547                     db_nslcmop=db_nslcmop,
2548                     db_vnfrs=db_vnfrs,
2549                     db_vnfds=db_vnfds,
2550                     n2vc_key_list=n2vc_key_list,
2551                     stage=stage,
2552                 )
2553             )
2554             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "instantiate_RO", task_ro)
2555             tasks_dict_info[task_ro] = "Deploying at VIM"
2556
2557             # n2vc_redesign STEP 3 to 6 Deploy N2VC
2558             stage[1] = "Deploying Execution Environments."
2559             self.logger.debug(logging_text + stage[1])
2560
2561             nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
2562             for vnf_profile in get_vnf_profiles(nsd):
2563                 vnfd_id = vnf_profile["vnfd-id"]
2564                 vnfd = find_in_list(db_vnfds, lambda a_vnf: a_vnf["id"] == vnfd_id)
2565                 member_vnf_index = str(vnf_profile["id"])
2566                 db_vnfr = db_vnfrs[member_vnf_index]
2567                 base_folder = vnfd["_admin"]["storage"]
2568                 vdu_id = None
2569                 vdu_index = 0
2570                 vdu_name = None
2571                 kdu_name = None
2572
2573                 # Get additional parameters
2574                 deploy_params = {"OSM": get_osm_params(db_vnfr)}
2575                 if db_vnfr.get("additionalParamsForVnf"):
2576                     deploy_params.update(
2577                         parse_yaml_strings(db_vnfr["additionalParamsForVnf"].copy())
2578                     )
2579
2580                 descriptor_config = get_configuration(vnfd, vnfd["id"])
2581                 if descriptor_config:
2582                     self._deploy_n2vc(
2583                         logging_text=logging_text
2584                         + "member_vnf_index={} ".format(member_vnf_index),
2585                         db_nsr=db_nsr,
2586                         db_vnfr=db_vnfr,
2587                         nslcmop_id=nslcmop_id,
2588                         nsr_id=nsr_id,
2589                         nsi_id=nsi_id,
2590                         vnfd_id=vnfd_id,
2591                         vdu_id=vdu_id,
2592                         kdu_name=kdu_name,
2593                         member_vnf_index=member_vnf_index,
2594                         vdu_index=vdu_index,
2595                         vdu_name=vdu_name,
2596                         deploy_params=deploy_params,
2597                         descriptor_config=descriptor_config,
2598                         base_folder=base_folder,
2599                         task_instantiation_info=tasks_dict_info,
2600                         stage=stage,
2601                     )
2602
2603                 # Deploy charms for each VDU that supports one.
2604                 for vdud in get_vdu_list(vnfd):
2605                     vdu_id = vdud["id"]
2606                     descriptor_config = get_configuration(vnfd, vdu_id)
2607                     vdur = find_in_list(
2608                         db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
2609                     )
2610
2611                     if vdur.get("additionalParams"):
2612                         deploy_params_vdu = parse_yaml_strings(vdur["additionalParams"])
2613                     else:
2614                         deploy_params_vdu = deploy_params
2615                     deploy_params_vdu["OSM"] = get_osm_params(
2616                         db_vnfr, vdu_id, vdu_count_index=0
2617                     )
2618                     vdud_count = get_number_of_instances(vnfd, vdu_id)
2619
2620                     self.logger.debug("VDUD > {}".format(vdud))
2621                     self.logger.debug(
2622                         "Descriptor config > {}".format(descriptor_config)
2623                     )
2624                     if descriptor_config:
2625                         vdu_name = None
2626                         kdu_name = None
2627                         for vdu_index in range(vdud_count):
2628                             # TODO vnfr_params["rw_mgmt_ip"] = vdur["ip-address"]
2629                             self._deploy_n2vc(
2630                                 logging_text=logging_text
2631                                 + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
2632                                     member_vnf_index, vdu_id, vdu_index
2633                                 ),
2634                                 db_nsr=db_nsr,
2635                                 db_vnfr=db_vnfr,
2636                                 nslcmop_id=nslcmop_id,
2637                                 nsr_id=nsr_id,
2638                                 nsi_id=nsi_id,
2639                                 vnfd_id=vnfd_id,
2640                                 vdu_id=vdu_id,
2641                                 kdu_name=kdu_name,
2642                                 member_vnf_index=member_vnf_index,
2643                                 vdu_index=vdu_index,
2644                                 vdu_name=vdu_name,
2645                                 deploy_params=deploy_params_vdu,
2646                                 descriptor_config=descriptor_config,
2647                                 base_folder=base_folder,
2648                                 task_instantiation_info=tasks_dict_info,
2649                                 stage=stage,
2650                             )
2651                 for kdud in get_kdu_list(vnfd):
2652                     kdu_name = kdud["name"]
2653                     descriptor_config = get_configuration(vnfd, kdu_name)
2654                     if descriptor_config:
2655                         vdu_id = None
2656                         vdu_index = 0
2657                         vdu_name = None
2658                         kdur = next(
2659                             x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name
2660                         )
2661                         deploy_params_kdu = {"OSM": get_osm_params(db_vnfr)}
2662                         if kdur.get("additionalParams"):
2663                             deploy_params_kdu.update(
2664                                 parse_yaml_strings(kdur["additionalParams"].copy())
2665                             )
2666
2667                         self._deploy_n2vc(
2668                             logging_text=logging_text,
2669                             db_nsr=db_nsr,
2670                             db_vnfr=db_vnfr,
2671                             nslcmop_id=nslcmop_id,
2672                             nsr_id=nsr_id,
2673                             nsi_id=nsi_id,
2674                             vnfd_id=vnfd_id,
2675                             vdu_id=vdu_id,
2676                             kdu_name=kdu_name,
2677                             member_vnf_index=member_vnf_index,
2678                             vdu_index=vdu_index,
2679                             vdu_name=vdu_name,
2680                             deploy_params=deploy_params_kdu,
2681                             descriptor_config=descriptor_config,
2682                             base_folder=base_folder,
2683                             task_instantiation_info=tasks_dict_info,
2684                             stage=stage,
2685                         )
2686
2687             # Check if this NS has a charm configuration
2688             descriptor_config = nsd.get("ns-configuration")
2689             if descriptor_config and descriptor_config.get("juju"):
2690                 vnfd_id = None
2691                 db_vnfr = None
2692                 member_vnf_index = None
2693                 vdu_id = None
2694                 kdu_name = None
2695                 vdu_index = 0
2696                 vdu_name = None
2697
2698                 # Get additional parameters
2699                 deploy_params = {"OSM": {"vim_account_id": ns_params["vimAccountId"]}}
2700                 if db_nsr.get("additionalParamsForNs"):
2701                     deploy_params.update(
2702                         parse_yaml_strings(db_nsr["additionalParamsForNs"].copy())
2703                     )
2704                 base_folder = nsd["_admin"]["storage"]
2705                 self._deploy_n2vc(
2706                     logging_text=logging_text,
2707                     db_nsr=db_nsr,
2708                     db_vnfr=db_vnfr,
2709                     nslcmop_id=nslcmop_id,
2710                     nsr_id=nsr_id,
2711                     nsi_id=nsi_id,
2712                     vnfd_id=vnfd_id,
2713                     vdu_id=vdu_id,
2714                     kdu_name=kdu_name,
2715                     member_vnf_index=member_vnf_index,
2716                     vdu_index=vdu_index,
2717                     vdu_name=vdu_name,
2718                     deploy_params=deploy_params,
2719                     descriptor_config=descriptor_config,
2720                     base_folder=base_folder,
2721                     task_instantiation_info=tasks_dict_info,
2722                     stage=stage,
2723                 )
2724
2725             # rest of staff will be done at finally
2726
2727         except (
2728             ROclient.ROClientException,
2729             DbException,
2730             LcmException,
2731             N2VCException,
2732         ) as e:
2733             self.logger.error(
2734                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e)
2735             )
2736             exc = e
2737         except asyncio.CancelledError:
2738             self.logger.error(
2739                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
2740             )
2741             exc = "Operation was cancelled"
2742         except Exception as e:
2743             exc = traceback.format_exc()
2744             self.logger.critical(
2745                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
2746                 exc_info=True,
2747             )
2748         finally:
2749             if exc:
2750                 error_list.append(str(exc))
2751             try:
2752                 # wait for pending tasks
2753                 if tasks_dict_info:
2754                     stage[1] = "Waiting for instantiate pending tasks."
2755                     self.logger.debug(logging_text + stage[1])
2756                     error_list += await self._wait_for_tasks(
2757                         logging_text,
2758                         tasks_dict_info,
2759                         timeout_ns_deploy,
2760                         stage,
2761                         nslcmop_id,
2762                         nsr_id=nsr_id,
2763                     )
2764                 stage[1] = stage[2] = ""
2765             except asyncio.CancelledError:
2766                 error_list.append("Cancelled")
2767                 # TODO cancel all tasks
2768             except Exception as exc:
2769                 error_list.append(str(exc))
2770
2771             # update operation-status
2772             db_nsr_update["operational-status"] = "running"
2773             # let's begin with VCA 'configured' status (later we can change it)
2774             db_nsr_update["config-status"] = "configured"
2775             for task, task_name in tasks_dict_info.items():
2776                 if not task.done() or task.cancelled() or task.exception():
2777                     if task_name.startswith(self.task_name_deploy_vca):
2778                         # A N2VC task is pending
2779                         db_nsr_update["config-status"] = "failed"
2780                     else:
2781                         # RO or KDU task is pending
2782                         db_nsr_update["operational-status"] = "failed"
2783
2784             # update status at database
2785             if error_list:
2786                 error_detail = ". ".join(error_list)
2787                 self.logger.error(logging_text + error_detail)
2788                 error_description_nslcmop = "{} Detail: {}".format(
2789                     stage[0], error_detail
2790                 )
2791                 error_description_nsr = "Operation: INSTANTIATING.{}, {}".format(
2792                     nslcmop_id, stage[0]
2793                 )
2794
2795                 db_nsr_update["detailed-status"] = (
2796                     error_description_nsr + " Detail: " + error_detail
2797                 )
2798                 db_nslcmop_update["detailed-status"] = error_detail
2799                 nslcmop_operation_state = "FAILED"
2800                 ns_state = "BROKEN"
2801             else:
2802                 error_detail = None
2803                 error_description_nsr = error_description_nslcmop = None
2804                 ns_state = "READY"
2805                 db_nsr_update["detailed-status"] = "Done"
2806                 db_nslcmop_update["detailed-status"] = "Done"
2807                 nslcmop_operation_state = "COMPLETED"
2808
2809             if db_nsr:
2810                 self._write_ns_status(
2811                     nsr_id=nsr_id,
2812                     ns_state=ns_state,
2813                     current_operation="IDLE",
2814                     current_operation_id=None,
2815                     error_description=error_description_nsr,
2816                     error_detail=error_detail,
2817                     other_update=db_nsr_update,
2818                 )
2819             self._write_op_status(
2820                 op_id=nslcmop_id,
2821                 stage="",
2822                 error_message=error_description_nslcmop,
2823                 operation_state=nslcmop_operation_state,
2824                 other_update=db_nslcmop_update,
2825             )
2826
2827             if nslcmop_operation_state:
2828                 try:
2829                     await self.msg.aiowrite(
2830                         "ns",
2831                         "instantiated",
2832                         {
2833                             "nsr_id": nsr_id,
2834                             "nslcmop_id": nslcmop_id,
2835                             "operationState": nslcmop_operation_state,
2836                         },
2837                         loop=self.loop,
2838                     )
2839                 except Exception as e:
2840                     self.logger.error(
2841                         logging_text + "kafka_write notification Exception {}".format(e)
2842                     )
2843
2844             self.logger.debug(logging_text + "Exit")
2845             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_instantiate")
2846
2847     def _get_vnfd(self, vnfd_id: str, cached_vnfds: Dict[str, Any]):
2848         if vnfd_id not in cached_vnfds:
2849             cached_vnfds[vnfd_id] = self.db.get_one("vnfds", {"id": vnfd_id})
2850         return cached_vnfds[vnfd_id]
2851
2852     def _get_vnfr(self, nsr_id: str, vnf_profile_id: str, cached_vnfrs: Dict[str, Any]):
2853         if vnf_profile_id not in cached_vnfrs:
2854             cached_vnfrs[vnf_profile_id] = self.db.get_one(
2855                 "vnfrs",
2856                 {
2857                     "member-vnf-index-ref": vnf_profile_id,
2858                     "nsr-id-ref": nsr_id,
2859                 },
2860             )
2861         return cached_vnfrs[vnf_profile_id]
2862
2863     def _is_deployed_vca_in_relation(
2864         self, vca: DeployedVCA, relation: Relation
2865     ) -> bool:
2866         found = False
2867         for endpoint in (relation.provider, relation.requirer):
2868             if endpoint["kdu-resource-profile-id"]:
2869                 continue
2870             found = (
2871                 vca.vnf_profile_id == endpoint.vnf_profile_id
2872                 and vca.vdu_profile_id == endpoint.vdu_profile_id
2873                 and vca.execution_environment_ref == endpoint.execution_environment_ref
2874             )
2875             if found:
2876                 break
2877         return found
2878
2879     def _update_ee_relation_data_with_implicit_data(
2880         self, nsr_id, nsd, ee_relation_data, cached_vnfds, vnf_profile_id: str = None
2881     ):
2882         ee_relation_data = safe_get_ee_relation(
2883             nsr_id, ee_relation_data, vnf_profile_id=vnf_profile_id
2884         )
2885         ee_relation_level = EELevel.get_level(ee_relation_data)
2886         if (ee_relation_level in (EELevel.VNF, EELevel.VDU)) and not ee_relation_data[
2887             "execution-environment-ref"
2888         ]:
2889             vnf_profile = get_vnf_profile(nsd, ee_relation_data["vnf-profile-id"])
2890             vnfd_id = vnf_profile["vnfd-id"]
2891             db_vnfd = self._get_vnfd(vnfd_id, cached_vnfds)
2892             entity_id = (
2893                 vnfd_id
2894                 if ee_relation_level == EELevel.VNF
2895                 else ee_relation_data["vdu-profile-id"]
2896             )
2897             ee = get_juju_ee_ref(db_vnfd, entity_id)
2898             if not ee:
2899                 raise Exception(
2900                     f"not execution environments found for ee_relation {ee_relation_data}"
2901                 )
2902             ee_relation_data["execution-environment-ref"] = ee["id"]
2903         return ee_relation_data
2904
2905     def _get_ns_relations(
2906         self,
2907         nsr_id: str,
2908         nsd: Dict[str, Any],
2909         vca: DeployedVCA,
2910         cached_vnfds: Dict[str, Any],
2911     ) -> List[Relation]:
2912         relations = []
2913         db_ns_relations = get_ns_configuration_relation_list(nsd)
2914         for r in db_ns_relations:
2915             provider_dict = None
2916             requirer_dict = None
2917             if all(key in r for key in ("provider", "requirer")):
2918                 provider_dict = r["provider"]
2919                 requirer_dict = r["requirer"]
2920             elif "entities" in r:
2921                 provider_id = r["entities"][0]["id"]
2922                 provider_dict = {
2923                     "nsr-id": nsr_id,
2924                     "endpoint": r["entities"][0]["endpoint"],
2925                 }
2926                 if provider_id != nsd["id"]:
2927                     provider_dict["vnf-profile-id"] = provider_id
2928                 requirer_id = r["entities"][1]["id"]
2929                 requirer_dict = {
2930                     "nsr-id": nsr_id,
2931                     "endpoint": r["entities"][1]["endpoint"],
2932                 }
2933                 if requirer_id != nsd["id"]:
2934                     requirer_dict["vnf-profile-id"] = requirer_id
2935             else:
2936                 raise Exception(
2937                     "provider/requirer or entities must be included in the relation."
2938                 )
2939             relation_provider = self._update_ee_relation_data_with_implicit_data(
2940                 nsr_id, nsd, provider_dict, cached_vnfds
2941             )
2942             relation_requirer = self._update_ee_relation_data_with_implicit_data(
2943                 nsr_id, nsd, requirer_dict, cached_vnfds
2944             )
2945             provider = EERelation(relation_provider)
2946             requirer = EERelation(relation_requirer)
2947             relation = Relation(r["name"], provider, requirer)
2948             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
2949             if vca_in_relation:
2950                 relations.append(relation)
2951         return relations
2952
2953     def _get_vnf_relations(
2954         self,
2955         nsr_id: str,
2956         nsd: Dict[str, Any],
2957         vca: DeployedVCA,
2958         cached_vnfds: Dict[str, Any],
2959     ) -> List[Relation]:
2960         relations = []
2961         vnf_profile = get_vnf_profile(nsd, vca.vnf_profile_id)
2962         vnf_profile_id = vnf_profile["id"]
2963         vnfd_id = vnf_profile["vnfd-id"]
2964         db_vnfd = self._get_vnfd(vnfd_id, cached_vnfds)
2965         db_vnf_relations = get_relation_list(db_vnfd, vnfd_id)
2966         for r in db_vnf_relations:
2967             provider_dict = None
2968             requirer_dict = None
2969             if all(key in r for key in ("provider", "requirer")):
2970                 provider_dict = r["provider"]
2971                 requirer_dict = r["requirer"]
2972             elif "entities" in r:
2973                 provider_id = r["entities"][0]["id"]
2974                 provider_dict = {
2975                     "nsr-id": nsr_id,
2976                     "vnf-profile-id": vnf_profile_id,
2977                     "endpoint": r["entities"][0]["endpoint"],
2978                 }
2979                 if provider_id != vnfd_id:
2980                     provider_dict["vdu-profile-id"] = provider_id
2981                 requirer_id = r["entities"][1]["id"]
2982                 requirer_dict = {
2983                     "nsr-id": nsr_id,
2984                     "vnf-profile-id": vnf_profile_id,
2985                     "endpoint": r["entities"][1]["endpoint"],
2986                 }
2987                 if requirer_id != vnfd_id:
2988                     requirer_dict["vdu-profile-id"] = requirer_id
2989             else:
2990                 raise Exception(
2991                     "provider/requirer or entities must be included in the relation."
2992                 )
2993             relation_provider = self._update_ee_relation_data_with_implicit_data(
2994                 nsr_id, nsd, provider_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
2995             )
2996             relation_requirer = self._update_ee_relation_data_with_implicit_data(
2997                 nsr_id, nsd, requirer_dict, cached_vnfds, vnf_profile_id=vnf_profile_id
2998             )
2999             provider = EERelation(relation_provider)
3000             requirer = EERelation(relation_requirer)
3001             relation = Relation(r["name"], provider, requirer)
3002             vca_in_relation = self._is_deployed_vca_in_relation(vca, relation)
3003             if vca_in_relation:
3004                 relations.append(relation)
3005         return relations
3006
3007     def _get_kdu_resource_data(
3008         self,
3009         ee_relation: EERelation,
3010         db_nsr: Dict[str, Any],
3011         cached_vnfds: Dict[str, Any],
3012     ) -> DeployedK8sResource:
3013         nsd = get_nsd(db_nsr)
3014         vnf_profiles = get_vnf_profiles(nsd)
3015         vnfd_id = find_in_list(
3016             vnf_profiles,
3017             lambda vnf_profile: vnf_profile["id"] == ee_relation.vnf_profile_id,
3018         )["vnfd-id"]
3019         db_vnfd = self._get_vnfd(vnfd_id, cached_vnfds)
3020         kdu_resource_profile = get_kdu_resource_profile(
3021             db_vnfd, ee_relation.kdu_resource_profile_id
3022         )
3023         kdu_name = kdu_resource_profile["kdu-name"]
3024         deployed_kdu, _ = get_deployed_kdu(
3025             db_nsr.get("_admin", ()).get("deployed", ()),
3026             kdu_name,
3027             ee_relation.vnf_profile_id,
3028         )
3029         deployed_kdu.update({"resource-name": kdu_resource_profile["resource-name"]})
3030         return deployed_kdu
3031
3032     def _get_deployed_component(
3033         self,
3034         ee_relation: EERelation,
3035         db_nsr: Dict[str, Any],
3036         cached_vnfds: Dict[str, Any],
3037     ) -> DeployedComponent:
3038         nsr_id = db_nsr["_id"]
3039         deployed_component = None
3040         ee_level = EELevel.get_level(ee_relation)
3041         if ee_level == EELevel.NS:
3042             vca = get_deployed_vca(db_nsr, {"vdu_id": None, "member-vnf-index": None})
3043             if vca:
3044                 deployed_component = DeployedVCA(nsr_id, vca)
3045         elif ee_level == EELevel.VNF:
3046             vca = get_deployed_vca(
3047                 db_nsr,
3048                 {
3049                     "vdu_id": None,
3050                     "member-vnf-index": ee_relation.vnf_profile_id,
3051                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3052                 },
3053             )
3054             if vca:
3055                 deployed_component = DeployedVCA(nsr_id, vca)
3056         elif ee_level == EELevel.VDU:
3057             vca = get_deployed_vca(
3058                 db_nsr,
3059                 {
3060                     "vdu_id": ee_relation.vdu_profile_id,
3061                     "member-vnf-index": ee_relation.vnf_profile_id,
3062                     "ee_descriptor_id": ee_relation.execution_environment_ref,
3063                 },
3064             )
3065             if vca:
3066                 deployed_component = DeployedVCA(nsr_id, vca)
3067         elif ee_level == EELevel.KDU:
3068             kdu_resource_data = self._get_kdu_resource_data(
3069                 ee_relation, db_nsr, cached_vnfds
3070             )
3071             if kdu_resource_data:
3072                 deployed_component = DeployedK8sResource(kdu_resource_data)
3073         return deployed_component
3074
3075     async def _add_relation(
3076         self,
3077         relation: Relation,
3078         vca_type: str,
3079         db_nsr: Dict[str, Any],
3080         cached_vnfds: Dict[str, Any],
3081         cached_vnfrs: Dict[str, Any],
3082     ) -> bool:
3083         deployed_provider = self._get_deployed_component(
3084             relation.provider, db_nsr, cached_vnfds
3085         )
3086         deployed_requirer = self._get_deployed_component(
3087             relation.requirer, db_nsr, cached_vnfds
3088         )
3089         if (
3090             deployed_provider
3091             and deployed_requirer
3092             and deployed_provider.config_sw_installed
3093             and deployed_requirer.config_sw_installed
3094         ):
3095             provider_db_vnfr = (
3096                 self._get_vnfr(
3097                     relation.provider.nsr_id,
3098                     relation.provider.vnf_profile_id,
3099                     cached_vnfrs,
3100                 )
3101                 if relation.provider.vnf_profile_id
3102                 else None
3103             )
3104             requirer_db_vnfr = (
3105                 self._get_vnfr(
3106                     relation.requirer.nsr_id,
3107                     relation.requirer.vnf_profile_id,
3108                     cached_vnfrs,
3109                 )
3110                 if relation.requirer.vnf_profile_id
3111                 else None
3112             )
3113             provider_vca_id = self.get_vca_id(provider_db_vnfr, db_nsr)
3114             requirer_vca_id = self.get_vca_id(requirer_db_vnfr, db_nsr)
3115             provider_relation_endpoint = RelationEndpoint(
3116                 deployed_provider.ee_id,
3117                 provider_vca_id,
3118                 relation.provider.endpoint,
3119             )
3120             requirer_relation_endpoint = RelationEndpoint(
3121                 deployed_requirer.ee_id,
3122                 requirer_vca_id,
3123                 relation.requirer.endpoint,
3124             )
3125             await self.vca_map[vca_type].add_relation(
3126                 provider=provider_relation_endpoint,
3127                 requirer=requirer_relation_endpoint,
3128             )
3129             # remove entry from relations list
3130             return True
3131         return False
3132
3133     async def _add_vca_relations(
3134         self,
3135         logging_text,
3136         nsr_id,
3137         vca_type: str,
3138         vca_index: int,
3139         timeout: int = 3600,
3140     ) -> bool:
3141
3142         # steps:
3143         # 1. find all relations for this VCA
3144         # 2. wait for other peers related
3145         # 3. add relations
3146
3147         try:
3148             # STEP 1: find all relations for this VCA
3149
3150             # read nsr record
3151             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3152             nsd = get_nsd(db_nsr)
3153
3154             # this VCA data
3155             deployed_vca_dict = get_deployed_vca_list(db_nsr)[vca_index]
3156             my_vca = DeployedVCA(nsr_id, deployed_vca_dict)
3157
3158             cached_vnfds = {}
3159             cached_vnfrs = {}
3160             relations = []
3161             relations.extend(self._get_ns_relations(nsr_id, nsd, my_vca, cached_vnfds))
3162             relations.extend(self._get_vnf_relations(nsr_id, nsd, my_vca, cached_vnfds))
3163
3164             # if no relations, terminate
3165             if not relations:
3166                 self.logger.debug(logging_text + " No relations")
3167                 return True
3168
3169             self.logger.debug(logging_text + " adding relations {}".format(relations))
3170
3171             # add all relations
3172             start = time()
3173             while True:
3174                 # check timeout
3175                 now = time()
3176                 if now - start >= timeout:
3177                     self.logger.error(logging_text + " : timeout adding relations")
3178                     return False
3179
3180                 # reload nsr from database (we need to update record: _admin.deployed.VCA)
3181                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
3182
3183                 # for each relation, find the VCA's related
3184                 for relation in relations.copy():
3185                     added = await self._add_relation(
3186                         relation,
3187                         vca_type,
3188                         db_nsr,
3189                         cached_vnfds,
3190                         cached_vnfrs,
3191                     )
3192                     if added:
3193                         relations.remove(relation)
3194
3195                 if not relations:
3196                     self.logger.debug("Relations added")
3197                     break
3198                 await asyncio.sleep(5.0)
3199
3200             return True
3201
3202         except Exception as e:
3203             self.logger.warn(logging_text + " ERROR adding relations: {}".format(e))
3204             return False
3205
3206     async def _install_kdu(
3207         self,
3208         nsr_id: str,
3209         nsr_db_path: str,
3210         vnfr_data: dict,
3211         kdu_index: int,
3212         kdud: dict,
3213         vnfd: dict,
3214         k8s_instance_info: dict,
3215         k8params: dict = None,
3216         timeout: int = 600,
3217         vca_id: str = None,
3218     ):
3219
3220         try:
3221             k8sclustertype = k8s_instance_info["k8scluster-type"]
3222             # Instantiate kdu
3223             db_dict_install = {
3224                 "collection": "nsrs",
3225                 "filter": {"_id": nsr_id},
3226                 "path": nsr_db_path,
3227             }
3228
3229             if k8s_instance_info.get("kdu-deployment-name"):
3230                 kdu_instance = k8s_instance_info.get("kdu-deployment-name")
3231             else:
3232                 kdu_instance = self.k8scluster_map[
3233                     k8sclustertype
3234                 ].generate_kdu_instance_name(
3235                     db_dict=db_dict_install,
3236                     kdu_model=k8s_instance_info["kdu-model"],
3237                     kdu_name=k8s_instance_info["kdu-name"],
3238                 )
3239
3240             # Update the nsrs table with the kdu-instance value
3241             self.update_db_2(
3242                 item="nsrs",
3243                 _id=nsr_id,
3244                 _desc={nsr_db_path + ".kdu-instance": kdu_instance},
3245             )
3246
3247             # Update the nsrs table with the actual namespace being used, if the k8scluster-type is `juju` or
3248             # `juju-bundle`. This verification is needed because there is not a standard/homogeneous namespace
3249             # between the Helm Charts and Juju Bundles-based KNFs. If we found a way of having an homogeneous
3250             # namespace, this first verification could be removed, and the next step would be done for any kind
3251             # of KNF.
3252             # TODO -> find a way to have an homogeneous namespace between the Helm Charts and Juju Bundles-based
3253             # KNFs (Bug 2027: https://osm.etsi.org/bugzilla/show_bug.cgi?id=2027)
3254             if k8sclustertype in ("juju", "juju-bundle"):
3255                 # First, verify if the current namespace is present in the `_admin.projects_read` (if not, it means
3256                 # that the user passed a namespace which he wants its KDU to be deployed in)
3257                 if (
3258                     self.db.count(
3259                         table="nsrs",
3260                         q_filter={
3261                             "_id": nsr_id,
3262                             "_admin.projects_write": k8s_instance_info["namespace"],
3263                             "_admin.projects_read": k8s_instance_info["namespace"],
3264                         },
3265                     )
3266                     > 0
3267                 ):
3268                     self.logger.debug(
3269                         f"Updating namespace/model for Juju Bundle from {k8s_instance_info['namespace']} to {kdu_instance}"
3270                     )
3271                     self.update_db_2(
3272                         item="nsrs",
3273                         _id=nsr_id,
3274                         _desc={f"{nsr_db_path}.namespace": kdu_instance},
3275                     )
3276                     k8s_instance_info["namespace"] = kdu_instance
3277
3278             await self.k8scluster_map[k8sclustertype].install(
3279                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3280                 kdu_model=k8s_instance_info["kdu-model"],
3281                 atomic=True,
3282                 params=k8params,
3283                 db_dict=db_dict_install,
3284                 timeout=timeout,
3285                 kdu_name=k8s_instance_info["kdu-name"],
3286                 namespace=k8s_instance_info["namespace"],
3287                 kdu_instance=kdu_instance,
3288                 vca_id=vca_id,
3289             )
3290
3291             # Obtain services to obtain management service ip
3292             services = await self.k8scluster_map[k8sclustertype].get_services(
3293                 cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3294                 kdu_instance=kdu_instance,
3295                 namespace=k8s_instance_info["namespace"],
3296             )
3297
3298             # Obtain management service info (if exists)
3299             vnfr_update_dict = {}
3300             kdu_config = get_configuration(vnfd, kdud["name"])
3301             if kdu_config:
3302                 target_ee_list = kdu_config.get("execution-environment-list", [])
3303             else:
3304                 target_ee_list = []
3305
3306             if services:
3307                 vnfr_update_dict["kdur.{}.services".format(kdu_index)] = services
3308                 mgmt_services = [
3309                     service
3310                     for service in kdud.get("service", [])
3311                     if service.get("mgmt-service")
3312                 ]
3313                 for mgmt_service in mgmt_services:
3314                     for service in services:
3315                         if service["name"].startswith(mgmt_service["name"]):
3316                             # Mgmt service found, Obtain service ip
3317                             ip = service.get("external_ip", service.get("cluster_ip"))
3318                             if isinstance(ip, list) and len(ip) == 1:
3319                                 ip = ip[0]
3320
3321                             vnfr_update_dict[
3322                                 "kdur.{}.ip-address".format(kdu_index)
3323                             ] = ip
3324
3325                             # Check if must update also mgmt ip at the vnf
3326                             service_external_cp = mgmt_service.get(
3327                                 "external-connection-point-ref"
3328                             )
3329                             if service_external_cp:
3330                                 if (
3331                                     deep_get(vnfd, ("mgmt-interface", "cp"))
3332                                     == service_external_cp
3333                                 ):
3334                                     vnfr_update_dict["ip-address"] = ip
3335
3336                                 if find_in_list(
3337                                     target_ee_list,
3338                                     lambda ee: ee.get(
3339                                         "external-connection-point-ref", ""
3340                                     )
3341                                     == service_external_cp,
3342                                 ):
3343                                     vnfr_update_dict[
3344                                         "kdur.{}.ip-address".format(kdu_index)
3345                                     ] = ip
3346                             break
3347                     else:
3348                         self.logger.warn(
3349                             "Mgmt service name: {} not found".format(
3350                                 mgmt_service["name"]
3351                             )
3352                         )
3353
3354             vnfr_update_dict["kdur.{}.status".format(kdu_index)] = "READY"
3355             self.update_db_2("vnfrs", vnfr_data.get("_id"), vnfr_update_dict)
3356
3357             kdu_config = get_configuration(vnfd, k8s_instance_info["kdu-name"])
3358             if (
3359                 kdu_config
3360                 and kdu_config.get("initial-config-primitive")
3361                 and get_juju_ee_ref(vnfd, k8s_instance_info["kdu-name"]) is None
3362             ):
3363                 initial_config_primitive_list = kdu_config.get(
3364                     "initial-config-primitive"
3365                 )
3366                 initial_config_primitive_list.sort(key=lambda val: int(val["seq"]))
3367
3368                 for initial_config_primitive in initial_config_primitive_list:
3369                     primitive_params_ = self._map_primitive_params(
3370                         initial_config_primitive, {}, {}
3371                     )
3372
3373                     await asyncio.wait_for(
3374                         self.k8scluster_map[k8sclustertype].exec_primitive(
3375                             cluster_uuid=k8s_instance_info["k8scluster-uuid"],
3376                             kdu_instance=kdu_instance,
3377                             primitive_name=initial_config_primitive["name"],
3378                             params=primitive_params_,
3379                             db_dict=db_dict_install,
3380                             vca_id=vca_id,
3381                         ),
3382                         timeout=timeout,
3383                     )
3384
3385         except Exception as e:
3386             # Prepare update db with error and raise exception
3387             try:
3388                 self.update_db_2(
3389                     "nsrs", nsr_id, {nsr_db_path + ".detailed-status": str(e)}
3390                 )
3391                 self.update_db_2(
3392                     "vnfrs",
3393                     vnfr_data.get("_id"),
3394                     {"kdur.{}.status".format(kdu_index): "ERROR"},
3395                 )
3396             except Exception:
3397                 # ignore to keep original exception
3398                 pass
3399             # reraise original error
3400             raise
3401
3402         return kdu_instance
3403
3404     async def deploy_kdus(
3405         self,
3406         logging_text,
3407         nsr_id,
3408         nslcmop_id,
3409         db_vnfrs,
3410         db_vnfds,
3411         task_instantiation_info,
3412     ):
3413         # Launch kdus if present in the descriptor
3414
3415         k8scluster_id_2_uuic = {
3416             "helm-chart-v3": {},
3417             "helm-chart": {},
3418             "juju-bundle": {},
3419         }
3420
3421         async def _get_cluster_id(cluster_id, cluster_type):
3422             nonlocal k8scluster_id_2_uuic
3423             if cluster_id in k8scluster_id_2_uuic[cluster_type]:
3424                 return k8scluster_id_2_uuic[cluster_type][cluster_id]
3425
3426             # check if K8scluster is creating and wait look if previous tasks in process
3427             task_name, task_dependency = self.lcm_tasks.lookfor_related(
3428                 "k8scluster", cluster_id
3429             )
3430             if task_dependency:
3431                 text = "Waiting for related tasks '{}' on k8scluster {} to be completed".format(
3432                     task_name, cluster_id
3433                 )
3434                 self.logger.debug(logging_text + text)
3435                 await asyncio.wait(task_dependency, timeout=3600)
3436
3437             db_k8scluster = self.db.get_one(
3438                 "k8sclusters", {"_id": cluster_id}, fail_on_empty=False
3439             )
3440             if not db_k8scluster:
3441                 raise LcmException("K8s cluster {} cannot be found".format(cluster_id))
3442
3443             k8s_id = deep_get(db_k8scluster, ("_admin", cluster_type, "id"))
3444             if not k8s_id:
3445                 if cluster_type == "helm-chart-v3":
3446                     try:
3447                         # backward compatibility for existing clusters that have not been initialized for helm v3
3448                         k8s_credentials = yaml.safe_dump(
3449                             db_k8scluster.get("credentials")
3450                         )
3451                         k8s_id, uninstall_sw = await self.k8sclusterhelm3.init_env(
3452                             k8s_credentials, reuse_cluster_uuid=cluster_id
3453                         )
3454                         db_k8scluster_update = {}
3455                         db_k8scluster_update["_admin.helm-chart-v3.error_msg"] = None
3456                         db_k8scluster_update["_admin.helm-chart-v3.id"] = k8s_id
3457                         db_k8scluster_update[
3458                             "_admin.helm-chart-v3.created"
3459                         ] = uninstall_sw
3460                         db_k8scluster_update[
3461                             "_admin.helm-chart-v3.operationalState"
3462                         ] = "ENABLED"
3463                         self.update_db_2(
3464                             "k8sclusters", cluster_id, db_k8scluster_update
3465                         )
3466                     except Exception as e:
3467                         self.logger.error(
3468                             logging_text
3469                             + "error initializing helm-v3 cluster: {}".format(str(e))
3470                         )
3471                         raise LcmException(
3472                             "K8s cluster '{}' has not been initialized for '{}'".format(
3473                                 cluster_id, cluster_type
3474                             )
3475                         )
3476                 else:
3477                     raise LcmException(
3478                         "K8s cluster '{}' has not been initialized for '{}'".format(
3479                             cluster_id, cluster_type
3480                         )
3481                     )
3482             k8scluster_id_2_uuic[cluster_type][cluster_id] = k8s_id
3483             return k8s_id
3484
3485         logging_text += "Deploy kdus: "
3486         step = ""
3487         try:
3488             db_nsr_update = {"_admin.deployed.K8s": []}
3489             self.update_db_2("nsrs", nsr_id, db_nsr_update)
3490
3491             index = 0
3492             updated_cluster_list = []
3493             updated_v3_cluster_list = []
3494
3495             for vnfr_data in db_vnfrs.values():
3496                 vca_id = self.get_vca_id(vnfr_data, {})
3497                 for kdu_index, kdur in enumerate(get_iterable(vnfr_data, "kdur")):
3498                     # Step 0: Prepare and set parameters
3499                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
3500                     vnfd_id = vnfr_data.get("vnfd-id")
3501                     vnfd_with_id = find_in_list(
3502                         db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3503                     )
3504                     kdud = next(
3505                         kdud
3506                         for kdud in vnfd_with_id["kdu"]
3507                         if kdud["name"] == kdur["kdu-name"]
3508                     )
3509                     namespace = kdur.get("k8s-namespace")
3510                     kdu_deployment_name = kdur.get("kdu-deployment-name")
3511                     if kdur.get("helm-chart"):
3512                         kdumodel = kdur["helm-chart"]
3513                         # Default version: helm3, if helm-version is v2 assign v2
3514                         k8sclustertype = "helm-chart-v3"
3515                         self.logger.debug("kdur: {}".format(kdur))
3516                         if (
3517                             kdur.get("helm-version")
3518                             and kdur.get("helm-version") == "v2"
3519                         ):
3520                             k8sclustertype = "helm-chart"
3521                     elif kdur.get("juju-bundle"):
3522                         kdumodel = kdur["juju-bundle"]
3523                         k8sclustertype = "juju-bundle"
3524                     else:
3525                         raise LcmException(
3526                             "kdu type for kdu='{}.{}' is neither helm-chart nor "
3527                             "juju-bundle. Maybe an old NBI version is running".format(
3528                                 vnfr_data["member-vnf-index-ref"], kdur["kdu-name"]
3529                             )
3530                         )
3531                     # check if kdumodel is a file and exists
3532                     try:
3533                         vnfd_with_id = find_in_list(
3534                             db_vnfds, lambda vnfd: vnfd["_id"] == vnfd_id
3535                         )
3536                         storage = deep_get(vnfd_with_id, ("_admin", "storage"))
3537                         if storage:  # may be not present if vnfd has not artifacts
3538                             # path format: /vnfdid/pkkdir/helm-charts|juju-bundles/kdumodel
3539                             if storage["pkg-dir"]:
3540                                 filename = "{}/{}/{}s/{}".format(
3541                                     storage["folder"],
3542                                     storage["pkg-dir"],
3543                                     k8sclustertype,
3544                                     kdumodel,
3545                                 )
3546                             else:
3547                                 filename = "{}/Scripts/{}s/{}".format(
3548                                     storage["folder"],
3549                                     k8sclustertype,
3550                                     kdumodel,
3551                                 )
3552                             if self.fs.file_exists(
3553                                 filename, mode="file"
3554                             ) or self.fs.file_exists(filename, mode="dir"):
3555                                 kdumodel = self.fs.path + filename
3556                     except (asyncio.TimeoutError, asyncio.CancelledError):
3557                         raise
3558                     except Exception:  # it is not a file
3559                         pass
3560
3561                     k8s_cluster_id = kdur["k8s-cluster"]["id"]
3562                     step = "Synchronize repos for k8s cluster '{}'".format(
3563                         k8s_cluster_id
3564                     )
3565                     cluster_uuid = await _get_cluster_id(k8s_cluster_id, k8sclustertype)
3566
3567                     # Synchronize  repos
3568                     if (
3569                         k8sclustertype == "helm-chart"
3570                         and cluster_uuid not in updated_cluster_list
3571                     ) or (
3572                         k8sclustertype == "helm-chart-v3"
3573                         and cluster_uuid not in updated_v3_cluster_list
3574                     ):
3575                         del_repo_list, added_repo_dict = await asyncio.ensure_future(
3576                             self.k8scluster_map[k8sclustertype].synchronize_repos(
3577                                 cluster_uuid=cluster_uuid
3578                             )
3579                         )
3580                         if del_repo_list or added_repo_dict:
3581                             if k8sclustertype == "helm-chart":
3582                                 unset = {
3583                                     "_admin.helm_charts_added." + item: None
3584                                     for item in del_repo_list
3585                                 }
3586                                 updated = {
3587                                     "_admin.helm_charts_added." + item: name
3588                                     for item, name in added_repo_dict.items()
3589                                 }
3590                                 updated_cluster_list.append(cluster_uuid)
3591                             elif k8sclustertype == "helm-chart-v3":
3592                                 unset = {
3593                                     "_admin.helm_charts_v3_added." + item: None
3594                                     for item in del_repo_list
3595                                 }
3596                                 updated = {
3597                                     "_admin.helm_charts_v3_added." + item: name
3598                                     for item, name in added_repo_dict.items()
3599                                 }
3600                                 updated_v3_cluster_list.append(cluster_uuid)
3601                             self.logger.debug(
3602                                 logging_text + "repos synchronized on k8s cluster "
3603                                 "'{}' to_delete: {}, to_add: {}".format(
3604                                     k8s_cluster_id, del_repo_list, added_repo_dict
3605                                 )
3606                             )
3607                             self.db.set_one(
3608                                 "k8sclusters",
3609                                 {"_id": k8s_cluster_id},
3610                                 updated,
3611                                 unset=unset,
3612                             )
3613
3614                     # Instantiate kdu
3615                     step = "Instantiating KDU {}.{} in k8s cluster {}".format(
3616                         vnfr_data["member-vnf-index-ref"],
3617                         kdur["kdu-name"],
3618                         k8s_cluster_id,
3619                     )
3620                     k8s_instance_info = {
3621                         "kdu-instance": None,
3622                         "k8scluster-uuid": cluster_uuid,
3623                         "k8scluster-type": k8sclustertype,
3624                         "member-vnf-index": vnfr_data["member-vnf-index-ref"],
3625                         "kdu-name": kdur["kdu-name"],
3626                         "kdu-model": kdumodel,
3627                         "namespace": namespace,
3628                         "kdu-deployment-name": kdu_deployment_name,
3629                     }
3630                     db_path = "_admin.deployed.K8s.{}".format(index)
3631                     db_nsr_update[db_path] = k8s_instance_info
3632                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
3633                     vnfd_with_id = find_in_list(
3634                         db_vnfds, lambda vnf: vnf["_id"] == vnfd_id
3635                     )
3636                     task = asyncio.ensure_future(
3637                         self._install_kdu(
3638                             nsr_id,
3639                             db_path,
3640                             vnfr_data,
3641                             kdu_index,
3642                             kdud,
3643                             vnfd_with_id,
3644                             k8s_instance_info,
3645                             k8params=desc_params,
3646                             timeout=1800,
3647                             vca_id=vca_id,
3648                         )
3649                     )
3650                     self.lcm_tasks.register(
3651                         "ns",
3652                         nsr_id,
3653                         nslcmop_id,
3654                         "instantiate_KDU-{}".format(index),
3655                         task,
3656                     )
3657                     task_instantiation_info[task] = "Deploying KDU {}".format(
3658                         kdur["kdu-name"]
3659                     )
3660
3661                     index += 1
3662
3663         except (LcmException, asyncio.CancelledError):
3664             raise
3665         except Exception as e:
3666             msg = "Exception {} while {}: {}".format(type(e).__name__, step, e)
3667             if isinstance(e, (N2VCException, DbException)):
3668                 self.logger.error(logging_text + msg)
3669             else:
3670                 self.logger.critical(logging_text + msg, exc_info=True)
3671             raise LcmException(msg)
3672         finally:
3673             if db_nsr_update:
3674                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
3675
3676     def _deploy_n2vc(
3677         self,
3678         logging_text,
3679         db_nsr,
3680         db_vnfr,
3681         nslcmop_id,
3682         nsr_id,
3683         nsi_id,
3684         vnfd_id,
3685         vdu_id,
3686         kdu_name,
3687         member_vnf_index,
3688         vdu_index,
3689         vdu_name,
3690         deploy_params,
3691         descriptor_config,
3692         base_folder,
3693         task_instantiation_info,
3694         stage,
3695     ):
3696         # launch instantiate_N2VC in a asyncio task and register task object
3697         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
3698         # if not found, create one entry and update database
3699         # fill db_nsr._admin.deployed.VCA.<index>
3700
3701         self.logger.debug(
3702             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
3703         )
3704         if "execution-environment-list" in descriptor_config:
3705             ee_list = descriptor_config.get("execution-environment-list", [])
3706         elif "juju" in descriptor_config:
3707             ee_list = [descriptor_config]  # ns charms
3708         else:  # other types as script are not supported
3709             ee_list = []
3710
3711         for ee_item in ee_list:
3712             self.logger.debug(
3713                 logging_text
3714                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
3715                     ee_item.get("juju"), ee_item.get("helm-chart")
3716                 )
3717             )
3718             ee_descriptor_id = ee_item.get("id")
3719             if ee_item.get("juju"):
3720                 vca_name = ee_item["juju"].get("charm")
3721                 vca_type = (
3722                     "lxc_proxy_charm"
3723                     if ee_item["juju"].get("charm") is not None
3724                     else "native_charm"
3725                 )
3726                 if ee_item["juju"].get("cloud") == "k8s":
3727                     vca_type = "k8s_proxy_charm"
3728                 elif ee_item["juju"].get("proxy") is False:
3729                     vca_type = "native_charm"
3730             elif ee_item.get("helm-chart"):
3731                 vca_name = ee_item["helm-chart"]
3732                 if ee_item.get("helm-version") and ee_item.get("helm-version") == "v2":
3733                     vca_type = "helm"
3734                 else:
3735                     vca_type = "helm-v3"
3736             else:
3737                 self.logger.debug(
3738                     logging_text + "skipping non juju neither charm configuration"
3739                 )
3740                 continue
3741
3742             vca_index = -1
3743             for vca_index, vca_deployed in enumerate(
3744                 db_nsr["_admin"]["deployed"]["VCA"]
3745             ):
3746                 if not vca_deployed:
3747                     continue
3748                 if (
3749                     vca_deployed.get("member-vnf-index") == member_vnf_index
3750                     and vca_deployed.get("vdu_id") == vdu_id
3751                     and vca_deployed.get("kdu_name") == kdu_name
3752                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
3753                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
3754                 ):
3755                     break
3756             else:
3757                 # not found, create one.
3758                 target = (
3759                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
3760                 )
3761                 if vdu_id:
3762                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
3763                 elif kdu_name:
3764                     target += "/kdu/{}".format(kdu_name)
3765                 vca_deployed = {
3766                     "target_element": target,
3767                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
3768                     "member-vnf-index": member_vnf_index,
3769                     "vdu_id": vdu_id,
3770                     "kdu_name": kdu_name,
3771                     "vdu_count_index": vdu_index,
3772                     "operational-status": "init",  # TODO revise
3773                     "detailed-status": "",  # TODO revise
3774                     "step": "initial-deploy",  # TODO revise
3775                     "vnfd_id": vnfd_id,
3776                     "vdu_name": vdu_name,
3777                     "type": vca_type,
3778                     "ee_descriptor_id": ee_descriptor_id,
3779                 }
3780                 vca_index += 1
3781
3782                 # create VCA and configurationStatus in db
3783                 db_dict = {
3784                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
3785                     "configurationStatus.{}".format(vca_index): dict(),
3786                 }
3787                 self.update_db_2("nsrs", nsr_id, db_dict)
3788
3789                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
3790
3791             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
3792             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
3793             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
3794
3795             # Launch task
3796             task_n2vc = asyncio.ensure_future(
3797                 self.instantiate_N2VC(
3798                     logging_text=logging_text,
3799                     vca_index=vca_index,
3800                     nsi_id=nsi_id,
3801                     db_nsr=db_nsr,
3802                     db_vnfr=db_vnfr,
3803                     vdu_id=vdu_id,
3804                     kdu_name=kdu_name,
3805                     vdu_index=vdu_index,
3806                     deploy_params=deploy_params,
3807                     config_descriptor=descriptor_config,
3808                     base_folder=base_folder,
3809                     nslcmop_id=nslcmop_id,
3810                     stage=stage,
3811                     vca_type=vca_type,
3812                     vca_name=vca_name,
3813                     ee_config_descriptor=ee_item,
3814                 )
3815             )
3816             self.lcm_tasks.register(
3817                 "ns",
3818                 nsr_id,
3819                 nslcmop_id,
3820                 "instantiate_N2VC-{}".format(vca_index),
3821                 task_n2vc,
3822             )
3823             task_instantiation_info[
3824                 task_n2vc
3825             ] = self.task_name_deploy_vca + " {}.{}".format(
3826                 member_vnf_index or "", vdu_id or ""
3827             )
3828
3829     @staticmethod
3830     def _create_nslcmop(nsr_id, operation, params):
3831         """
3832         Creates a ns-lcm-opp content to be stored at database.
3833         :param nsr_id: internal id of the instance
3834         :param operation: instantiate, terminate, scale, action, ...
3835         :param params: user parameters for the operation
3836         :return: dictionary following SOL005 format
3837         """
3838         # Raise exception if invalid arguments
3839         if not (nsr_id and operation and params):
3840             raise LcmException(
3841                 "Parameters 'nsr_id', 'operation' and 'params' needed to create primitive not provided"
3842             )
3843         now = time()
3844         _id = str(uuid4())
3845         nslcmop = {
3846             "id": _id,
3847             "_id": _id,
3848             # COMPLETED,PARTIALLY_COMPLETED,FAILED_TEMP,FAILED,ROLLING_BACK,ROLLED_BACK
3849             "operationState": "PROCESSING",
3850             "statusEnteredTime": now,
3851             "nsInstanceId": nsr_id,
3852             "lcmOperationType": operation,
3853             "startTime": now,
3854             "isAutomaticInvocation": False,
3855             "operationParams": params,
3856             "isCancelPending": False,
3857             "links": {
3858                 "self": "/osm/nslcm/v1/ns_lcm_op_occs/" + _id,
3859                 "nsInstance": "/osm/nslcm/v1/ns_instances/" + nsr_id,
3860             },
3861         }
3862         return nslcmop
3863
3864     def _format_additional_params(self, params):
3865         params = params or {}
3866         for key, value in params.items():
3867             if str(value).startswith("!!yaml "):
3868                 params[key] = yaml.safe_load(value[7:])
3869         return params
3870
3871     def _get_terminate_primitive_params(self, seq, vnf_index):
3872         primitive = seq.get("name")
3873         primitive_params = {}
3874         params = {
3875             "member_vnf_index": vnf_index,
3876             "primitive": primitive,
3877             "primitive_params": primitive_params,
3878         }
3879         desc_params = {}
3880         return self._map_primitive_params(seq, params, desc_params)
3881
3882     # sub-operations
3883
3884     def _retry_or_skip_suboperation(self, db_nslcmop, op_index):
3885         op = deep_get(db_nslcmop, ("_admin", "operations"), [])[op_index]
3886         if op.get("operationState") == "COMPLETED":
3887             # b. Skip sub-operation
3888             # _ns_execute_primitive() or RO.create_action() will NOT be executed
3889             return self.SUBOPERATION_STATUS_SKIP
3890         else:
3891             # c. retry executing sub-operation
3892             # The sub-operation exists, and operationState != 'COMPLETED'
3893             # Update operationState = 'PROCESSING' to indicate a retry.
3894             operationState = "PROCESSING"
3895             detailed_status = "In progress"
3896             self._update_suboperation_status(
3897                 db_nslcmop, op_index, operationState, detailed_status
3898             )
3899             # Return the sub-operation index
3900             # _ns_execute_primitive() or RO.create_action() will be called from scale()
3901             # with arguments extracted from the sub-operation
3902             return op_index
3903
3904     # Find a sub-operation where all keys in a matching dictionary must match
3905     # Returns the index of the matching sub-operation, or SUBOPERATION_STATUS_NOT_FOUND if no match
3906     def _find_suboperation(self, db_nslcmop, match):
3907         if db_nslcmop and match:
3908             op_list = db_nslcmop.get("_admin", {}).get("operations", [])
3909             for i, op in enumerate(op_list):
3910                 if all(op.get(k) == match[k] for k in match):
3911                     return i
3912         return self.SUBOPERATION_STATUS_NOT_FOUND
3913
3914     # Update status for a sub-operation given its index
3915     def _update_suboperation_status(
3916         self, db_nslcmop, op_index, operationState, detailed_status
3917     ):
3918         # Update DB for HA tasks
3919         q_filter = {"_id": db_nslcmop["_id"]}
3920         update_dict = {
3921             "_admin.operations.{}.operationState".format(op_index): operationState,
3922             "_admin.operations.{}.detailed-status".format(op_index): detailed_status,
3923         }
3924         self.db.set_one(
3925             "nslcmops", q_filter=q_filter, update_dict=update_dict, fail_on_empty=False
3926         )
3927
3928     # Add sub-operation, return the index of the added sub-operation
3929     # Optionally, set operationState, detailed-status, and operationType
3930     # Status and type are currently set for 'scale' sub-operations:
3931     # 'operationState' : 'PROCESSING' | 'COMPLETED' | 'FAILED'
3932     # 'detailed-status' : status message
3933     # 'operationType': may be any type, in the case of scaling: 'PRE-SCALE' | 'POST-SCALE'
3934     # Status and operation type are currently only used for 'scale', but NOT for 'terminate' sub-operations.
3935     def _add_suboperation(
3936         self,
3937         db_nslcmop,
3938         vnf_index,
3939         vdu_id,
3940         vdu_count_index,
3941         vdu_name,
3942         primitive,
3943         mapped_primitive_params,
3944         operationState=None,
3945         detailed_status=None,
3946         operationType=None,
3947         RO_nsr_id=None,
3948         RO_scaling_info=None,
3949     ):
3950         if not db_nslcmop:
3951             return self.SUBOPERATION_STATUS_NOT_FOUND
3952         # Get the "_admin.operations" list, if it exists
3953         db_nslcmop_admin = db_nslcmop.get("_admin", {})
3954         op_list = db_nslcmop_admin.get("operations")
3955         # Create or append to the "_admin.operations" list
3956         new_op = {
3957             "member_vnf_index": vnf_index,
3958             "vdu_id": vdu_id,
3959             "vdu_count_index": vdu_count_index,
3960             "primitive": primitive,
3961             "primitive_params": mapped_primitive_params,
3962         }
3963         if operationState:
3964             new_op["operationState"] = operationState
3965         if detailed_status:
3966             new_op["detailed-status"] = detailed_status
3967         if operationType:
3968             new_op["lcmOperationType"] = operationType
3969         if RO_nsr_id:
3970             new_op["RO_nsr_id"] = RO_nsr_id
3971         if RO_scaling_info:
3972             new_op["RO_scaling_info"] = RO_scaling_info
3973         if not op_list:
3974             # No existing operations, create key 'operations' with current operation as first list element
3975             db_nslcmop_admin.update({"operations": [new_op]})
3976             op_list = db_nslcmop_admin.get("operations")
3977         else:
3978             # Existing operations, append operation to list
3979             op_list.append(new_op)
3980
3981         db_nslcmop_update = {"_admin.operations": op_list}
3982         self.update_db_2("nslcmops", db_nslcmop["_id"], db_nslcmop_update)
3983         op_index = len(op_list) - 1
3984         return op_index
3985
3986     # Helper methods for scale() sub-operations
3987
3988     # pre-scale/post-scale:
3989     # Check for 3 different cases:
3990     # a. New: First time execution, return SUBOPERATION_STATUS_NEW
3991     # b. Skip: Existing sub-operation exists, operationState == 'COMPLETED', return SUBOPERATION_STATUS_SKIP
3992     # c. retry: Existing sub-operation exists, operationState != 'COMPLETED', return op_index to re-execute
3993     def _check_or_add_scale_suboperation(
3994         self,
3995         db_nslcmop,
3996         vnf_index,
3997         vnf_config_primitive,
3998         primitive_params,
3999         operationType,
4000         RO_nsr_id=None,
4001         RO_scaling_info=None,
4002     ):
4003         # Find this sub-operation
4004         if RO_nsr_id and RO_scaling_info:
4005             operationType = "SCALE-RO"
4006             match = {
4007                 "member_vnf_index": vnf_index,
4008                 "RO_nsr_id": RO_nsr_id,
4009                 "RO_scaling_info": RO_scaling_info,
4010             }
4011         else:
4012             match = {
4013                 "member_vnf_index": vnf_index,
4014                 "primitive": vnf_config_primitive,
4015                 "primitive_params": primitive_params,
4016                 "lcmOperationType": operationType,
4017             }
4018         op_index = self._find_suboperation(db_nslcmop, match)
4019         if op_index == self.SUBOPERATION_STATUS_NOT_FOUND:
4020             # a. New sub-operation
4021             # The sub-operation does not exist, add it.
4022             # _ns_execute_primitive() will be called from scale() as usual, with non-modified arguments
4023             # The following parameters are set to None for all kind of scaling:
4024             vdu_id = None
4025             vdu_count_index = None
4026             vdu_name = None
4027             if RO_nsr_id and RO_scaling_info:
4028                 vnf_config_primitive = None
4029                 primitive_params = None
4030             else:
4031                 RO_nsr_id = None
4032                 RO_scaling_info = None
4033             # Initial status for sub-operation
4034             operationState = "PROCESSING"
4035             detailed_status = "In progress"
4036             # Add sub-operation for pre/post-scaling (zero or more operations)
4037             self._add_suboperation(
4038                 db_nslcmop,
4039                 vnf_index,
4040                 vdu_id,
4041                 vdu_count_index,
4042                 vdu_name,
4043                 vnf_config_primitive,
4044                 primitive_params,
4045                 operationState,
4046                 detailed_status,
4047                 operationType,
4048                 RO_nsr_id,
4049                 RO_scaling_info,
4050             )
4051             return self.SUBOPERATION_STATUS_NEW
4052         else:
4053             # Return either SUBOPERATION_STATUS_SKIP (operationState == 'COMPLETED'),
4054             # or op_index (operationState != 'COMPLETED')
4055             return self._retry_or_skip_suboperation(db_nslcmop, op_index)
4056
4057     # Function to return execution_environment id
4058
4059     def _get_ee_id(self, vnf_index, vdu_id, vca_deployed_list):
4060         # TODO vdu_index_count
4061         for vca in vca_deployed_list:
4062             if vca["member-vnf-index"] == vnf_index and vca["vdu_id"] == vdu_id:
4063                 return vca["ee_id"]
4064
4065     async def destroy_N2VC(
4066         self,
4067         logging_text,
4068         db_nslcmop,
4069         vca_deployed,
4070         config_descriptor,
4071         vca_index,
4072         destroy_ee=True,
4073         exec_primitives=True,
4074         scaling_in=False,
4075         vca_id: str = None,
4076     ):
4077         """
4078         Execute the terminate primitives and destroy the execution environment (if destroy_ee=False
4079         :param logging_text:
4080         :param db_nslcmop:
4081         :param vca_deployed: Dictionary of deployment info at db_nsr._admin.depoloyed.VCA.<INDEX>
4082         :param config_descriptor: Configuration descriptor of the NSD, VNFD, VNFD.vdu or VNFD.kdu
4083         :param vca_index: index in the database _admin.deployed.VCA
4084         :param destroy_ee: False to do not destroy, because it will be destroyed all of then at once
4085         :param exec_primitives: False to do not execute terminate primitives, because the config is not completed or has
4086                             not executed properly
4087         :param scaling_in: True destroys the application, False destroys the model
4088         :return: None or exception
4089         """
4090
4091         self.logger.debug(
4092             logging_text
4093             + " vca_index: {}, vca_deployed: {}, config_descriptor: {}, destroy_ee: {}".format(
4094                 vca_index, vca_deployed, config_descriptor, destroy_ee
4095             )
4096         )
4097
4098         vca_type = vca_deployed.get("type", "lxc_proxy_charm")
4099
4100         # execute terminate_primitives
4101         if exec_primitives:
4102             terminate_primitives = get_ee_sorted_terminate_config_primitive_list(
4103                 config_descriptor.get("terminate-config-primitive"),
4104                 vca_deployed.get("ee_descriptor_id"),
4105             )
4106             vdu_id = vca_deployed.get("vdu_id")
4107             vdu_count_index = vca_deployed.get("vdu_count_index")
4108             vdu_name = vca_deployed.get("vdu_name")
4109             vnf_index = vca_deployed.get("member-vnf-index")
4110             if terminate_primitives and vca_deployed.get("needed_terminate"):
4111                 for seq in terminate_primitives:
4112                     # For each sequence in list, get primitive and call _ns_execute_primitive()
4113                     step = "Calling terminate action for vnf_member_index={} primitive={}".format(
4114                         vnf_index, seq.get("name")
4115                     )
4116                     self.logger.debug(logging_text + step)
4117                     # Create the primitive for each sequence, i.e. "primitive": "touch"
4118                     primitive = seq.get("name")
4119                     mapped_primitive_params = self._get_terminate_primitive_params(
4120                         seq, vnf_index
4121                     )
4122
4123                     # Add sub-operation
4124                     self._add_suboperation(
4125                         db_nslcmop,
4126                         vnf_index,
4127                         vdu_id,
4128                         vdu_count_index,
4129                         vdu_name,
4130                         primitive,
4131                         mapped_primitive_params,
4132                     )
4133                     # Sub-operations: Call _ns_execute_primitive() instead of action()
4134                     try:
4135                         result, result_detail = await self._ns_execute_primitive(
4136                             vca_deployed["ee_id"],
4137                             primitive,
4138                             mapped_primitive_params,
4139                             vca_type=vca_type,
4140                             vca_id=vca_id,
4141                         )
4142                     except LcmException:
4143                         # this happens when VCA is not deployed. In this case it is not needed to terminate
4144                         continue
4145                     result_ok = ["COMPLETED", "PARTIALLY_COMPLETED"]
4146                     if result not in result_ok:
4147                         raise LcmException(
4148                             "terminate_primitive {}  for vnf_member_index={} fails with "
4149                             "error {}".format(seq.get("name"), vnf_index, result_detail)
4150                         )
4151                 # set that this VCA do not need terminated
4152                 db_update_entry = "_admin.deployed.VCA.{}.needed_terminate".format(
4153                     vca_index
4154                 )
4155                 self.update_db_2(
4156                     "nsrs", db_nslcmop["nsInstanceId"], {db_update_entry: False}
4157                 )
4158
4159         # Delete Prometheus Jobs if any
4160         # This uses NSR_ID, so it will destroy any jobs under this index
4161         self.db.del_list("prometheus_jobs", {"nsr_id": db_nslcmop["nsInstanceId"]})
4162
4163         if destroy_ee:
4164             await self.vca_map[vca_type].delete_execution_environment(
4165                 vca_deployed["ee_id"],
4166                 scaling_in=scaling_in,
4167                 vca_type=vca_type,
4168                 vca_id=vca_id,
4169             )
4170
4171     async def _delete_all_N2VC(self, db_nsr: dict, vca_id: str = None):
4172         self._write_all_config_status(db_nsr=db_nsr, status="TERMINATING")
4173         namespace = "." + db_nsr["_id"]
4174         try:
4175             await self.n2vc.delete_namespace(
4176                 namespace=namespace,
4177                 total_timeout=self.timeout_charm_delete,
4178                 vca_id=vca_id,
4179             )
4180         except N2VCNotFound:  # already deleted. Skip
4181             pass
4182         self._write_all_config_status(db_nsr=db_nsr, status="DELETED")
4183
4184     async def _terminate_RO(
4185         self, logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
4186     ):
4187         """
4188         Terminates a deployment from RO
4189         :param logging_text:
4190         :param nsr_deployed: db_nsr._admin.deployed
4191         :param nsr_id:
4192         :param nslcmop_id:
4193         :param stage: list of string with the content to write on db_nslcmop.detailed-status.
4194             this method will update only the index 2, but it will write on database the concatenated content of the list
4195         :return:
4196         """
4197         db_nsr_update = {}
4198         failed_detail = []
4199         ro_nsr_id = ro_delete_action = None
4200         if nsr_deployed and nsr_deployed.get("RO"):
4201             ro_nsr_id = nsr_deployed["RO"].get("nsr_id")
4202             ro_delete_action = nsr_deployed["RO"].get("nsr_delete_action_id")
4203         try:
4204             if ro_nsr_id:
4205                 stage[2] = "Deleting ns from VIM."
4206                 db_nsr_update["detailed-status"] = " ".join(stage)
4207                 self._write_op_status(nslcmop_id, stage)
4208                 self.logger.debug(logging_text + stage[2])
4209                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
4210                 self._write_op_status(nslcmop_id, stage)
4211                 desc = await self.RO.delete("ns", ro_nsr_id)
4212                 ro_delete_action = desc["action_id"]
4213                 db_nsr_update[
4214                     "_admin.deployed.RO.nsr_delete_action_id"
4215                 ] = ro_delete_action
4216                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
4217                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
4218             if ro_delete_action:
4219                 # wait until NS is deleted from VIM
4220                 stage[2] = "Waiting ns deleted from VIM."
4221                 detailed_status_old = None
4222                 self.logger.debug(
4223                     logging_text
4224                     + stage[2]
4225                     + " RO_id={} ro_delete_action={}".format(
4226                         ro_nsr_id, ro_delete_action
4227                     )
4228                 )
4229                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
4230                 self._write_op_status(nslcmop_id, stage)
4231
4232                 delete_timeout = 20 * 60  # 20 minutes
4233                 while delete_timeout > 0:
4234                     desc = await self.RO.show(
4235                         "ns",
4236                         item_id_name=ro_nsr_id,
4237                         extra_item="action",
4238                         extra_item_id=ro_delete_action,
4239                     )
4240
4241                     # deploymentStatus
4242                     self._on_update_ro_db(nsrs_id=nsr_id, ro_descriptor=desc)
4243
4244                     ns_status, ns_status_info = self.RO.check_action_status(desc)
4245                     if ns_status == "ERROR":
4246                         raise ROclient.ROClientException(ns_status_info)
4247                     elif ns_status == "BUILD":
4248                         stage[2] = "Deleting from VIM {}".format(ns_status_info)
4249                     elif ns_status == "ACTIVE":
4250                         db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
4251                         db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
4252                         break
4253                     else:
4254                         assert (
4255                             False
4256                         ), "ROclient.check_action_status returns unknown {}".format(
4257                             ns_status
4258                         )
4259                     if stage[2] != detailed_status_old:
4260                         detailed_status_old = stage[2]
4261                         db_nsr_update["detailed-status"] = " ".join(stage)
4262                         self._write_op_status(nslcmop_id, stage)
4263                         self.update_db_2("nsrs", nsr_id, db_nsr_update)
4264                     await asyncio.sleep(5, loop=self.loop)
4265                     delete_timeout -= 5
4266                 else:  # delete_timeout <= 0:
4267                     raise ROclient.ROClientException(
4268                         "Timeout waiting ns deleted from VIM"
4269                     )
4270
4271         except Exception as e:
4272             self.update_db_2("nsrs", nsr_id, db_nsr_update)
4273             if (
4274                 isinstance(e, ROclient.ROClientException) and e.http_code == 404
4275             ):  # not found
4276                 db_nsr_update["_admin.deployed.RO.nsr_id"] = None
4277                 db_nsr_update["_admin.deployed.RO.nsr_status"] = "DELETED"
4278                 db_nsr_update["_admin.deployed.RO.nsr_delete_action_id"] = None
4279                 self.logger.debug(
4280                     logging_text + "RO_ns_id={} already deleted".format(ro_nsr_id)
4281                 )
4282             elif (
4283                 isinstance(e, ROclient.ROClientException) and e.http_code == 409
4284             ):  # conflict
4285                 failed_detail.append("delete conflict: {}".format(e))
4286                 self.logger.debug(
4287                     logging_text
4288                     + "RO_ns_id={} delete conflict: {}".format(ro_nsr_id, e)
4289                 )
4290             else:
4291                 failed_detail.append("delete error: {}".format(e))
4292                 self.logger.error(
4293                     logging_text + "RO_ns_id={} delete error: {}".format(ro_nsr_id, e)
4294                 )
4295
4296         # Delete nsd
4297         if not failed_detail and deep_get(nsr_deployed, ("RO", "nsd_id")):
4298             ro_nsd_id = nsr_deployed["RO"]["nsd_id"]
4299             try:
4300                 stage[2] = "Deleting nsd from RO."
4301                 db_nsr_update["detailed-status"] = " ".join(stage)
4302                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
4303                 self._write_op_status(nslcmop_id, stage)
4304                 await self.RO.delete("nsd", ro_nsd_id)
4305                 self.logger.debug(
4306                     logging_text + "ro_nsd_id={} deleted".format(ro_nsd_id)
4307                 )
4308                 db_nsr_update["_admin.deployed.RO.nsd_id"] = None
4309             except Exception as e:
4310                 if (
4311                     isinstance(e, ROclient.ROClientException) and e.http_code == 404
4312                 ):  # not found
4313                     db_nsr_update["_admin.deployed.RO.nsd_id"] = None
4314                     self.logger.debug(
4315                         logging_text + "ro_nsd_id={} already deleted".format(ro_nsd_id)
4316                     )
4317                 elif (
4318                     isinstance(e, ROclient.ROClientException) and e.http_code == 409
4319                 ):  # conflict
4320                     failed_detail.append(
4321                         "ro_nsd_id={} delete conflict: {}".format(ro_nsd_id, e)
4322                     )
4323                     self.logger.debug(logging_text + failed_detail[-1])
4324                 else:
4325                     failed_detail.append(
4326                         "ro_nsd_id={} delete error: {}".format(ro_nsd_id, e)
4327                     )
4328                     self.logger.error(logging_text + failed_detail[-1])
4329
4330         if not failed_detail and deep_get(nsr_deployed, ("RO", "vnfd")):
4331             for index, vnf_deployed in enumerate(nsr_deployed["RO"]["vnfd"]):
4332                 if not vnf_deployed or not vnf_deployed["id"]:
4333                     continue
4334                 try:
4335                     ro_vnfd_id = vnf_deployed["id"]
4336                     stage[
4337                         2
4338                     ] = "Deleting member_vnf_index={} ro_vnfd_id={} from RO.".format(
4339                         vnf_deployed["member-vnf-index"], ro_vnfd_id
4340                     )
4341                     db_nsr_update["detailed-status"] = " ".join(stage)
4342                     self.update_db_2("nsrs", nsr_id, db_nsr_update)
4343                     self._write_op_status(nslcmop_id, stage)
4344                     await self.RO.delete("vnfd", ro_vnfd_id)
4345                     self.logger.debug(
4346                         logging_text + "ro_vnfd_id={} deleted".format(ro_vnfd_id)
4347                     )
4348                     db_nsr_update["_admin.deployed.RO.vnfd.{}.id".format(index)] = None
4349                 except Exception as e:
4350                     if (
4351                         isinstance(e, ROclient.ROClientException) and e.http_code == 404
4352                     ):  # not found
4353                         db_nsr_update[
4354                             "_admin.deployed.RO.vnfd.{}.id".format(index)
4355                         ] = None
4356                         self.logger.debug(
4357                             logging_text
4358                             + "ro_vnfd_id={} already deleted ".format(ro_vnfd_id)
4359                         )
4360                     elif (
4361                         isinstance(e, ROclient.ROClientException) and e.http_code == 409
4362                     ):  # conflict
4363                         failed_detail.append(
4364                             "ro_vnfd_id={} delete conflict: {}".format(ro_vnfd_id, e)
4365                         )
4366                         self.logger.debug(logging_text + failed_detail[-1])
4367                     else:
4368                         failed_detail.append(
4369                             "ro_vnfd_id={} delete error: {}".format(ro_vnfd_id, e)
4370                         )
4371                         self.logger.error(logging_text + failed_detail[-1])
4372
4373         if failed_detail:
4374             stage[2] = "Error deleting from VIM"
4375         else:
4376             stage[2] = "Deleted from VIM"
4377         db_nsr_update["detailed-status"] = " ".join(stage)
4378         self.update_db_2("nsrs", nsr_id, db_nsr_update)
4379         self._write_op_status(nslcmop_id, stage)
4380
4381         if failed_detail:
4382             raise LcmException("; ".join(failed_detail))
4383
4384     async def terminate(self, nsr_id, nslcmop_id):
4385         # Try to lock HA task here
4386         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
4387         if not task_is_locked_by_me:
4388             return
4389
4390         logging_text = "Task ns={} terminate={} ".format(nsr_id, nslcmop_id)
4391         self.logger.debug(logging_text + "Enter")
4392         timeout_ns_terminate = self.timeout_ns_terminate
4393         db_nsr = None
4394         db_nslcmop = None
4395         operation_params = None
4396         exc = None
4397         error_list = []  # annotates all failed error messages
4398         db_nslcmop_update = {}
4399         autoremove = False  # autoremove after terminated
4400         tasks_dict_info = {}
4401         db_nsr_update = {}
4402         stage = [
4403             "Stage 1/3: Preparing task.",
4404             "Waiting for previous operations to terminate.",
4405             "",
4406         ]
4407         # ^ contains [stage, step, VIM-status]
4408         try:
4409             # wait for any previous tasks in process
4410             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
4411
4412             stage[1] = "Getting nslcmop={} from db.".format(nslcmop_id)
4413             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
4414             operation_params = db_nslcmop.get("operationParams") or {}
4415             if operation_params.get("timeout_ns_terminate"):
4416                 timeout_ns_terminate = operation_params["timeout_ns_terminate"]
4417             stage[1] = "Getting nsr={} from db.".format(nsr_id)
4418             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4419
4420             db_nsr_update["operational-status"] = "terminating"
4421             db_nsr_update["config-status"] = "terminating"
4422             self._write_ns_status(
4423                 nsr_id=nsr_id,
4424                 ns_state="TERMINATING",
4425                 current_operation="TERMINATING",
4426                 current_operation_id=nslcmop_id,
4427                 other_update=db_nsr_update,
4428             )
4429             self._write_op_status(op_id=nslcmop_id, queuePosition=0, stage=stage)
4430             nsr_deployed = deepcopy(db_nsr["_admin"].get("deployed")) or {}
4431             if db_nsr["_admin"]["nsState"] == "NOT_INSTANTIATED":
4432                 return
4433
4434             stage[1] = "Getting vnf descriptors from db."
4435             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
4436             db_vnfrs_dict = {
4437                 db_vnfr["member-vnf-index-ref"]: db_vnfr for db_vnfr in db_vnfrs_list
4438             }
4439             db_vnfds_from_id = {}
4440             db_vnfds_from_member_index = {}
4441             # Loop over VNFRs
4442             for vnfr in db_vnfrs_list:
4443                 vnfd_id = vnfr["vnfd-id"]
4444                 if vnfd_id not in db_vnfds_from_id:
4445                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
4446                     db_vnfds_from_id[vnfd_id] = vnfd
4447                 db_vnfds_from_member_index[
4448                     vnfr["member-vnf-index-ref"]
4449                 ] = db_vnfds_from_id[vnfd_id]
4450
4451             # Destroy individual execution environments when there are terminating primitives.
4452             # Rest of EE will be deleted at once
4453             # TODO - check before calling _destroy_N2VC
4454             # if not operation_params.get("skip_terminate_primitives"):#
4455             # or not vca.get("needed_terminate"):
4456             stage[0] = "Stage 2/3 execute terminating primitives."
4457             self.logger.debug(logging_text + stage[0])
4458             stage[1] = "Looking execution environment that needs terminate."
4459             self.logger.debug(logging_text + stage[1])
4460
4461             for vca_index, vca in enumerate(get_iterable(nsr_deployed, "VCA")):
4462                 config_descriptor = None
4463                 vca_member_vnf_index = vca.get("member-vnf-index")
4464                 vca_id = self.get_vca_id(
4465                     db_vnfrs_dict.get(vca_member_vnf_index)
4466                     if vca_member_vnf_index
4467                     else None,
4468                     db_nsr,
4469                 )
4470                 if not vca or not vca.get("ee_id"):
4471                     continue
4472                 if not vca.get("member-vnf-index"):
4473                     # ns
4474                     config_descriptor = db_nsr.get("ns-configuration")
4475                 elif vca.get("vdu_id"):
4476                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4477                     config_descriptor = get_configuration(db_vnfd, vca.get("vdu_id"))
4478                 elif vca.get("kdu_name"):
4479                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4480                     config_descriptor = get_configuration(db_vnfd, vca.get("kdu_name"))
4481                 else:
4482                     db_vnfd = db_vnfds_from_member_index[vca["member-vnf-index"]]
4483                     config_descriptor = get_configuration(db_vnfd, db_vnfd["id"])
4484                 vca_type = vca.get("type")
4485                 exec_terminate_primitives = not operation_params.get(
4486                     "skip_terminate_primitives"
4487                 ) and vca.get("needed_terminate")
4488                 # For helm we must destroy_ee. Also for native_charm, as juju_model cannot be deleted if there are
4489                 # pending native charms
4490                 destroy_ee = (
4491                     True if vca_type in ("helm", "helm-v3", "native_charm") else False
4492                 )
4493                 # self.logger.debug(logging_text + "vca_index: {}, ee_id: {}, vca_type: {} destroy_ee: {}".format(
4494                 #     vca_index, vca.get("ee_id"), vca_type, destroy_ee))
4495                 task = asyncio.ensure_future(
4496                     self.destroy_N2VC(
4497                         logging_text,
4498                         db_nslcmop,
4499                         vca,
4500                         config_descriptor,
4501                         vca_index,
4502                         destroy_ee,
4503                         exec_terminate_primitives,
4504                         vca_id=vca_id,
4505                     )
4506                 )
4507                 tasks_dict_info[task] = "Terminating VCA {}".format(vca.get("ee_id"))
4508
4509             # wait for pending tasks of terminate primitives
4510             if tasks_dict_info:
4511                 self.logger.debug(
4512                     logging_text
4513                     + "Waiting for tasks {}".format(list(tasks_dict_info.keys()))
4514                 )
4515                 error_list = await self._wait_for_tasks(
4516                     logging_text,
4517                     tasks_dict_info,
4518                     min(self.timeout_charm_delete, timeout_ns_terminate),
4519                     stage,
4520                     nslcmop_id,
4521                 )
4522                 tasks_dict_info.clear()
4523                 if error_list:
4524                     return  # raise LcmException("; ".join(error_list))
4525
4526             # remove All execution environments at once
4527             stage[0] = "Stage 3/3 delete all."
4528
4529             if nsr_deployed.get("VCA"):
4530                 stage[1] = "Deleting all execution environments."
4531                 self.logger.debug(logging_text + stage[1])
4532                 vca_id = self.get_vca_id({}, db_nsr)
4533                 task_delete_ee = asyncio.ensure_future(
4534                     asyncio.wait_for(
4535                         self._delete_all_N2VC(db_nsr=db_nsr, vca_id=vca_id),
4536                         timeout=self.timeout_charm_delete,
4537                     )
4538                 )
4539                 # task_delete_ee = asyncio.ensure_future(self.n2vc.delete_namespace(namespace="." + nsr_id))
4540                 tasks_dict_info[task_delete_ee] = "Terminating all VCA"
4541
4542             # Delete from k8scluster
4543             stage[1] = "Deleting KDUs."
4544             self.logger.debug(logging_text + stage[1])
4545             # print(nsr_deployed)
4546             for kdu in get_iterable(nsr_deployed, "K8s"):
4547                 if not kdu or not kdu.get("kdu-instance"):
4548                     continue
4549                 kdu_instance = kdu.get("kdu-instance")
4550                 if kdu.get("k8scluster-type") in self.k8scluster_map:
4551                     # TODO: Uninstall kdu instances taking into account they could be deployed in different VIMs
4552                     vca_id = self.get_vca_id({}, db_nsr)
4553                     task_delete_kdu_instance = asyncio.ensure_future(
4554                         self.k8scluster_map[kdu["k8scluster-type"]].uninstall(
4555                             cluster_uuid=kdu.get("k8scluster-uuid"),
4556                             kdu_instance=kdu_instance,
4557                             vca_id=vca_id,
4558                             namespace=kdu.get("namespace"),
4559                         )
4560                     )
4561                 else:
4562                     self.logger.error(
4563                         logging_text
4564                         + "Unknown k8s deployment type {}".format(
4565                             kdu.get("k8scluster-type")
4566                         )
4567                     )
4568                     continue
4569                 tasks_dict_info[
4570                     task_delete_kdu_instance
4571                 ] = "Terminating KDU '{}'".format(kdu.get("kdu-name"))
4572
4573             # remove from RO
4574             stage[1] = "Deleting ns from VIM."
4575             if self.ng_ro:
4576                 task_delete_ro = asyncio.ensure_future(
4577                     self._terminate_ng_ro(
4578                         logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
4579                     )
4580                 )
4581             else:
4582                 task_delete_ro = asyncio.ensure_future(
4583                     self._terminate_RO(
4584                         logging_text, nsr_deployed, nsr_id, nslcmop_id, stage
4585                     )
4586                 )
4587             tasks_dict_info[task_delete_ro] = "Removing deployment from VIM"
4588
4589             # rest of staff will be done at finally
4590
4591         except (
4592             ROclient.ROClientException,
4593             DbException,
4594             LcmException,
4595             N2VCException,
4596         ) as e:
4597             self.logger.error(logging_text + "Exit Exception {}".format(e))
4598             exc = e
4599         except asyncio.CancelledError:
4600             self.logger.error(
4601                 logging_text + "Cancelled Exception while '{}'".format(stage[1])
4602             )
4603             exc = "Operation was cancelled"
4604         except Exception as e:
4605             exc = traceback.format_exc()
4606             self.logger.critical(
4607                 logging_text + "Exit Exception while '{}': {}".format(stage[1], e),
4608                 exc_info=True,
4609             )
4610         finally:
4611             if exc:
4612                 error_list.append(str(exc))
4613             try:
4614                 # wait for pending tasks
4615                 if tasks_dict_info:
4616                     stage[1] = "Waiting for terminate pending tasks."
4617                     self.logger.debug(logging_text + stage[1])
4618                     error_list += await self._wait_for_tasks(
4619                         logging_text,
4620                         tasks_dict_info,
4621                         timeout_ns_terminate,
4622                         stage,
4623                         nslcmop_id,
4624                     )
4625                 stage[1] = stage[2] = ""
4626             except asyncio.CancelledError:
4627                 error_list.append("Cancelled")
4628                 # TODO cancell all tasks
4629             except Exception as exc:
4630                 error_list.append(str(exc))
4631             # update status at database
4632             if error_list:
4633                 error_detail = "; ".join(error_list)
4634                 # self.logger.error(logging_text + error_detail)
4635                 error_description_nslcmop = "{} Detail: {}".format(
4636                     stage[0], error_detail
4637                 )
4638                 error_description_nsr = "Operation: TERMINATING.{}, {}.".format(
4639                     nslcmop_id, stage[0]
4640                 )
4641
4642                 db_nsr_update["operational-status"] = "failed"
4643                 db_nsr_update["detailed-status"] = (
4644                     error_description_nsr + " Detail: " + error_detail
4645                 )
4646                 db_nslcmop_update["detailed-status"] = error_detail
4647                 nslcmop_operation_state = "FAILED"
4648                 ns_state = "BROKEN"
4649             else:
4650                 error_detail = None
4651                 error_description_nsr = error_description_nslcmop = None
4652                 ns_state = "NOT_INSTANTIATED"
4653                 db_nsr_update["operational-status"] = "terminated"
4654                 db_nsr_update["detailed-status"] = "Done"
4655                 db_nsr_update["_admin.nsState"] = "NOT_INSTANTIATED"
4656                 db_nslcmop_update["detailed-status"] = "Done"
4657                 nslcmop_operation_state = "COMPLETED"
4658
4659             if db_nsr:
4660                 self._write_ns_status(
4661                     nsr_id=nsr_id,
4662                     ns_state=ns_state,
4663                     current_operation="IDLE",
4664                     current_operation_id=None,
4665                     error_description=error_description_nsr,
4666                     error_detail=error_detail,
4667                     other_update=db_nsr_update,
4668                 )
4669             self._write_op_status(
4670                 op_id=nslcmop_id,
4671                 stage="",
4672                 error_message=error_description_nslcmop,
4673                 operation_state=nslcmop_operation_state,
4674                 other_update=db_nslcmop_update,
4675             )
4676             if ns_state == "NOT_INSTANTIATED":
4677                 try:
4678                     self.db.set_list(
4679                         "vnfrs",
4680                         {"nsr-id-ref": nsr_id},
4681                         {"_admin.nsState": "NOT_INSTANTIATED"},
4682                     )
4683                 except DbException as e:
4684                     self.logger.warn(
4685                         logging_text
4686                         + "Error writing VNFR status for nsr-id-ref: {} -> {}".format(
4687                             nsr_id, e
4688                         )
4689                     )
4690             if operation_params:
4691                 autoremove = operation_params.get("autoremove", False)
4692             if nslcmop_operation_state:
4693                 try:
4694                     await self.msg.aiowrite(
4695                         "ns",
4696                         "terminated",
4697                         {
4698                             "nsr_id": nsr_id,
4699                             "nslcmop_id": nslcmop_id,
4700                             "operationState": nslcmop_operation_state,
4701                             "autoremove": autoremove,
4702                         },
4703                         loop=self.loop,
4704                     )
4705                 except Exception as e:
4706                     self.logger.error(
4707                         logging_text + "kafka_write notification Exception {}".format(e)
4708                     )
4709
4710             self.logger.debug(logging_text + "Exit")
4711             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_terminate")
4712
4713     async def _wait_for_tasks(
4714         self, logging_text, created_tasks_info, timeout, stage, nslcmop_id, nsr_id=None
4715     ):
4716         time_start = time()
4717         error_detail_list = []
4718         error_list = []
4719         pending_tasks = list(created_tasks_info.keys())
4720         num_tasks = len(pending_tasks)
4721         num_done = 0
4722         stage[1] = "{}/{}.".format(num_done, num_tasks)
4723         self._write_op_status(nslcmop_id, stage)
4724         while pending_tasks:
4725             new_error = None
4726             _timeout = timeout + time_start - time()
4727             done, pending_tasks = await asyncio.wait(
4728                 pending_tasks, timeout=_timeout, return_when=asyncio.FIRST_COMPLETED
4729             )
4730             num_done += len(done)
4731             if not done:  # Timeout
4732                 for task in pending_tasks:
4733                     new_error = created_tasks_info[task] + ": Timeout"
4734                     error_detail_list.append(new_error)
4735                     error_list.append(new_error)
4736                 break
4737             for task in done:
4738                 if task.cancelled():
4739                     exc = "Cancelled"
4740                 else:
4741                     exc = task.exception()
4742                 if exc:
4743                     if isinstance(exc, asyncio.TimeoutError):
4744                         exc = "Timeout"
4745                     new_error = created_tasks_info[task] + ": {}".format(exc)
4746                     error_list.append(created_tasks_info[task])
4747                     error_detail_list.append(new_error)
4748                     if isinstance(
4749                         exc,
4750                         (
4751                             str,
4752                             DbException,
4753                             N2VCException,
4754                             ROclient.ROClientException,
4755                             LcmException,
4756                             K8sException,
4757                             NgRoException,
4758                         ),
4759                     ):
4760                         self.logger.error(logging_text + new_error)
4761                     else:
4762                         exc_traceback = "".join(
4763                             traceback.format_exception(None, exc, exc.__traceback__)
4764                         )
4765                         self.logger.error(
4766                             logging_text
4767                             + created_tasks_info[task]
4768                             + " "
4769                             + exc_traceback
4770                         )
4771                 else:
4772                     self.logger.debug(
4773                         logging_text + created_tasks_info[task] + ": Done"
4774                     )
4775             stage[1] = "{}/{}.".format(num_done, num_tasks)
4776             if new_error:
4777                 stage[1] += " Errors: " + ". ".join(error_detail_list) + "."
4778                 if nsr_id:  # update also nsr
4779                     self.update_db_2(
4780                         "nsrs",
4781                         nsr_id,
4782                         {
4783                             "errorDescription": "Error at: " + ", ".join(error_list),
4784                             "errorDetail": ". ".join(error_detail_list),
4785                         },
4786                     )
4787             self._write_op_status(nslcmop_id, stage)
4788         return error_detail_list
4789
4790     @staticmethod
4791     def _map_primitive_params(primitive_desc, params, instantiation_params):
4792         """
4793         Generates the params to be provided to charm before executing primitive. If user does not provide a parameter,
4794         The default-value is used. If it is between < > it look for a value at instantiation_params
4795         :param primitive_desc: portion of VNFD/NSD that describes primitive
4796         :param params: Params provided by user
4797         :param instantiation_params: Instantiation params provided by user
4798         :return: a dictionary with the calculated params
4799         """
4800         calculated_params = {}
4801         for parameter in primitive_desc.get("parameter", ()):
4802             param_name = parameter["name"]
4803             if param_name in params:
4804                 calculated_params[param_name] = params[param_name]
4805             elif "default-value" in parameter or "value" in parameter:
4806                 if "value" in parameter:
4807                     calculated_params[param_name] = parameter["value"]
4808                 else:
4809                     calculated_params[param_name] = parameter["default-value"]
4810                 if (
4811                     isinstance(calculated_params[param_name], str)
4812                     and calculated_params[param_name].startswith("<")
4813                     and calculated_params[param_name].endswith(">")
4814                 ):
4815                     if calculated_params[param_name][1:-1] in instantiation_params:
4816                         calculated_params[param_name] = instantiation_params[
4817                             calculated_params[param_name][1:-1]
4818                         ]
4819                     else:
4820                         raise LcmException(
4821                             "Parameter {} needed to execute primitive {} not provided".format(
4822                                 calculated_params[param_name], primitive_desc["name"]
4823                             )
4824                         )
4825             else:
4826                 raise LcmException(
4827                     "Parameter {} needed to execute primitive {} not provided".format(
4828                         param_name, primitive_desc["name"]
4829                     )
4830                 )
4831
4832             if isinstance(calculated_params[param_name], (dict, list, tuple)):
4833                 calculated_params[param_name] = yaml.safe_dump(
4834                     calculated_params[param_name], default_flow_style=True, width=256
4835                 )
4836             elif isinstance(calculated_params[param_name], str) and calculated_params[
4837                 param_name
4838             ].startswith("!!yaml "):
4839                 calculated_params[param_name] = calculated_params[param_name][7:]
4840             if parameter.get("data-type") == "INTEGER":
4841                 try:
4842                     calculated_params[param_name] = int(calculated_params[param_name])
4843                 except ValueError:  # error converting string to int
4844                     raise LcmException(
4845                         "Parameter {} of primitive {} must be integer".format(
4846                             param_name, primitive_desc["name"]
4847                         )
4848                     )
4849             elif parameter.get("data-type") == "BOOLEAN":
4850                 calculated_params[param_name] = not (
4851                     (str(calculated_params[param_name])).lower() == "false"
4852                 )
4853
4854         # add always ns_config_info if primitive name is config
4855         if primitive_desc["name"] == "config":
4856             if "ns_config_info" in instantiation_params:
4857                 calculated_params["ns_config_info"] = instantiation_params[
4858                     "ns_config_info"
4859                 ]
4860         return calculated_params
4861
4862     def _look_for_deployed_vca(
4863         self,
4864         deployed_vca,
4865         member_vnf_index,
4866         vdu_id,
4867         vdu_count_index,
4868         kdu_name=None,
4869         ee_descriptor_id=None,
4870     ):
4871         # find vca_deployed record for this action. Raise LcmException if not found or there is not any id.
4872         for vca in deployed_vca:
4873             if not vca:
4874                 continue
4875             if member_vnf_index != vca["member-vnf-index"] or vdu_id != vca["vdu_id"]:
4876                 continue
4877             if (
4878                 vdu_count_index is not None
4879                 and vdu_count_index != vca["vdu_count_index"]
4880             ):
4881                 continue
4882             if kdu_name and kdu_name != vca["kdu_name"]:
4883                 continue
4884             if ee_descriptor_id and ee_descriptor_id != vca["ee_descriptor_id"]:
4885                 continue
4886             break
4887         else:
4888             # vca_deployed not found
4889             raise LcmException(
4890                 "charm for member_vnf_index={} vdu_id={}.{} kdu_name={} execution-environment-list.id={}"
4891                 " is not deployed".format(
4892                     member_vnf_index,
4893                     vdu_id,
4894                     vdu_count_index,
4895                     kdu_name,
4896                     ee_descriptor_id,
4897                 )
4898             )
4899         # get ee_id
4900         ee_id = vca.get("ee_id")
4901         vca_type = vca.get(
4902             "type", "lxc_proxy_charm"
4903         )  # default value for backward compatibility - proxy charm
4904         if not ee_id:
4905             raise LcmException(
4906                 "charm for member_vnf_index={} vdu_id={} kdu_name={} vdu_count_index={} has not "
4907                 "execution environment".format(
4908                     member_vnf_index, vdu_id, kdu_name, vdu_count_index
4909                 )
4910             )
4911         return ee_id, vca_type
4912
4913     async def _ns_execute_primitive(
4914         self,
4915         ee_id,
4916         primitive,
4917         primitive_params,
4918         retries=0,
4919         retries_interval=30,
4920         timeout=None,
4921         vca_type=None,
4922         db_dict=None,
4923         vca_id: str = None,
4924     ) -> (str, str):
4925         try:
4926             if primitive == "config":
4927                 primitive_params = {"params": primitive_params}
4928
4929             vca_type = vca_type or "lxc_proxy_charm"
4930
4931             while retries >= 0:
4932                 try:
4933                     output = await asyncio.wait_for(
4934                         self.vca_map[vca_type].exec_primitive(
4935                             ee_id=ee_id,
4936                             primitive_name=primitive,
4937                             params_dict=primitive_params,
4938                             progress_timeout=self.timeout_progress_primitive,
4939                             total_timeout=self.timeout_primitive,
4940                             db_dict=db_dict,
4941                             vca_id=vca_id,
4942                             vca_type=vca_type,
4943                         ),
4944                         timeout=timeout or self.timeout_primitive,
4945                     )
4946                     # execution was OK
4947                     break
4948                 except asyncio.CancelledError:
4949                     raise
4950                 except Exception as e:  # asyncio.TimeoutError
4951                     if isinstance(e, asyncio.TimeoutError):
4952                         e = "Timeout"
4953                     retries -= 1
4954                     if retries >= 0:
4955                         self.logger.debug(
4956                             "Error executing action {} on {} -> {}".format(
4957                                 primitive, ee_id, e
4958                             )
4959                         )
4960                         # wait and retry
4961                         await asyncio.sleep(retries_interval, loop=self.loop)
4962                     else:
4963                         return "FAILED", str(e)
4964
4965             return "COMPLETED", output
4966
4967         except (LcmException, asyncio.CancelledError):
4968             raise
4969         except Exception as e:
4970             return "FAIL", "Error executing action {}: {}".format(primitive, e)
4971
4972     async def vca_status_refresh(self, nsr_id, nslcmop_id):
4973         """
4974         Updating the vca_status with latest juju information in nsrs record
4975         :param: nsr_id: Id of the nsr
4976         :param: nslcmop_id: Id of the nslcmop
4977         :return: None
4978         """
4979
4980         self.logger.debug("Task ns={} action={} Enter".format(nsr_id, nslcmop_id))
4981         db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
4982         vca_id = self.get_vca_id({}, db_nsr)
4983         if db_nsr["_admin"]["deployed"]["K8s"]:
4984             for _, k8s in enumerate(db_nsr["_admin"]["deployed"]["K8s"]):
4985                 cluster_uuid, kdu_instance, cluster_type = (
4986                     k8s["k8scluster-uuid"],
4987                     k8s["kdu-instance"],
4988                     k8s["k8scluster-type"],
4989                 )
4990                 await self._on_update_k8s_db(
4991                     cluster_uuid=cluster_uuid,
4992                     kdu_instance=kdu_instance,
4993                     filter={"_id": nsr_id},
4994                     vca_id=vca_id,
4995                     cluster_type=cluster_type,
4996                 )
4997         else:
4998             for vca_index, _ in enumerate(db_nsr["_admin"]["deployed"]["VCA"]):
4999                 table, filter = "nsrs", {"_id": nsr_id}
5000                 path = "_admin.deployed.VCA.{}.".format(vca_index)
5001                 await self._on_update_n2vc_db(table, filter, path, {})
5002
5003         self.logger.debug("Task ns={} action={} Exit".format(nsr_id, nslcmop_id))
5004         self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_vca_status_refresh")
5005
5006     async def action(self, nsr_id, nslcmop_id):
5007         # Try to lock HA task here
5008         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5009         if not task_is_locked_by_me:
5010             return
5011
5012         logging_text = "Task ns={} action={} ".format(nsr_id, nslcmop_id)
5013         self.logger.debug(logging_text + "Enter")
5014         # get all needed from database
5015         db_nsr = None
5016         db_nslcmop = None
5017         db_nsr_update = {}
5018         db_nslcmop_update = {}
5019         nslcmop_operation_state = None
5020         error_description_nslcmop = None
5021         exc = None
5022         try:
5023             # wait for any previous tasks in process
5024             step = "Waiting for previous operations to terminate"
5025             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5026
5027             self._write_ns_status(
5028                 nsr_id=nsr_id,
5029                 ns_state=None,
5030                 current_operation="RUNNING ACTION",
5031                 current_operation_id=nslcmop_id,
5032             )
5033
5034             step = "Getting information from database"
5035             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5036             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5037             if db_nslcmop["operationParams"].get("primitive_params"):
5038                 db_nslcmop["operationParams"]["primitive_params"] = json.loads(
5039                     db_nslcmop["operationParams"]["primitive_params"]
5040                 )
5041
5042             nsr_deployed = db_nsr["_admin"].get("deployed")
5043             vnf_index = db_nslcmop["operationParams"].get("member_vnf_index")
5044             vdu_id = db_nslcmop["operationParams"].get("vdu_id")
5045             kdu_name = db_nslcmop["operationParams"].get("kdu_name")
5046             vdu_count_index = db_nslcmop["operationParams"].get("vdu_count_index")
5047             primitive = db_nslcmop["operationParams"]["primitive"]
5048             primitive_params = db_nslcmop["operationParams"]["primitive_params"]
5049             timeout_ns_action = db_nslcmop["operationParams"].get(
5050                 "timeout_ns_action", self.timeout_primitive
5051             )
5052
5053             if vnf_index:
5054                 step = "Getting vnfr from database"
5055                 db_vnfr = self.db.get_one(
5056                     "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
5057                 )
5058                 if db_vnfr.get("kdur"):
5059                     kdur_list = []
5060                     for kdur in db_vnfr["kdur"]:
5061                         if kdur.get("additionalParams"):
5062                             kdur["additionalParams"] = json.loads(
5063                                 kdur["additionalParams"]
5064                             )
5065                         kdur_list.append(kdur)
5066                     db_vnfr["kdur"] = kdur_list
5067                 step = "Getting vnfd from database"
5068                 db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
5069
5070                 # Sync filesystem before running a primitive
5071                 self.fs.sync(db_vnfr["vnfd-id"])
5072             else:
5073                 step = "Getting nsd from database"
5074                 db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
5075
5076             vca_id = self.get_vca_id(db_vnfr, db_nsr)
5077             # for backward compatibility
5078             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
5079                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
5080                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
5081                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5082
5083             # look for primitive
5084             config_primitive_desc = descriptor_configuration = None
5085             if vdu_id:
5086                 descriptor_configuration = get_configuration(db_vnfd, vdu_id)
5087             elif kdu_name:
5088                 descriptor_configuration = get_configuration(db_vnfd, kdu_name)
5089             elif vnf_index:
5090                 descriptor_configuration = get_configuration(db_vnfd, db_vnfd["id"])
5091             else:
5092                 descriptor_configuration = db_nsd.get("ns-configuration")
5093
5094             if descriptor_configuration and descriptor_configuration.get(
5095                 "config-primitive"
5096             ):
5097                 for config_primitive in descriptor_configuration["config-primitive"]:
5098                     if config_primitive["name"] == primitive:
5099                         config_primitive_desc = config_primitive
5100                         break
5101
5102             if not config_primitive_desc:
5103                 if not (kdu_name and primitive in ("upgrade", "rollback", "status")):
5104                     raise LcmException(
5105                         "Primitive {} not found at [ns|vnf|vdu]-configuration:config-primitive ".format(
5106                             primitive
5107                         )
5108                     )
5109                 primitive_name = primitive
5110                 ee_descriptor_id = None
5111             else:
5112                 primitive_name = config_primitive_desc.get(
5113                     "execution-environment-primitive", primitive
5114                 )
5115                 ee_descriptor_id = config_primitive_desc.get(
5116                     "execution-environment-ref"
5117                 )
5118
5119             if vnf_index:
5120                 if vdu_id:
5121                     vdur = next(
5122                         (x for x in db_vnfr["vdur"] if x["vdu-id-ref"] == vdu_id), None
5123                     )
5124                     desc_params = parse_yaml_strings(vdur.get("additionalParams"))
5125                 elif kdu_name:
5126                     kdur = next(
5127                         (x for x in db_vnfr["kdur"] if x["kdu-name"] == kdu_name), None
5128                     )
5129                     desc_params = parse_yaml_strings(kdur.get("additionalParams"))
5130                 else:
5131                     desc_params = parse_yaml_strings(
5132                         db_vnfr.get("additionalParamsForVnf")
5133                     )
5134             else:
5135                 desc_params = parse_yaml_strings(db_nsr.get("additionalParamsForNs"))
5136             if kdu_name and get_configuration(db_vnfd, kdu_name):
5137                 kdu_configuration = get_configuration(db_vnfd, kdu_name)
5138                 actions = set()
5139                 for primitive in kdu_configuration.get("initial-config-primitive", []):
5140                     actions.add(primitive["name"])
5141                 for primitive in kdu_configuration.get("config-primitive", []):
5142                     actions.add(primitive["name"])
5143                 kdu = find_in_list(
5144                     nsr_deployed["K8s"],
5145                     lambda kdu: kdu_name == kdu["kdu-name"]
5146                     and kdu["member-vnf-index"] == vnf_index,
5147                 )
5148                 kdu_action = (
5149                     True
5150                     if primitive_name in actions
5151                     and kdu["k8scluster-type"] not in ("helm-chart", "helm-chart-v3")
5152                     else False
5153                 )
5154
5155             # TODO check if ns is in a proper status
5156             if kdu_name and (
5157                 primitive_name in ("upgrade", "rollback", "status") or kdu_action
5158             ):
5159                 # kdur and desc_params already set from before
5160                 if primitive_params:
5161                     desc_params.update(primitive_params)
5162                 # TODO Check if we will need something at vnf level
5163                 for index, kdu in enumerate(get_iterable(nsr_deployed, "K8s")):
5164                     if (
5165                         kdu_name == kdu["kdu-name"]
5166                         and kdu["member-vnf-index"] == vnf_index
5167                     ):
5168                         break
5169                 else:
5170                     raise LcmException(
5171                         "KDU '{}' for vnf '{}' not deployed".format(kdu_name, vnf_index)
5172                     )
5173
5174                 if kdu.get("k8scluster-type") not in self.k8scluster_map:
5175                     msg = "unknown k8scluster-type '{}'".format(
5176                         kdu.get("k8scluster-type")
5177                     )
5178                     raise LcmException(msg)
5179
5180                 db_dict = {
5181                     "collection": "nsrs",
5182                     "filter": {"_id": nsr_id},
5183                     "path": "_admin.deployed.K8s.{}".format(index),
5184                 }
5185                 self.logger.debug(
5186                     logging_text
5187                     + "Exec k8s {} on {}.{}".format(primitive_name, vnf_index, kdu_name)
5188                 )
5189                 step = "Executing kdu {}".format(primitive_name)
5190                 if primitive_name == "upgrade":
5191                     if desc_params.get("kdu_model"):
5192                         kdu_model = desc_params.get("kdu_model")
5193                         del desc_params["kdu_model"]
5194                     else:
5195                         kdu_model = kdu.get("kdu-model")
5196                         parts = kdu_model.split(sep=":")
5197                         if len(parts) == 2:
5198                             kdu_model = parts[0]
5199
5200                     detailed_status = await asyncio.wait_for(
5201                         self.k8scluster_map[kdu["k8scluster-type"]].upgrade(
5202                             cluster_uuid=kdu.get("k8scluster-uuid"),
5203                             kdu_instance=kdu.get("kdu-instance"),
5204                             atomic=True,
5205                             kdu_model=kdu_model,
5206                             params=desc_params,
5207                             db_dict=db_dict,
5208                             timeout=timeout_ns_action,
5209                         ),
5210                         timeout=timeout_ns_action + 10,
5211                     )
5212                     self.logger.debug(
5213                         logging_text + " Upgrade of kdu {} done".format(detailed_status)
5214                     )
5215                 elif primitive_name == "rollback":
5216                     detailed_status = await asyncio.wait_for(
5217                         self.k8scluster_map[kdu["k8scluster-type"]].rollback(
5218                             cluster_uuid=kdu.get("k8scluster-uuid"),
5219                             kdu_instance=kdu.get("kdu-instance"),
5220                             db_dict=db_dict,
5221                         ),
5222                         timeout=timeout_ns_action,
5223                     )
5224                 elif primitive_name == "status":
5225                     detailed_status = await asyncio.wait_for(
5226                         self.k8scluster_map[kdu["k8scluster-type"]].status_kdu(
5227                             cluster_uuid=kdu.get("k8scluster-uuid"),
5228                             kdu_instance=kdu.get("kdu-instance"),
5229                             vca_id=vca_id,
5230                         ),
5231                         timeout=timeout_ns_action,
5232                     )
5233                 else:
5234                     kdu_instance = kdu.get("kdu-instance") or "{}-{}".format(
5235                         kdu["kdu-name"], nsr_id
5236                     )
5237                     params = self._map_primitive_params(
5238                         config_primitive_desc, primitive_params, desc_params
5239                     )
5240
5241                     detailed_status = await asyncio.wait_for(
5242                         self.k8scluster_map[kdu["k8scluster-type"]].exec_primitive(
5243                             cluster_uuid=kdu.get("k8scluster-uuid"),
5244                             kdu_instance=kdu_instance,
5245                             primitive_name=primitive_name,
5246                             params=params,
5247                             db_dict=db_dict,
5248                             timeout=timeout_ns_action,
5249                             vca_id=vca_id,
5250                         ),
5251                         timeout=timeout_ns_action,
5252                     )
5253
5254                 if detailed_status:
5255                     nslcmop_operation_state = "COMPLETED"
5256                 else:
5257                     detailed_status = ""
5258                     nslcmop_operation_state = "FAILED"
5259             else:
5260                 ee_id, vca_type = self._look_for_deployed_vca(
5261                     nsr_deployed["VCA"],
5262                     member_vnf_index=vnf_index,
5263                     vdu_id=vdu_id,
5264                     vdu_count_index=vdu_count_index,
5265                     ee_descriptor_id=ee_descriptor_id,
5266                 )
5267                 for vca_index, vca_deployed in enumerate(
5268                     db_nsr["_admin"]["deployed"]["VCA"]
5269                 ):
5270                     if vca_deployed.get("member-vnf-index") == vnf_index:
5271                         db_dict = {
5272                             "collection": "nsrs",
5273                             "filter": {"_id": nsr_id},
5274                             "path": "_admin.deployed.VCA.{}.".format(vca_index),
5275                         }
5276                         break
5277                 (
5278                     nslcmop_operation_state,
5279                     detailed_status,
5280                 ) = await self._ns_execute_primitive(
5281                     ee_id,
5282                     primitive=primitive_name,
5283                     primitive_params=self._map_primitive_params(
5284                         config_primitive_desc, primitive_params, desc_params
5285                     ),
5286                     timeout=timeout_ns_action,
5287                     vca_type=vca_type,
5288                     db_dict=db_dict,
5289                     vca_id=vca_id,
5290                 )
5291
5292             db_nslcmop_update["detailed-status"] = detailed_status
5293             error_description_nslcmop = (
5294                 detailed_status if nslcmop_operation_state == "FAILED" else ""
5295             )
5296             self.logger.debug(
5297                 logging_text
5298                 + " task Done with result {} {}".format(
5299                     nslcmop_operation_state, detailed_status
5300                 )
5301             )
5302             return  # database update is called inside finally
5303
5304         except (DbException, LcmException, N2VCException, K8sException) as e:
5305             self.logger.error(logging_text + "Exit Exception {}".format(e))
5306             exc = e
5307         except asyncio.CancelledError:
5308             self.logger.error(
5309                 logging_text + "Cancelled Exception while '{}'".format(step)
5310             )
5311             exc = "Operation was cancelled"
5312         except asyncio.TimeoutError:
5313             self.logger.error(logging_text + "Timeout while '{}'".format(step))
5314             exc = "Timeout"
5315         except Exception as e:
5316             exc = traceback.format_exc()
5317             self.logger.critical(
5318                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
5319                 exc_info=True,
5320             )
5321         finally:
5322             if exc:
5323                 db_nslcmop_update[
5324                     "detailed-status"
5325                 ] = (
5326                     detailed_status
5327                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
5328                 nslcmop_operation_state = "FAILED"
5329             if db_nsr:
5330                 self._write_ns_status(
5331                     nsr_id=nsr_id,
5332                     ns_state=db_nsr[
5333                         "nsState"
5334                     ],  # TODO check if degraded. For the moment use previous status
5335                     current_operation="IDLE",
5336                     current_operation_id=None,
5337                     # error_description=error_description_nsr,
5338                     # error_detail=error_detail,
5339                     other_update=db_nsr_update,
5340                 )
5341
5342             self._write_op_status(
5343                 op_id=nslcmop_id,
5344                 stage="",
5345                 error_message=error_description_nslcmop,
5346                 operation_state=nslcmop_operation_state,
5347                 other_update=db_nslcmop_update,
5348             )
5349
5350             if nslcmop_operation_state:
5351                 try:
5352                     await self.msg.aiowrite(
5353                         "ns",
5354                         "actioned",
5355                         {
5356                             "nsr_id": nsr_id,
5357                             "nslcmop_id": nslcmop_id,
5358                             "operationState": nslcmop_operation_state,
5359                         },
5360                         loop=self.loop,
5361                     )
5362                 except Exception as e:
5363                     self.logger.error(
5364                         logging_text + "kafka_write notification Exception {}".format(e)
5365                     )
5366             self.logger.debug(logging_text + "Exit")
5367             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_action")
5368             return nslcmop_operation_state, detailed_status
5369
5370     async def terminate_vdus(
5371         self, db_vnfr, member_vnf_index, db_nsr, update_db_nslcmops, stage, logging_text
5372     ):
5373         """This method terminates VDUs
5374
5375         Args:
5376             db_vnfr: VNF instance record
5377             member_vnf_index: VNF index to identify the VDUs to be removed
5378             db_nsr: NS instance record
5379             update_db_nslcmops: Nslcmop update record
5380         """
5381         vca_scaling_info = []
5382         scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5383         scaling_info["scaling_direction"] = "IN"
5384         scaling_info["vdu-delete"] = {}
5385         scaling_info["kdu-delete"] = {}
5386         db_vdur = db_vnfr.get("vdur")
5387         vdur_list = copy(db_vdur)
5388         count_index = 0
5389         for index, vdu in enumerate(vdur_list):
5390             vca_scaling_info.append(
5391                 {
5392                     "osm_vdu_id": vdu["vdu-id-ref"],
5393                     "member-vnf-index": member_vnf_index,
5394                     "type": "delete",
5395                     "vdu_index": count_index,
5396                     })
5397             scaling_info["vdu-delete"][vdu["vdu-id-ref"]] = count_index
5398             scaling_info["vdu"].append(
5399                 {
5400                     "name": vdu.get("name") or vdu.get("vdu-name"),
5401                     "vdu_id": vdu["vdu-id-ref"],
5402                     "interface": [],
5403                     })
5404             for interface in vdu["interfaces"]:
5405                 scaling_info["vdu"][index]["interface"].append(
5406                     {
5407                         "name": interface["name"],
5408                         "ip_address": interface["ip-address"],
5409                         "mac_address": interface.get("mac-address"),
5410                         })
5411             self.logger.info("NS update scaling info{}".format(scaling_info))
5412             stage[2] = "Terminating VDUs"
5413             if scaling_info.get("vdu-delete"):
5414                 # scale_process = "RO"
5415                 if self.ro_config.get("ng"):
5416                     await self._scale_ng_ro(
5417                         logging_text, db_nsr, update_db_nslcmops, db_vnfr, scaling_info, stage
5418                     )
5419
5420     async def remove_vnf(
5421         self, nsr_id, nslcmop_id, vnf_instance_id
5422     ):
5423         """This method is to Remove VNF instances from NS.
5424
5425         Args:
5426             nsr_id: NS instance id
5427             nslcmop_id: nslcmop id of update
5428             vnf_instance_id: id of the VNF instance to be removed
5429
5430         Returns:
5431             result: (str, str) COMPLETED/FAILED, details
5432         """
5433         try:
5434             db_nsr_update = {}
5435             logging_text = "Task ns={} update ".format(nsr_id)
5436             check_vnfr_count = len(self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}))
5437             self.logger.info("check_vnfr_count {}".format(check_vnfr_count))
5438             if check_vnfr_count > 1:
5439                 stage = ["", "", ""]
5440                 step = "Getting nslcmop from database"
5441                 self.logger.debug(step + " after having waited for previous tasks to be completed")
5442                 # db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5443                 db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5444                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
5445                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
5446                 """ db_vnfr = self.db.get_one(
5447                     "vnfrs", {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id}) """
5448
5449                 update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5450                 await self.terminate_vdus(db_vnfr, member_vnf_index, db_nsr, update_db_nslcmops, stage, logging_text)
5451
5452                 constituent_vnfr = db_nsr.get("constituent-vnfr-ref")
5453                 constituent_vnfr.remove(db_vnfr.get("_id"))
5454                 db_nsr_update["constituent-vnfr-ref"] = db_nsr.get("constituent-vnfr-ref")
5455                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5456                 self.db.del_one("vnfrs", {"_id": db_vnfr.get("_id")})
5457                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
5458                 return "COMPLETED", "Done"
5459             else:
5460                 step = "Terminate VNF Failed with"
5461                 raise LcmException("{} Cannot terminate the last VNF in this NS.".format(
5462                     vnf_instance_id))
5463         except (LcmException, asyncio.CancelledError):
5464             raise
5465         except Exception as e:
5466             self.logger.debug("Error removing VNF {}".format(e))
5467             return "FAILED", "Error removing VNF {}".format(e)
5468
5469     async def _ns_redeploy_vnf(
5470         self, nsr_id, nslcmop_id, db_vnfd, db_vnfr, db_nsr,
5471     ):
5472         """This method updates and redeploys VNF instances
5473
5474         Args:
5475             nsr_id: NS instance id
5476             nslcmop_id:   nslcmop id
5477             db_vnfd: VNF descriptor
5478             db_vnfr: VNF instance record
5479             db_nsr: NS instance record
5480
5481         Returns:
5482             result: (str, str) COMPLETED/FAILED, details
5483         """
5484         try:
5485             count_index = 0
5486             stage = ["", "", ""]
5487             logging_text = "Task ns={} update ".format(nsr_id)
5488             latest_vnfd_revision = db_vnfd["_admin"].get("revision")
5489             member_vnf_index = db_vnfr["member-vnf-index-ref"]
5490
5491             # Terminate old VNF resources
5492             update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5493             await self.terminate_vdus(db_vnfr, member_vnf_index, db_nsr, update_db_nslcmops, stage, logging_text)
5494
5495             # old_vnfd_id = db_vnfr["vnfd-id"]
5496             # new_db_vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
5497             new_db_vnfd = db_vnfd
5498             # new_vnfd_ref = new_db_vnfd["id"]
5499             # new_vnfd_id = vnfd_id
5500
5501             # Create VDUR
5502             new_vnfr_cp = []
5503             for cp in new_db_vnfd.get("ext-cpd", ()):
5504                 vnf_cp = {
5505                     "name": cp.get("id"),
5506                     "connection-point-id": cp.get("int-cpd", {}).get("cpd"),
5507                     "connection-point-vdu-id": cp.get("int-cpd", {}).get("vdu-id"),
5508                     "id": cp.get("id"),
5509                 }
5510                 new_vnfr_cp.append(vnf_cp)
5511             new_vdur = update_db_nslcmops["operationParams"]["newVdur"]
5512             # new_vdur = self._create_vdur_descriptor_from_vnfd(db_nsd, db_vnfd, old_db_vnfd, vnfd_id, db_nsr, member_vnf_index)
5513             # new_vnfr_update = {"vnfd-ref": new_vnfd_ref, "vnfd-id": new_vnfd_id, "connection-point": new_vnfr_cp, "vdur": new_vdur, "ip-address": ""}
5514             new_vnfr_update = {"revision": latest_vnfd_revision, "connection-point": new_vnfr_cp, "vdur": new_vdur, "ip-address": ""}
5515             self.update_db_2("vnfrs", db_vnfr["_id"], new_vnfr_update)
5516             updated_db_vnfr = self.db.get_one(
5517                 "vnfrs", {"member-vnf-index-ref": member_vnf_index, "nsr-id-ref": nsr_id}
5518             )
5519
5520             # Instantiate new VNF resources
5521             # update_db_nslcmops = self.db.get_one("nslcmops", {"_id": nslcmop_id})
5522             vca_scaling_info = []
5523             scaling_info = {"scaling_group_name": "vdu_autoscale", "vdu": [], "kdu": []}
5524             scaling_info["scaling_direction"] = "OUT"
5525             scaling_info["vdu-create"] = {}
5526             scaling_info["kdu-create"] = {}
5527             vdud_instantiate_list = db_vnfd["vdu"]
5528             for index, vdud in enumerate(vdud_instantiate_list):
5529                 cloud_init_text = self._get_vdu_cloud_init_content(
5530                     vdud, db_vnfd
5531                 )
5532                 if cloud_init_text:
5533                     additional_params = (
5534                         self._get_vdu_additional_params(updated_db_vnfr, vdud["id"])
5535                         or {}
5536                     )
5537                 cloud_init_list = []
5538                 if cloud_init_text:
5539                     # TODO Information of its own ip is not available because db_vnfr is not updated.
5540                     additional_params["OSM"] = get_osm_params(
5541                         updated_db_vnfr, vdud["id"], 1
5542                     )
5543                     cloud_init_list.append(
5544                         self._parse_cloud_init(
5545                             cloud_init_text,
5546                             additional_params,
5547                             db_vnfd["id"],
5548                             vdud["id"],
5549                         )
5550                     )
5551                     vca_scaling_info.append(
5552                         {
5553                             "osm_vdu_id": vdud["id"],
5554                             "member-vnf-index": member_vnf_index,
5555                             "type": "create",
5556                             "vdu_index": count_index,
5557                         }
5558                     )
5559                 scaling_info["vdu-create"][vdud["id"]] = count_index
5560             if self.ro_config.get("ng"):
5561                 self.logger.debug(
5562                     "New Resources to be deployed: {}".format(scaling_info))
5563                 await self._scale_ng_ro(
5564                     logging_text, db_nsr, update_db_nslcmops, updated_db_vnfr, scaling_info, stage
5565                 )
5566                 return "COMPLETED", "Done"
5567         except (LcmException, asyncio.CancelledError):
5568             raise
5569         except Exception as e:
5570             self.logger.debug("Error updating VNF {}".format(e))
5571             return "FAILED", "Error updating VNF {}".format(e)
5572
5573     async def _ns_charm_upgrade(
5574         self,
5575         ee_id,
5576         charm_id,
5577         charm_type,
5578         path,
5579         timeout: float = None,
5580     ) -> (str, str):
5581         """This method upgrade charms in VNF instances
5582
5583         Args:
5584             ee_id:  Execution environment id
5585             path:   Local path to the charm
5586             charm_id: charm-id
5587             charm_type: Charm type can be lxc-proxy-charm, native-charm or k8s-proxy-charm
5588             timeout: (Float)    Timeout for the ns update operation
5589
5590         Returns:
5591             result: (str, str) COMPLETED/FAILED, details
5592         """
5593         try:
5594             charm_type = charm_type or "lxc_proxy_charm"
5595             output = await self.vca_map[charm_type].upgrade_charm(
5596                 ee_id=ee_id,
5597                 path=path,
5598                 charm_id=charm_id,
5599                 charm_type=charm_type,
5600                 timeout=timeout or self.timeout_ns_update,
5601             )
5602
5603             if output:
5604                 return "COMPLETED", output
5605
5606         except (LcmException, asyncio.CancelledError):
5607             raise
5608
5609         except Exception as e:
5610
5611             self.logger.debug("Error upgrading charm {}".format(path))
5612
5613             return "FAILED", "Error upgrading charm {}: {}".format(path, e)
5614
5615     async def update(self, nsr_id, nslcmop_id):
5616         """Update NS according to different update types
5617
5618         This method performs upgrade of VNF instances then updates the revision
5619         number in VNF record
5620
5621         Args:
5622             nsr_id: Network service will be updated
5623             nslcmop_id: ns lcm operation id
5624
5625         Returns:
5626              It may raise DbException, LcmException, N2VCException, K8sException
5627
5628         """
5629         # Try to lock HA task here
5630         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5631         if not task_is_locked_by_me:
5632             return
5633
5634         logging_text = "Task ns={} update={} ".format(nsr_id, nslcmop_id)
5635         self.logger.debug(logging_text + "Enter")
5636
5637         # Set the required variables to be filled up later
5638         db_nsr = None
5639         db_nslcmop_update = {}
5640         vnfr_update = {}
5641         nslcmop_operation_state = None
5642         db_nsr_update = {}
5643         error_description_nslcmop = ""
5644         exc = None
5645         change_type = "updated"
5646         detailed_status = ""
5647
5648         try:
5649             # wait for any previous tasks in process
5650             step = "Waiting for previous operations to terminate"
5651             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
5652             self._write_ns_status(
5653                 nsr_id=nsr_id,
5654                 ns_state=None,
5655                 current_operation="UPDATING",
5656                 current_operation_id=nslcmop_id,
5657             )
5658
5659             step = "Getting nslcmop from database"
5660             db_nslcmop = self.db.get_one(
5661                 "nslcmops", {"_id": nslcmop_id}, fail_on_empty=False
5662             )
5663             update_type = db_nslcmop["operationParams"]["updateType"]
5664
5665             step = "Getting nsr from database"
5666             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
5667             old_operational_status = db_nsr["operational-status"]
5668             db_nsr_update["operational-status"] = "updating"
5669             self.update_db_2("nsrs", nsr_id, db_nsr_update)
5670             nsr_deployed = db_nsr["_admin"].get("deployed")
5671
5672             if update_type == "CHANGE_VNFPKG":
5673
5674                 # Get the input parameters given through update request
5675                 vnf_instance_id = db_nslcmop["operationParams"][
5676                     "changeVnfPackageData"
5677                 ].get("vnfInstanceId")
5678
5679                 vnfd_id = db_nslcmop["operationParams"]["changeVnfPackageData"].get(
5680                     "vnfdId"
5681                 )
5682                 timeout_seconds = db_nslcmop["operationParams"].get("timeout_ns_update")
5683
5684                 step = "Getting vnfr from database"
5685                 db_vnfr = self.db.get_one(
5686                     "vnfrs", {"_id": vnf_instance_id}, fail_on_empty=False
5687                 )
5688
5689                 step = "Getting vnfds from database"
5690                 # Latest VNFD
5691                 latest_vnfd = self.db.get_one(
5692                     "vnfds", {"_id": vnfd_id}, fail_on_empty=False
5693                 )
5694                 latest_vnfd_revision = latest_vnfd["_admin"].get("revision")
5695
5696                 # Current VNFD
5697                 current_vnf_revision = db_vnfr.get("revision", 1)
5698                 current_vnfd = self.db.get_one(
5699                     "vnfds_revisions",
5700                     {"_id": vnfd_id + ":" + str(current_vnf_revision)},
5701                     fail_on_empty=False,
5702                 )
5703                 # Charm artifact paths will be filled up later
5704                 (
5705                     current_charm_artifact_path,
5706                     target_charm_artifact_path,
5707                     charm_artifact_paths,
5708                 ) = ([], [], [])
5709
5710                 step = "Checking if revision has changed in VNFD"
5711                 if current_vnf_revision != latest_vnfd_revision:
5712
5713                     change_type = "policy_updated"
5714
5715                     # There is new revision of VNFD, update operation is required
5716                     current_vnfd_path = vnfd_id + ":" + str(current_vnf_revision)
5717                     latest_vnfd_path = vnfd_id + ":" + str(latest_vnfd_revision)
5718
5719                     step = "Removing the VNFD packages if they exist in the local path"
5720                     shutil.rmtree(self.fs.path + current_vnfd_path, ignore_errors=True)
5721                     shutil.rmtree(self.fs.path + latest_vnfd_path, ignore_errors=True)
5722
5723                     step = "Get the VNFD packages from FSMongo"
5724                     self.fs.sync(from_path=latest_vnfd_path)
5725                     self.fs.sync(from_path=current_vnfd_path)
5726
5727                     step = (
5728                         "Get the charm-type, charm-id, ee-id if there is deployed VCA"
5729                     )
5730                     base_folder = latest_vnfd["_admin"]["storage"]
5731
5732                     for charm_index, charm_deployed in enumerate(
5733                         get_iterable(nsr_deployed, "VCA")
5734                     ):
5735                         vnf_index = db_vnfr.get("member-vnf-index-ref")
5736
5737                         # Getting charm-id and charm-type
5738                         if charm_deployed.get("member-vnf-index") == vnf_index:
5739                             charm_id = self.get_vca_id(db_vnfr, db_nsr)
5740                             charm_type = charm_deployed.get("type")
5741
5742                             # Getting ee-id
5743                             ee_id = charm_deployed.get("ee_id")
5744
5745                             step = "Getting descriptor config"
5746                             descriptor_config = get_configuration(
5747                                 current_vnfd, current_vnfd["id"]
5748                             )
5749
5750                             if "execution-environment-list" in descriptor_config:
5751                                 ee_list = descriptor_config.get(
5752                                     "execution-environment-list", []
5753                                 )
5754                             else:
5755                                 ee_list = []
5756
5757                             # There could be several charm used in the same VNF
5758                             for ee_item in ee_list:
5759                                 if ee_item.get("juju"):
5760
5761                                     step = "Getting charm name"
5762                                     charm_name = ee_item["juju"].get("charm")
5763
5764                                     step = "Setting Charm artifact paths"
5765                                     current_charm_artifact_path.append(
5766                                         get_charm_artifact_path(
5767                                             base_folder,
5768                                             charm_name,
5769                                             charm_type,
5770                                             current_vnf_revision,
5771                                         )
5772                                     )
5773                                     target_charm_artifact_path.append(
5774                                         get_charm_artifact_path(
5775                                             base_folder,
5776                                             charm_name,
5777                                             charm_type,
5778                                             latest_vnfd_revision,
5779                                         )
5780                                     )
5781
5782                             charm_artifact_paths = zip(
5783                                 current_charm_artifact_path, target_charm_artifact_path
5784                             )
5785
5786                     step = "Checking if software version has changed in VNFD"
5787                     if find_software_version(current_vnfd) != find_software_version(
5788                         latest_vnfd
5789                     ):
5790
5791                         step = "Checking if existing VNF has charm"
5792                         for current_charm_path, target_charm_path in list(
5793                             charm_artifact_paths
5794                         ):
5795                             if current_charm_path:
5796                                 raise LcmException(
5797                                     "Software version change is not supported as VNF instance {} has charm.".format(
5798                                         vnf_instance_id
5799                                     )
5800                                 )
5801
5802                         # There is no change in the charm package, then redeploy the VNF
5803                         # based on new descriptor
5804                         step = "Redeploying VNF"
5805                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5806                         (
5807                             result,
5808                             detailed_status
5809                         ) = await self._ns_redeploy_vnf(
5810                             nsr_id,
5811                             nslcmop_id,
5812                             latest_vnfd,
5813                             db_vnfr,
5814                             db_nsr
5815                         )
5816                         if result == "FAILED":
5817                             nslcmop_operation_state = result
5818                             error_description_nslcmop = detailed_status
5819                         db_nslcmop_update["detailed-status"] = detailed_status
5820                         self.logger.debug(
5821                             logging_text
5822                             + " step {} Done with result {} {}".format(
5823                                 step, nslcmop_operation_state, detailed_status
5824                             )
5825                         )
5826
5827                     else:
5828                         step = "Checking if any charm package has changed or not"
5829                         for current_charm_path, target_charm_path in list(
5830                             charm_artifact_paths
5831                         ):
5832                             if (
5833                                 current_charm_path
5834                                 and target_charm_path
5835                                 and self.check_charm_hash_changed(
5836                                     current_charm_path, target_charm_path
5837                                 )
5838                             ):
5839
5840                                 step = "Checking whether VNF uses juju bundle"
5841                                 if check_juju_bundle_existence(current_vnfd):
5842
5843                                     raise LcmException(
5844                                         "Charm upgrade is not supported for the instance which"
5845                                         " uses juju-bundle: {}".format(
5846                                             check_juju_bundle_existence(current_vnfd)
5847                                         )
5848                                     )
5849
5850                                 step = "Upgrading Charm"
5851                                 (
5852                                     result,
5853                                     detailed_status,
5854                                 ) = await self._ns_charm_upgrade(
5855                                     ee_id=ee_id,
5856                                     charm_id=charm_id,
5857                                     charm_type=charm_type,
5858                                     path=self.fs.path + target_charm_path,
5859                                     timeout=timeout_seconds,
5860                                 )
5861
5862                                 if result == "FAILED":
5863                                     nslcmop_operation_state = result
5864                                     error_description_nslcmop = detailed_status
5865
5866                                 db_nslcmop_update["detailed-status"] = detailed_status
5867                                 self.logger.debug(
5868                                     logging_text
5869                                     + " step {} Done with result {} {}".format(
5870                                         step, nslcmop_operation_state, detailed_status
5871                                     )
5872                                 )
5873
5874                         step = "Updating policies"
5875                         member_vnf_index = db_vnfr["member-vnf-index-ref"]
5876                         result = "COMPLETED"
5877                         detailed_status = "Done"
5878                         db_nslcmop_update["detailed-status"] = "Done"
5879
5880                     #  If nslcmop_operation_state is None, so any operation is not failed.
5881                     if not nslcmop_operation_state:
5882                         nslcmop_operation_state = "COMPLETED"
5883
5884                         # If update CHANGE_VNFPKG nslcmop_operation is successful
5885                         # vnf revision need to be updated
5886                         vnfr_update["revision"] = latest_vnfd_revision
5887                         self.update_db_2("vnfrs", db_vnfr["_id"], vnfr_update)
5888
5889                     self.logger.debug(
5890                         logging_text
5891                         + " task Done with result {} {}".format(
5892                             nslcmop_operation_state, detailed_status
5893                         )
5894                     )
5895             elif update_type == "REMOVE_VNF":
5896                 # This part is included in https://osm.etsi.org/gerrit/11876
5897                 vnf_instance_id = db_nslcmop["operationParams"]["removeVnfInstanceId"]
5898                 db_vnfr = self.db.get_one("vnfrs", {"_id": vnf_instance_id})
5899                 member_vnf_index = db_vnfr["member-vnf-index-ref"]
5900                 step = "Removing VNF"
5901                 (result, detailed_status) = await self.remove_vnf(nsr_id, nslcmop_id, vnf_instance_id)
5902                 if result == "FAILED":
5903                     nslcmop_operation_state = result
5904                     error_description_nslcmop = detailed_status
5905                 db_nslcmop_update["detailed-status"] = detailed_status
5906                 change_type = "vnf_terminated"
5907                 if not nslcmop_operation_state:
5908                     nslcmop_operation_state = "COMPLETED"
5909                 self.logger.debug(
5910                     logging_text
5911                     + " task Done with result {} {}".format(
5912                         nslcmop_operation_state, detailed_status
5913                     )
5914                 )
5915
5916             #  If nslcmop_operation_state is None, so any operation is not failed.
5917             #  All operations are executed in overall.
5918             if not nslcmop_operation_state:
5919                 nslcmop_operation_state = "COMPLETED"
5920             db_nsr_update["operational-status"] = old_operational_status
5921
5922         except (DbException, LcmException, N2VCException, K8sException) as e:
5923             self.logger.error(logging_text + "Exit Exception {}".format(e))
5924             exc = e
5925         except asyncio.CancelledError:
5926             self.logger.error(
5927                 logging_text + "Cancelled Exception while '{}'".format(step)
5928             )
5929             exc = "Operation was cancelled"
5930         except asyncio.TimeoutError:
5931             self.logger.error(logging_text + "Timeout while '{}'".format(step))
5932             exc = "Timeout"
5933         except Exception as e:
5934             exc = traceback.format_exc()
5935             self.logger.critical(
5936                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
5937                 exc_info=True,
5938             )
5939         finally:
5940             if exc:
5941                 db_nslcmop_update[
5942                     "detailed-status"
5943                 ] = (
5944                     detailed_status
5945                 ) = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
5946                 nslcmop_operation_state = "FAILED"
5947                 db_nsr_update["operational-status"] = old_operational_status
5948             if db_nsr:
5949                 self._write_ns_status(
5950                     nsr_id=nsr_id,
5951                     ns_state=db_nsr["nsState"],
5952                     current_operation="IDLE",
5953                     current_operation_id=None,
5954                     other_update=db_nsr_update,
5955                 )
5956
5957             self._write_op_status(
5958                 op_id=nslcmop_id,
5959                 stage="",
5960                 error_message=error_description_nslcmop,
5961                 operation_state=nslcmop_operation_state,
5962                 other_update=db_nslcmop_update,
5963             )
5964
5965             if nslcmop_operation_state:
5966                 try:
5967                     msg = {
5968                         "nsr_id": nsr_id,
5969                         "nslcmop_id": nslcmop_id,
5970                         "operationState": nslcmop_operation_state,
5971                     }
5972                     if change_type in ("vnf_terminated", "policy_updated"):
5973                         msg.update({"vnf_member_index": member_vnf_index})
5974                     await self.msg.aiowrite("ns", change_type, msg, loop=self.loop)
5975                 except Exception as e:
5976                     self.logger.error(
5977                         logging_text + "kafka_write notification Exception {}".format(e)
5978                     )
5979             self.logger.debug(logging_text + "Exit")
5980             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_update")
5981             return nslcmop_operation_state, detailed_status
5982
5983     async def scale(self, nsr_id, nslcmop_id):
5984         # Try to lock HA task here
5985         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
5986         if not task_is_locked_by_me:
5987             return
5988
5989         logging_text = "Task ns={} scale={} ".format(nsr_id, nslcmop_id)
5990         stage = ["", "", ""]
5991         tasks_dict_info = {}
5992         # ^ stage, step, VIM progress
5993         self.logger.debug(logging_text + "Enter")
5994         # get all needed from database
5995         db_nsr = None
5996         db_nslcmop_update = {}
5997         db_nsr_update = {}
5998         exc = None
5999         # in case of error, indicates what part of scale was failed to put nsr at error status
6000         scale_process = None
6001         old_operational_status = ""
6002         old_config_status = ""
6003         nsi_id = None
6004         try:
6005             # wait for any previous tasks in process
6006             step = "Waiting for previous operations to terminate"
6007             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
6008             self._write_ns_status(
6009                 nsr_id=nsr_id,
6010                 ns_state=None,
6011                 current_operation="SCALING",
6012                 current_operation_id=nslcmop_id,
6013             )
6014
6015             step = "Getting nslcmop from database"
6016             self.logger.debug(
6017                 step + " after having waited for previous tasks to be completed"
6018             )
6019             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
6020
6021             step = "Getting nsr from database"
6022             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
6023             old_operational_status = db_nsr["operational-status"]
6024             old_config_status = db_nsr["config-status"]
6025
6026             step = "Parsing scaling parameters"
6027             db_nsr_update["operational-status"] = "scaling"
6028             self.update_db_2("nsrs", nsr_id, db_nsr_update)
6029             nsr_deployed = db_nsr["_admin"].get("deployed")
6030
6031             vnf_index = db_nslcmop["operationParams"]["scaleVnfData"][
6032                 "scaleByStepData"
6033             ]["member-vnf-index"]
6034             scaling_group = db_nslcmop["operationParams"]["scaleVnfData"][
6035                 "scaleByStepData"
6036             ]["scaling-group-descriptor"]
6037             scaling_type = db_nslcmop["operationParams"]["scaleVnfData"]["scaleVnfType"]
6038             # for backward compatibility
6039             if nsr_deployed and isinstance(nsr_deployed.get("VCA"), dict):
6040                 nsr_deployed["VCA"] = list(nsr_deployed["VCA"].values())
6041                 db_nsr_update["_admin.deployed.VCA"] = nsr_deployed["VCA"]
6042                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6043
6044             step = "Getting vnfr from database"
6045             db_vnfr = self.db.get_one(
6046                 "vnfrs", {"member-vnf-index-ref": vnf_index, "nsr-id-ref": nsr_id}
6047             )
6048
6049             vca_id = self.get_vca_id(db_vnfr, db_nsr)
6050
6051             step = "Getting vnfd from database"
6052             db_vnfd = self.db.get_one("vnfds", {"_id": db_vnfr["vnfd-id"]})
6053
6054             base_folder = db_vnfd["_admin"]["storage"]
6055
6056             step = "Getting scaling-group-descriptor"
6057             scaling_descriptor = find_in_list(
6058                 get_scaling_aspect(db_vnfd),
6059                 lambda scale_desc: scale_desc["name"] == scaling_group,
6060             )
6061             if not scaling_descriptor:
6062                 raise LcmException(
6063                     "input parameter 'scaleByStepData':'scaling-group-descriptor':'{}' is not present "
6064                     "at vnfd:scaling-group-descriptor".format(scaling_group)
6065                 )
6066
6067             step = "Sending scale order to VIM"
6068             # TODO check if ns is in a proper status
6069             nb_scale_op = 0
6070             if not db_nsr["_admin"].get("scaling-group"):
6071                 self.update_db_2(
6072                     "nsrs",
6073                     nsr_id,
6074                     {
6075                         "_admin.scaling-group": [
6076                             {"name": scaling_group, "nb-scale-op": 0}
6077                         ]
6078                     },
6079                 )
6080                 admin_scale_index = 0
6081             else:
6082                 for admin_scale_index, admin_scale_info in enumerate(
6083                     db_nsr["_admin"]["scaling-group"]
6084                 ):
6085                     if admin_scale_info["name"] == scaling_group:
6086                         nb_scale_op = admin_scale_info.get("nb-scale-op", 0)
6087                         break
6088                 else:  # not found, set index one plus last element and add new entry with the name
6089                     admin_scale_index += 1
6090                     db_nsr_update[
6091                         "_admin.scaling-group.{}.name".format(admin_scale_index)
6092                     ] = scaling_group
6093
6094             vca_scaling_info = []
6095             scaling_info = {"scaling_group_name": scaling_group, "vdu": [], "kdu": []}
6096             if scaling_type == "SCALE_OUT":
6097                 if "aspect-delta-details" not in scaling_descriptor:
6098                     raise LcmException(
6099                         "Aspect delta details not fount in scaling descriptor {}".format(
6100                             scaling_descriptor["name"]
6101                         )
6102                     )
6103                 # count if max-instance-count is reached
6104                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6105
6106                 scaling_info["scaling_direction"] = "OUT"
6107                 scaling_info["vdu-create"] = {}
6108                 scaling_info["kdu-create"] = {}
6109                 for delta in deltas:
6110                     for vdu_delta in delta.get("vdu-delta", {}):
6111                         vdud = get_vdu(db_vnfd, vdu_delta["id"])
6112                         # vdu_index also provides the number of instance of the targeted vdu
6113                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6114                         cloud_init_text = self._get_vdu_cloud_init_content(
6115                             vdud, db_vnfd
6116                         )
6117                         if cloud_init_text:
6118                             additional_params = (
6119                                 self._get_vdu_additional_params(db_vnfr, vdud["id"])
6120                                 or {}
6121                             )
6122                         cloud_init_list = []
6123
6124                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6125                         max_instance_count = 10
6126                         if vdu_profile and "max-number-of-instances" in vdu_profile:
6127                             max_instance_count = vdu_profile.get(
6128                                 "max-number-of-instances", 10
6129                             )
6130
6131                         default_instance_num = get_number_of_instances(
6132                             db_vnfd, vdud["id"]
6133                         )
6134                         instances_number = vdu_delta.get("number-of-instances", 1)
6135                         nb_scale_op += instances_number
6136
6137                         new_instance_count = nb_scale_op + default_instance_num
6138                         # Control if new count is over max and vdu count is less than max.
6139                         # Then assign new instance count
6140                         if new_instance_count > max_instance_count > vdu_count:
6141                             instances_number = new_instance_count - max_instance_count
6142                         else:
6143                             instances_number = instances_number
6144
6145                         if new_instance_count > max_instance_count:
6146                             raise LcmException(
6147                                 "reached the limit of {} (max-instance-count) "
6148                                 "scaling-out operations for the "
6149                                 "scaling-group-descriptor '{}'".format(
6150                                     nb_scale_op, scaling_group
6151                                 )
6152                             )
6153                         for x in range(vdu_delta.get("number-of-instances", 1)):
6154                             if cloud_init_text:
6155                                 # TODO Information of its own ip is not available because db_vnfr is not updated.
6156                                 additional_params["OSM"] = get_osm_params(
6157                                     db_vnfr, vdu_delta["id"], vdu_index + x
6158                                 )
6159                                 cloud_init_list.append(
6160                                     self._parse_cloud_init(
6161                                         cloud_init_text,
6162                                         additional_params,
6163                                         db_vnfd["id"],
6164                                         vdud["id"],
6165                                     )
6166                                 )
6167                                 vca_scaling_info.append(
6168                                     {
6169                                         "osm_vdu_id": vdu_delta["id"],
6170                                         "member-vnf-index": vnf_index,
6171                                         "type": "create",
6172                                         "vdu_index": vdu_index + x,
6173                                     }
6174                                 )
6175                         scaling_info["vdu-create"][vdu_delta["id"]] = instances_number
6176                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6177                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6178                         kdu_name = kdu_profile["kdu-name"]
6179                         resource_name = kdu_profile.get("resource-name", "")
6180
6181                         # Might have different kdus in the same delta
6182                         # Should have list for each kdu
6183                         if not scaling_info["kdu-create"].get(kdu_name, None):
6184                             scaling_info["kdu-create"][kdu_name] = []
6185
6186                         kdur = get_kdur(db_vnfr, kdu_name)
6187                         if kdur.get("helm-chart"):
6188                             k8s_cluster_type = "helm-chart-v3"
6189                             self.logger.debug("kdur: {}".format(kdur))
6190                             if (
6191                                 kdur.get("helm-version")
6192                                 and kdur.get("helm-version") == "v2"
6193                             ):
6194                                 k8s_cluster_type = "helm-chart"
6195                         elif kdur.get("juju-bundle"):
6196                             k8s_cluster_type = "juju-bundle"
6197                         else:
6198                             raise LcmException(
6199                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6200                                 "juju-bundle. Maybe an old NBI version is running".format(
6201                                     db_vnfr["member-vnf-index-ref"], kdu_name
6202                                 )
6203                             )
6204
6205                         max_instance_count = 10
6206                         if kdu_profile and "max-number-of-instances" in kdu_profile:
6207                             max_instance_count = kdu_profile.get(
6208                                 "max-number-of-instances", 10
6209                             )
6210
6211                         nb_scale_op += kdu_delta.get("number-of-instances", 1)
6212                         deployed_kdu, _ = get_deployed_kdu(
6213                             nsr_deployed, kdu_name, vnf_index
6214                         )
6215                         if deployed_kdu is None:
6216                             raise LcmException(
6217                                 "KDU '{}' for vnf '{}' not deployed".format(
6218                                     kdu_name, vnf_index
6219                                 )
6220                             )
6221                         kdu_instance = deployed_kdu.get("kdu-instance")
6222                         instance_num = await self.k8scluster_map[
6223                             k8s_cluster_type
6224                         ].get_scale_count(
6225                             resource_name,
6226                             kdu_instance,
6227                             vca_id=vca_id,
6228                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6229                             kdu_model=deployed_kdu.get("kdu-model"),
6230                         )
6231                         kdu_replica_count = instance_num + kdu_delta.get(
6232                             "number-of-instances", 1
6233                         )
6234
6235                         # Control if new count is over max and instance_num is less than max.
6236                         # Then assign max instance number to kdu replica count
6237                         if kdu_replica_count > max_instance_count > instance_num:
6238                             kdu_replica_count = max_instance_count
6239                         if kdu_replica_count > max_instance_count:
6240                             raise LcmException(
6241                                 "reached the limit of {} (max-instance-count) "
6242                                 "scaling-out operations for the "
6243                                 "scaling-group-descriptor '{}'".format(
6244                                     instance_num, scaling_group
6245                                 )
6246                             )
6247
6248                         for x in range(kdu_delta.get("number-of-instances", 1)):
6249                             vca_scaling_info.append(
6250                                 {
6251                                     "osm_kdu_id": kdu_name,
6252                                     "member-vnf-index": vnf_index,
6253                                     "type": "create",
6254                                     "kdu_index": instance_num + x - 1,
6255                                 }
6256                             )
6257                         scaling_info["kdu-create"][kdu_name].append(
6258                             {
6259                                 "member-vnf-index": vnf_index,
6260                                 "type": "create",
6261                                 "k8s-cluster-type": k8s_cluster_type,
6262                                 "resource-name": resource_name,
6263                                 "scale": kdu_replica_count,
6264                             }
6265                         )
6266             elif scaling_type == "SCALE_IN":
6267                 deltas = scaling_descriptor.get("aspect-delta-details")["deltas"]
6268
6269                 scaling_info["scaling_direction"] = "IN"
6270                 scaling_info["vdu-delete"] = {}
6271                 scaling_info["kdu-delete"] = {}
6272
6273                 for delta in deltas:
6274                     for vdu_delta in delta.get("vdu-delta", {}):
6275                         vdu_count = vdu_index = get_vdur_index(db_vnfr, vdu_delta)
6276                         min_instance_count = 0
6277                         vdu_profile = get_vdu_profile(db_vnfd, vdu_delta["id"])
6278                         if vdu_profile and "min-number-of-instances" in vdu_profile:
6279                             min_instance_count = vdu_profile["min-number-of-instances"]
6280
6281                         default_instance_num = get_number_of_instances(
6282                             db_vnfd, vdu_delta["id"]
6283                         )
6284                         instance_num = vdu_delta.get("number-of-instances", 1)
6285                         nb_scale_op -= instance_num
6286
6287                         new_instance_count = nb_scale_op + default_instance_num
6288
6289                         if new_instance_count < min_instance_count < vdu_count:
6290                             instances_number = min_instance_count - new_instance_count
6291                         else:
6292                             instances_number = instance_num
6293
6294                         if new_instance_count < min_instance_count:
6295                             raise LcmException(
6296                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6297                                 "scaling-group-descriptor '{}'".format(
6298                                     nb_scale_op, scaling_group
6299                                 )
6300                             )
6301                         for x in range(vdu_delta.get("number-of-instances", 1)):
6302                             vca_scaling_info.append(
6303                                 {
6304                                     "osm_vdu_id": vdu_delta["id"],
6305                                     "member-vnf-index": vnf_index,
6306                                     "type": "delete",
6307                                     "vdu_index": vdu_index - 1 - x,
6308                                 }
6309                             )
6310                         scaling_info["vdu-delete"][vdu_delta["id"]] = instances_number
6311                     for kdu_delta in delta.get("kdu-resource-delta", {}):
6312                         kdu_profile = get_kdu_resource_profile(db_vnfd, kdu_delta["id"])
6313                         kdu_name = kdu_profile["kdu-name"]
6314                         resource_name = kdu_profile.get("resource-name", "")
6315
6316                         if not scaling_info["kdu-delete"].get(kdu_name, None):
6317                             scaling_info["kdu-delete"][kdu_name] = []
6318
6319                         kdur = get_kdur(db_vnfr, kdu_name)
6320                         if kdur.get("helm-chart"):
6321                             k8s_cluster_type = "helm-chart-v3"
6322                             self.logger.debug("kdur: {}".format(kdur))
6323                             if (
6324                                 kdur.get("helm-version")
6325                                 and kdur.get("helm-version") == "v2"
6326                             ):
6327                                 k8s_cluster_type = "helm-chart"
6328                         elif kdur.get("juju-bundle"):
6329                             k8s_cluster_type = "juju-bundle"
6330                         else:
6331                             raise LcmException(
6332                                 "kdu type for kdu='{}.{}' is neither helm-chart nor "
6333                                 "juju-bundle. Maybe an old NBI version is running".format(
6334                                     db_vnfr["member-vnf-index-ref"], kdur["kdu-name"]
6335                                 )
6336                             )
6337
6338                         min_instance_count = 0
6339                         if kdu_profile and "min-number-of-instances" in kdu_profile:
6340                             min_instance_count = kdu_profile["min-number-of-instances"]
6341
6342                         nb_scale_op -= kdu_delta.get("number-of-instances", 1)
6343                         deployed_kdu, _ = get_deployed_kdu(
6344                             nsr_deployed, kdu_name, vnf_index
6345                         )
6346                         if deployed_kdu is None:
6347                             raise LcmException(
6348                                 "KDU '{}' for vnf '{}' not deployed".format(
6349                                     kdu_name, vnf_index
6350                                 )
6351                             )
6352                         kdu_instance = deployed_kdu.get("kdu-instance")
6353                         instance_num = await self.k8scluster_map[
6354                             k8s_cluster_type
6355                         ].get_scale_count(
6356                             resource_name,
6357                             kdu_instance,
6358                             vca_id=vca_id,
6359                             cluster_uuid=deployed_kdu.get("k8scluster-uuid"),
6360                             kdu_model=deployed_kdu.get("kdu-model"),
6361                         )
6362                         kdu_replica_count = instance_num - kdu_delta.get(
6363                             "number-of-instances", 1
6364                         )
6365
6366                         if kdu_replica_count < min_instance_count < instance_num:
6367                             kdu_replica_count = min_instance_count
6368                         if kdu_replica_count < min_instance_count:
6369                             raise LcmException(
6370                                 "reached the limit of {} (min-instance-count) scaling-in operations for the "
6371                                 "scaling-group-descriptor '{}'".format(
6372                                     instance_num, scaling_group
6373                                 )
6374                             )
6375
6376                         for x in range(kdu_delta.get("number-of-instances", 1)):
6377                             vca_scaling_info.append(
6378                                 {
6379                                     "osm_kdu_id": kdu_name,
6380                                     "member-vnf-index": vnf_index,
6381                                     "type": "delete",
6382                                     "kdu_index": instance_num - x - 1,
6383                                 }
6384                             )
6385                         scaling_info["kdu-delete"][kdu_name].append(
6386                             {
6387                                 "member-vnf-index": vnf_index,
6388                                 "type": "delete",
6389                                 "k8s-cluster-type": k8s_cluster_type,
6390                                 "resource-name": resource_name,
6391                                 "scale": kdu_replica_count,
6392                             }
6393                         )
6394
6395             # update VDU_SCALING_INFO with the VDUs to delete ip_addresses
6396             vdu_delete = copy(scaling_info.get("vdu-delete"))
6397             if scaling_info["scaling_direction"] == "IN":
6398                 for vdur in reversed(db_vnfr["vdur"]):
6399                     if vdu_delete.get(vdur["vdu-id-ref"]):
6400                         vdu_delete[vdur["vdu-id-ref"]] -= 1
6401                         scaling_info["vdu"].append(
6402                             {
6403                                 "name": vdur.get("name") or vdur.get("vdu-name"),
6404                                 "vdu_id": vdur["vdu-id-ref"],
6405                                 "interface": [],
6406                             }
6407                         )
6408                         for interface in vdur["interfaces"]:
6409                             scaling_info["vdu"][-1]["interface"].append(
6410                                 {
6411                                     "name": interface["name"],
6412                                     "ip_address": interface["ip-address"],
6413                                     "mac_address": interface.get("mac-address"),
6414                                 }
6415                             )
6416                 # vdu_delete = vdu_scaling_info.pop("vdu-delete")
6417
6418             # PRE-SCALE BEGIN
6419             step = "Executing pre-scale vnf-config-primitive"
6420             if scaling_descriptor.get("scaling-config-action"):
6421                 for scaling_config_action in scaling_descriptor[
6422                     "scaling-config-action"
6423                 ]:
6424                     if (
6425                         scaling_config_action.get("trigger") == "pre-scale-in"
6426                         and scaling_type == "SCALE_IN"
6427                     ) or (
6428                         scaling_config_action.get("trigger") == "pre-scale-out"
6429                         and scaling_type == "SCALE_OUT"
6430                     ):
6431                         vnf_config_primitive = scaling_config_action[
6432                             "vnf-config-primitive-name-ref"
6433                         ]
6434                         step = db_nslcmop_update[
6435                             "detailed-status"
6436                         ] = "executing pre-scale scaling-config-action '{}'".format(
6437                             vnf_config_primitive
6438                         )
6439
6440                         # look for primitive
6441                         for config_primitive in (
6442                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
6443                         ).get("config-primitive", ()):
6444                             if config_primitive["name"] == vnf_config_primitive:
6445                                 break
6446                         else:
6447                             raise LcmException(
6448                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-action"
6449                                 "[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:config-"
6450                                 "primitive".format(scaling_group, vnf_config_primitive)
6451                             )
6452
6453                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
6454                         if db_vnfr.get("additionalParamsForVnf"):
6455                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
6456
6457                         scale_process = "VCA"
6458                         db_nsr_update["config-status"] = "configuring pre-scaling"
6459                         primitive_params = self._map_primitive_params(
6460                             config_primitive, {}, vnfr_params
6461                         )
6462
6463                         # Pre-scale retry check: Check if this sub-operation has been executed before
6464                         op_index = self._check_or_add_scale_suboperation(
6465                             db_nslcmop,
6466                             vnf_index,
6467                             vnf_config_primitive,
6468                             primitive_params,
6469                             "PRE-SCALE",
6470                         )
6471                         if op_index == self.SUBOPERATION_STATUS_SKIP:
6472                             # Skip sub-operation
6473                             result = "COMPLETED"
6474                             result_detail = "Done"
6475                             self.logger.debug(
6476                                 logging_text
6477                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
6478                                     vnf_config_primitive, result, result_detail
6479                                 )
6480                             )
6481                         else:
6482                             if op_index == self.SUBOPERATION_STATUS_NEW:
6483                                 # New sub-operation: Get index of this sub-operation
6484                                 op_index = (
6485                                     len(db_nslcmop.get("_admin", {}).get("operations"))
6486                                     - 1
6487                                 )
6488                                 self.logger.debug(
6489                                     logging_text
6490                                     + "vnf_config_primitive={} New sub-operation".format(
6491                                         vnf_config_primitive
6492                                     )
6493                                 )
6494                             else:
6495                                 # retry:  Get registered params for this existing sub-operation
6496                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
6497                                     op_index
6498                                 ]
6499                                 vnf_index = op.get("member_vnf_index")
6500                                 vnf_config_primitive = op.get("primitive")
6501                                 primitive_params = op.get("primitive_params")
6502                                 self.logger.debug(
6503                                     logging_text
6504                                     + "vnf_config_primitive={} Sub-operation retry".format(
6505                                         vnf_config_primitive
6506                                     )
6507                                 )
6508                             # Execute the primitive, either with new (first-time) or registered (reintent) args
6509                             ee_descriptor_id = config_primitive.get(
6510                                 "execution-environment-ref"
6511                             )
6512                             primitive_name = config_primitive.get(
6513                                 "execution-environment-primitive", vnf_config_primitive
6514                             )
6515                             ee_id, vca_type = self._look_for_deployed_vca(
6516                                 nsr_deployed["VCA"],
6517                                 member_vnf_index=vnf_index,
6518                                 vdu_id=None,
6519                                 vdu_count_index=None,
6520                                 ee_descriptor_id=ee_descriptor_id,
6521                             )
6522                             result, result_detail = await self._ns_execute_primitive(
6523                                 ee_id,
6524                                 primitive_name,
6525                                 primitive_params,
6526                                 vca_type=vca_type,
6527                                 vca_id=vca_id,
6528                             )
6529                             self.logger.debug(
6530                                 logging_text
6531                                 + "vnf_config_primitive={} Done with result {} {}".format(
6532                                     vnf_config_primitive, result, result_detail
6533                                 )
6534                             )
6535                             # Update operationState = COMPLETED | FAILED
6536                             self._update_suboperation_status(
6537                                 db_nslcmop, op_index, result, result_detail
6538                             )
6539
6540                         if result == "FAILED":
6541                             raise LcmException(result_detail)
6542                         db_nsr_update["config-status"] = old_config_status
6543                         scale_process = None
6544             # PRE-SCALE END
6545
6546             db_nsr_update[
6547                 "_admin.scaling-group.{}.nb-scale-op".format(admin_scale_index)
6548             ] = nb_scale_op
6549             db_nsr_update[
6550                 "_admin.scaling-group.{}.time".format(admin_scale_index)
6551             ] = time()
6552
6553             # SCALE-IN VCA - BEGIN
6554             if vca_scaling_info:
6555                 step = db_nslcmop_update[
6556                     "detailed-status"
6557                 ] = "Deleting the execution environments"
6558                 scale_process = "VCA"
6559                 for vca_info in vca_scaling_info:
6560                     if vca_info["type"] == "delete" and not vca_info.get("osm_kdu_id"):
6561                         member_vnf_index = str(vca_info["member-vnf-index"])
6562                         self.logger.debug(
6563                             logging_text + "vdu info: {}".format(vca_info)
6564                         )
6565                         if vca_info.get("osm_vdu_id"):
6566                             vdu_id = vca_info["osm_vdu_id"]
6567                             vdu_index = int(vca_info["vdu_index"])
6568                             stage[
6569                                 1
6570                             ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
6571                                 member_vnf_index, vdu_id, vdu_index
6572                             )
6573                         stage[2] = step = "Scaling in VCA"
6574                         self._write_op_status(op_id=nslcmop_id, stage=stage)
6575                         vca_update = db_nsr["_admin"]["deployed"]["VCA"]
6576                         config_update = db_nsr["configurationStatus"]
6577                         for vca_index, vca in enumerate(vca_update):
6578                             if (
6579                                 (vca or vca.get("ee_id"))
6580                                 and vca["member-vnf-index"] == member_vnf_index
6581                                 and vca["vdu_count_index"] == vdu_index
6582                             ):
6583                                 if vca.get("vdu_id"):
6584                                     config_descriptor = get_configuration(
6585                                         db_vnfd, vca.get("vdu_id")
6586                                     )
6587                                 elif vca.get("kdu_name"):
6588                                     config_descriptor = get_configuration(
6589                                         db_vnfd, vca.get("kdu_name")
6590                                     )
6591                                 else:
6592                                     config_descriptor = get_configuration(
6593                                         db_vnfd, db_vnfd["id"]
6594                                     )
6595                                 operation_params = (
6596                                     db_nslcmop.get("operationParams") or {}
6597                                 )
6598                                 exec_terminate_primitives = not operation_params.get(
6599                                     "skip_terminate_primitives"
6600                                 ) and vca.get("needed_terminate")
6601                                 task = asyncio.ensure_future(
6602                                     asyncio.wait_for(
6603                                         self.destroy_N2VC(
6604                                             logging_text,
6605                                             db_nslcmop,
6606                                             vca,
6607                                             config_descriptor,
6608                                             vca_index,
6609                                             destroy_ee=True,
6610                                             exec_primitives=exec_terminate_primitives,
6611                                             scaling_in=True,
6612                                             vca_id=vca_id,
6613                                         ),
6614                                         timeout=self.timeout_charm_delete,
6615                                     )
6616                                 )
6617                                 tasks_dict_info[task] = "Terminating VCA {}".format(
6618                                     vca.get("ee_id")
6619                                 )
6620                                 del vca_update[vca_index]
6621                                 del config_update[vca_index]
6622                         # wait for pending tasks of terminate primitives
6623                         if tasks_dict_info:
6624                             self.logger.debug(
6625                                 logging_text
6626                                 + "Waiting for tasks {}".format(
6627                                     list(tasks_dict_info.keys())
6628                                 )
6629                             )
6630                             error_list = await self._wait_for_tasks(
6631                                 logging_text,
6632                                 tasks_dict_info,
6633                                 min(
6634                                     self.timeout_charm_delete, self.timeout_ns_terminate
6635                                 ),
6636                                 stage,
6637                                 nslcmop_id,
6638                             )
6639                             tasks_dict_info.clear()
6640                             if error_list:
6641                                 raise LcmException("; ".join(error_list))
6642
6643                         db_vca_and_config_update = {
6644                             "_admin.deployed.VCA": vca_update,
6645                             "configurationStatus": config_update,
6646                         }
6647                         self.update_db_2(
6648                             "nsrs", db_nsr["_id"], db_vca_and_config_update
6649                         )
6650             scale_process = None
6651             # SCALE-IN VCA - END
6652
6653             # SCALE RO - BEGIN
6654             if scaling_info.get("vdu-create") or scaling_info.get("vdu-delete"):
6655                 scale_process = "RO"
6656                 if self.ro_config.get("ng"):
6657                     await self._scale_ng_ro(
6658                         logging_text, db_nsr, db_nslcmop, db_vnfr, scaling_info, stage
6659                     )
6660             scaling_info.pop("vdu-create", None)
6661             scaling_info.pop("vdu-delete", None)
6662
6663             scale_process = None
6664             # SCALE RO - END
6665
6666             # SCALE KDU - BEGIN
6667             if scaling_info.get("kdu-create") or scaling_info.get("kdu-delete"):
6668                 scale_process = "KDU"
6669                 await self._scale_kdu(
6670                     logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
6671                 )
6672             scaling_info.pop("kdu-create", None)
6673             scaling_info.pop("kdu-delete", None)
6674
6675             scale_process = None
6676             # SCALE KDU - END
6677
6678             if db_nsr_update:
6679                 self.update_db_2("nsrs", nsr_id, db_nsr_update)
6680
6681             # SCALE-UP VCA - BEGIN
6682             if vca_scaling_info:
6683                 step = db_nslcmop_update[
6684                     "detailed-status"
6685                 ] = "Creating new execution environments"
6686                 scale_process = "VCA"
6687                 for vca_info in vca_scaling_info:
6688                     if vca_info["type"] == "create" and not vca_info.get("osm_kdu_id"):
6689                         member_vnf_index = str(vca_info["member-vnf-index"])
6690                         self.logger.debug(
6691                             logging_text + "vdu info: {}".format(vca_info)
6692                         )
6693                         vnfd_id = db_vnfr["vnfd-ref"]
6694                         if vca_info.get("osm_vdu_id"):
6695                             vdu_index = int(vca_info["vdu_index"])
6696                             deploy_params = {"OSM": get_osm_params(db_vnfr)}
6697                             if db_vnfr.get("additionalParamsForVnf"):
6698                                 deploy_params.update(
6699                                     parse_yaml_strings(
6700                                         db_vnfr["additionalParamsForVnf"].copy()
6701                                     )
6702                                 )
6703                             descriptor_config = get_configuration(
6704                                 db_vnfd, db_vnfd["id"]
6705                             )
6706                             if descriptor_config:
6707                                 vdu_id = None
6708                                 vdu_name = None
6709                                 kdu_name = None
6710                                 self._deploy_n2vc(
6711                                     logging_text=logging_text
6712                                     + "member_vnf_index={} ".format(member_vnf_index),
6713                                     db_nsr=db_nsr,
6714                                     db_vnfr=db_vnfr,
6715                                     nslcmop_id=nslcmop_id,
6716                                     nsr_id=nsr_id,
6717                                     nsi_id=nsi_id,
6718                                     vnfd_id=vnfd_id,
6719                                     vdu_id=vdu_id,
6720                                     kdu_name=kdu_name,
6721                                     member_vnf_index=member_vnf_index,
6722                                     vdu_index=vdu_index,
6723                                     vdu_name=vdu_name,
6724                                     deploy_params=deploy_params,
6725                                     descriptor_config=descriptor_config,
6726                                     base_folder=base_folder,
6727                                     task_instantiation_info=tasks_dict_info,
6728                                     stage=stage,
6729                                 )
6730                             vdu_id = vca_info["osm_vdu_id"]
6731                             vdur = find_in_list(
6732                                 db_vnfr["vdur"], lambda vdu: vdu["vdu-id-ref"] == vdu_id
6733                             )
6734                             descriptor_config = get_configuration(db_vnfd, vdu_id)
6735                             if vdur.get("additionalParams"):
6736                                 deploy_params_vdu = parse_yaml_strings(
6737                                     vdur["additionalParams"]
6738                                 )
6739                             else:
6740                                 deploy_params_vdu = deploy_params
6741                             deploy_params_vdu["OSM"] = get_osm_params(
6742                                 db_vnfr, vdu_id, vdu_count_index=vdu_index
6743                             )
6744                             if descriptor_config:
6745                                 vdu_name = None
6746                                 kdu_name = None
6747                                 stage[
6748                                     1
6749                                 ] = "Scaling member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
6750                                     member_vnf_index, vdu_id, vdu_index
6751                                 )
6752                                 stage[2] = step = "Scaling out VCA"
6753                                 self._write_op_status(op_id=nslcmop_id, stage=stage)
6754                                 self._deploy_n2vc(
6755                                     logging_text=logging_text
6756                                     + "member_vnf_index={}, vdu_id={}, vdu_index={} ".format(
6757                                         member_vnf_index, vdu_id, vdu_index
6758                                     ),
6759                                     db_nsr=db_nsr,
6760                                     db_vnfr=db_vnfr,
6761                                     nslcmop_id=nslcmop_id,
6762                                     nsr_id=nsr_id,
6763                                     nsi_id=nsi_id,
6764                                     vnfd_id=vnfd_id,
6765                                     vdu_id=vdu_id,
6766                                     kdu_name=kdu_name,
6767                                     member_vnf_index=member_vnf_index,
6768                                     vdu_index=vdu_index,
6769                                     vdu_name=vdu_name,
6770                                     deploy_params=deploy_params_vdu,
6771                                     descriptor_config=descriptor_config,
6772                                     base_folder=base_folder,
6773                                     task_instantiation_info=tasks_dict_info,
6774                                     stage=stage,
6775                                 )
6776             # SCALE-UP VCA - END
6777             scale_process = None
6778
6779             # POST-SCALE BEGIN
6780             # execute primitive service POST-SCALING
6781             step = "Executing post-scale vnf-config-primitive"
6782             if scaling_descriptor.get("scaling-config-action"):
6783                 for scaling_config_action in scaling_descriptor[
6784                     "scaling-config-action"
6785                 ]:
6786                     if (
6787                         scaling_config_action.get("trigger") == "post-scale-in"
6788                         and scaling_type == "SCALE_IN"
6789                     ) or (
6790                         scaling_config_action.get("trigger") == "post-scale-out"
6791                         and scaling_type == "SCALE_OUT"
6792                     ):
6793                         vnf_config_primitive = scaling_config_action[
6794                             "vnf-config-primitive-name-ref"
6795                         ]
6796                         step = db_nslcmop_update[
6797                             "detailed-status"
6798                         ] = "executing post-scale scaling-config-action '{}'".format(
6799                             vnf_config_primitive
6800                         )
6801
6802                         vnfr_params = {"VDU_SCALE_INFO": scaling_info}
6803                         if db_vnfr.get("additionalParamsForVnf"):
6804                             vnfr_params.update(db_vnfr["additionalParamsForVnf"])
6805
6806                         # look for primitive
6807                         for config_primitive in (
6808                             get_configuration(db_vnfd, db_vnfd["id"]) or {}
6809                         ).get("config-primitive", ()):
6810                             if config_primitive["name"] == vnf_config_primitive:
6811                                 break
6812                         else:
6813                             raise LcmException(
6814                                 "Invalid vnfd descriptor at scaling-group-descriptor[name='{}']:scaling-config-"
6815                                 "action[vnf-config-primitive-name-ref='{}'] does not match any vnf-configuration:"
6816                                 "config-primitive".format(
6817                                     scaling_group, vnf_config_primitive
6818                                 )
6819                             )
6820                         scale_process = "VCA"
6821                         db_nsr_update["config-status"] = "configuring post-scaling"
6822                         primitive_params = self._map_primitive_params(
6823                             config_primitive, {}, vnfr_params
6824                         )
6825
6826                         # Post-scale retry check: Check if this sub-operation has been executed before
6827                         op_index = self._check_or_add_scale_suboperation(
6828                             db_nslcmop,
6829                             vnf_index,
6830                             vnf_config_primitive,
6831                             primitive_params,
6832                             "POST-SCALE",
6833                         )
6834                         if op_index == self.SUBOPERATION_STATUS_SKIP:
6835                             # Skip sub-operation
6836                             result = "COMPLETED"
6837                             result_detail = "Done"
6838                             self.logger.debug(
6839                                 logging_text
6840                                 + "vnf_config_primitive={} Skipped sub-operation, result {} {}".format(
6841                                     vnf_config_primitive, result, result_detail
6842                                 )
6843                             )
6844                         else:
6845                             if op_index == self.SUBOPERATION_STATUS_NEW:
6846                                 # New sub-operation: Get index of this sub-operation
6847                                 op_index = (
6848                                     len(db_nslcmop.get("_admin", {}).get("operations"))
6849                                     - 1
6850                                 )
6851                                 self.logger.debug(
6852                                     logging_text
6853                                     + "vnf_config_primitive={} New sub-operation".format(
6854                                         vnf_config_primitive
6855                                     )
6856                                 )
6857                             else:
6858                                 # retry:  Get registered params for this existing sub-operation
6859                                 op = db_nslcmop.get("_admin", {}).get("operations", [])[
6860                                     op_index
6861                                 ]
6862                                 vnf_index = op.get("member_vnf_index")
6863                                 vnf_config_primitive = op.get("primitive")
6864                                 primitive_params = op.get("primitive_params")
6865                                 self.logger.debug(
6866                                     logging_text
6867                                     + "vnf_config_primitive={} Sub-operation retry".format(
6868                                         vnf_config_primitive
6869                                     )
6870                                 )
6871                             # Execute the primitive, either with new (first-time) or registered (reintent) args
6872                             ee_descriptor_id = config_primitive.get(
6873                                 "execution-environment-ref"
6874                             )
6875                             primitive_name = config_primitive.get(
6876                                 "execution-environment-primitive", vnf_config_primitive
6877                             )
6878                             ee_id, vca_type = self._look_for_deployed_vca(
6879                                 nsr_deployed["VCA"],
6880                                 member_vnf_index=vnf_index,
6881                                 vdu_id=None,
6882                                 vdu_count_index=None,
6883                                 ee_descriptor_id=ee_descriptor_id,
6884                             )
6885                             result, result_detail = await self._ns_execute_primitive(
6886                                 ee_id,
6887                                 primitive_name,
6888                                 primitive_params,
6889                                 vca_type=vca_type,
6890                                 vca_id=vca_id,
6891                             )
6892                             self.logger.debug(
6893                                 logging_text
6894                                 + "vnf_config_primitive={} Done with result {} {}".format(
6895                                     vnf_config_primitive, result, result_detail
6896                                 )
6897                             )
6898                             # Update operationState = COMPLETED | FAILED
6899                             self._update_suboperation_status(
6900                                 db_nslcmop, op_index, result, result_detail
6901                             )
6902
6903                         if result == "FAILED":
6904                             raise LcmException(result_detail)
6905                         db_nsr_update["config-status"] = old_config_status
6906                         scale_process = None
6907             # POST-SCALE END
6908
6909             db_nsr_update[
6910                 "detailed-status"
6911             ] = ""  # "scaled {} {}".format(scaling_group, scaling_type)
6912             db_nsr_update["operational-status"] = (
6913                 "running"
6914                 if old_operational_status == "failed"
6915                 else old_operational_status
6916             )
6917             db_nsr_update["config-status"] = old_config_status
6918             return
6919         except (
6920             ROclient.ROClientException,
6921             DbException,
6922             LcmException,
6923             NgRoException,
6924         ) as e:
6925             self.logger.error(logging_text + "Exit Exception {}".format(e))
6926             exc = e
6927         except asyncio.CancelledError:
6928             self.logger.error(
6929                 logging_text + "Cancelled Exception while '{}'".format(step)
6930             )
6931             exc = "Operation was cancelled"
6932         except Exception as e:
6933             exc = traceback.format_exc()
6934             self.logger.critical(
6935                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
6936                 exc_info=True,
6937             )
6938         finally:
6939             self._write_ns_status(
6940                 nsr_id=nsr_id,
6941                 ns_state=None,
6942                 current_operation="IDLE",
6943                 current_operation_id=None,
6944             )
6945             if tasks_dict_info:
6946                 stage[1] = "Waiting for instantiate pending tasks."
6947                 self.logger.debug(logging_text + stage[1])
6948                 exc = await self._wait_for_tasks(
6949                     logging_text,
6950                     tasks_dict_info,
6951                     self.timeout_ns_deploy,
6952                     stage,
6953                     nslcmop_id,
6954                     nsr_id=nsr_id,
6955                 )
6956             if exc:
6957                 db_nslcmop_update[
6958                     "detailed-status"
6959                 ] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
6960                 nslcmop_operation_state = "FAILED"
6961                 if db_nsr:
6962                     db_nsr_update["operational-status"] = old_operational_status
6963                     db_nsr_update["config-status"] = old_config_status
6964                     db_nsr_update["detailed-status"] = ""
6965                     if scale_process:
6966                         if "VCA" in scale_process:
6967                             db_nsr_update["config-status"] = "failed"
6968                         if "RO" in scale_process:
6969                             db_nsr_update["operational-status"] = "failed"
6970                         db_nsr_update[
6971                             "detailed-status"
6972                         ] = "FAILED scaling nslcmop={} {}: {}".format(
6973                             nslcmop_id, step, exc
6974                         )
6975             else:
6976                 error_description_nslcmop = None
6977                 nslcmop_operation_state = "COMPLETED"
6978                 db_nslcmop_update["detailed-status"] = "Done"
6979
6980             self._write_op_status(
6981                 op_id=nslcmop_id,
6982                 stage="",
6983                 error_message=error_description_nslcmop,
6984                 operation_state=nslcmop_operation_state,
6985                 other_update=db_nslcmop_update,
6986             )
6987             if db_nsr:
6988                 self._write_ns_status(
6989                     nsr_id=nsr_id,
6990                     ns_state=None,
6991                     current_operation="IDLE",
6992                     current_operation_id=None,
6993                     other_update=db_nsr_update,
6994                 )
6995
6996             if nslcmop_operation_state:
6997                 try:
6998                     msg = {
6999                         "nsr_id": nsr_id,
7000                         "nslcmop_id": nslcmop_id,
7001                         "operationState": nslcmop_operation_state,
7002                     }
7003                     await self.msg.aiowrite("ns", "scaled", msg, loop=self.loop)
7004                 except Exception as e:
7005                     self.logger.error(
7006                         logging_text + "kafka_write notification Exception {}".format(e)
7007                     )
7008             self.logger.debug(logging_text + "Exit")
7009             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_scale")
7010
7011     async def _scale_kdu(
7012         self, logging_text, nsr_id, nsr_deployed, db_vnfd, vca_id, scaling_info
7013     ):
7014         _scaling_info = scaling_info.get("kdu-create") or scaling_info.get("kdu-delete")
7015         for kdu_name in _scaling_info:
7016             for kdu_scaling_info in _scaling_info[kdu_name]:
7017                 deployed_kdu, index = get_deployed_kdu(
7018                     nsr_deployed, kdu_name, kdu_scaling_info["member-vnf-index"]
7019                 )
7020                 cluster_uuid = deployed_kdu["k8scluster-uuid"]
7021                 kdu_instance = deployed_kdu["kdu-instance"]
7022                 kdu_model = deployed_kdu.get("kdu-model")
7023                 scale = int(kdu_scaling_info["scale"])
7024                 k8s_cluster_type = kdu_scaling_info["k8s-cluster-type"]
7025
7026                 db_dict = {
7027                     "collection": "nsrs",
7028                     "filter": {"_id": nsr_id},
7029                     "path": "_admin.deployed.K8s.{}".format(index),
7030                 }
7031
7032                 step = "scaling application {}".format(
7033                     kdu_scaling_info["resource-name"]
7034                 )
7035                 self.logger.debug(logging_text + step)
7036
7037                 if kdu_scaling_info["type"] == "delete":
7038                     kdu_config = get_configuration(db_vnfd, kdu_name)
7039                     if (
7040                         kdu_config
7041                         and kdu_config.get("terminate-config-primitive")
7042                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7043                     ):
7044                         terminate_config_primitive_list = kdu_config.get(
7045                             "terminate-config-primitive"
7046                         )
7047                         terminate_config_primitive_list.sort(
7048                             key=lambda val: int(val["seq"])
7049                         )
7050
7051                         for (
7052                             terminate_config_primitive
7053                         ) in terminate_config_primitive_list:
7054                             primitive_params_ = self._map_primitive_params(
7055                                 terminate_config_primitive, {}, {}
7056                             )
7057                             step = "execute terminate config primitive"
7058                             self.logger.debug(logging_text + step)
7059                             await asyncio.wait_for(
7060                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7061                                     cluster_uuid=cluster_uuid,
7062                                     kdu_instance=kdu_instance,
7063                                     primitive_name=terminate_config_primitive["name"],
7064                                     params=primitive_params_,
7065                                     db_dict=db_dict,
7066                                     vca_id=vca_id,
7067                                 ),
7068                                 timeout=600,
7069                             )
7070
7071                 await asyncio.wait_for(
7072                     self.k8scluster_map[k8s_cluster_type].scale(
7073                         kdu_instance,
7074                         scale,
7075                         kdu_scaling_info["resource-name"],
7076                         vca_id=vca_id,
7077                         cluster_uuid=cluster_uuid,
7078                         kdu_model=kdu_model,
7079                         atomic=True,
7080                         db_dict=db_dict,
7081                     ),
7082                     timeout=self.timeout_vca_on_error,
7083                 )
7084
7085                 if kdu_scaling_info["type"] == "create":
7086                     kdu_config = get_configuration(db_vnfd, kdu_name)
7087                     if (
7088                         kdu_config
7089                         and kdu_config.get("initial-config-primitive")
7090                         and get_juju_ee_ref(db_vnfd, kdu_name) is None
7091                     ):
7092                         initial_config_primitive_list = kdu_config.get(
7093                             "initial-config-primitive"
7094                         )
7095                         initial_config_primitive_list.sort(
7096                             key=lambda val: int(val["seq"])
7097                         )
7098
7099                         for initial_config_primitive in initial_config_primitive_list:
7100                             primitive_params_ = self._map_primitive_params(
7101                                 initial_config_primitive, {}, {}
7102                             )
7103                             step = "execute initial config primitive"
7104                             self.logger.debug(logging_text + step)
7105                             await asyncio.wait_for(
7106                                 self.k8scluster_map[k8s_cluster_type].exec_primitive(
7107                                     cluster_uuid=cluster_uuid,
7108                                     kdu_instance=kdu_instance,
7109                                     primitive_name=initial_config_primitive["name"],
7110                                     params=primitive_params_,
7111                                     db_dict=db_dict,
7112                                     vca_id=vca_id,
7113                                 ),
7114                                 timeout=600,
7115                             )
7116
7117     async def _scale_ng_ro(
7118         self, logging_text, db_nsr, db_nslcmop, db_vnfr, vdu_scaling_info, stage
7119     ):
7120         nsr_id = db_nslcmop["nsInstanceId"]
7121         db_nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7122         db_vnfrs = {}
7123
7124         # read from db: vnfd's for every vnf
7125         db_vnfds = []
7126
7127         # for each vnf in ns, read vnfd
7128         for vnfr in self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id}):
7129             db_vnfrs[vnfr["member-vnf-index-ref"]] = vnfr
7130             vnfd_id = vnfr["vnfd-id"]  # vnfd uuid for this vnf
7131             # if we haven't this vnfd, read it from db
7132             if not find_in_list(db_vnfds, lambda a_vnfd: a_vnfd["id"] == vnfd_id):
7133                 # read from db
7134                 vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7135                 db_vnfds.append(vnfd)
7136         n2vc_key = self.n2vc.get_public_key()
7137         n2vc_key_list = [n2vc_key]
7138         self.scale_vnfr(
7139             db_vnfr,
7140             vdu_scaling_info.get("vdu-create"),
7141             vdu_scaling_info.get("vdu-delete"),
7142             mark_delete=True,
7143         )
7144         # db_vnfr has been updated, update db_vnfrs to use it
7145         db_vnfrs[db_vnfr["member-vnf-index-ref"]] = db_vnfr
7146         await self._instantiate_ng_ro(
7147             logging_text,
7148             nsr_id,
7149             db_nsd,
7150             db_nsr,
7151             db_nslcmop,
7152             db_vnfrs,
7153             db_vnfds,
7154             n2vc_key_list,
7155             stage=stage,
7156             start_deploy=time(),
7157             timeout_ns_deploy=self.timeout_ns_deploy,
7158         )
7159         if vdu_scaling_info.get("vdu-delete"):
7160             self.scale_vnfr(
7161                 db_vnfr, None, vdu_scaling_info["vdu-delete"], mark_delete=False
7162             )
7163
7164     async def extract_prometheus_scrape_jobs(
7165         self, ee_id, artifact_path, ee_config_descriptor, vnfr_id, nsr_id, target_ip
7166     ):
7167         # look if exist a file called 'prometheus*.j2' and
7168         artifact_content = self.fs.dir_ls(artifact_path)
7169         job_file = next(
7170             (
7171                 f
7172                 for f in artifact_content
7173                 if f.startswith("prometheus") and f.endswith(".j2")
7174             ),
7175             None,
7176         )
7177         if not job_file:
7178             return
7179         with self.fs.file_open((artifact_path, job_file), "r") as f:
7180             job_data = f.read()
7181
7182         # TODO get_service
7183         _, _, service = ee_id.partition(".")  # remove prefix   "namespace."
7184         host_name = "{}-{}".format(service, ee_config_descriptor["metric-service"])
7185         host_port = "80"
7186         vnfr_id = vnfr_id.replace("-", "")
7187         variables = {
7188             "JOB_NAME": vnfr_id,
7189             "TARGET_IP": target_ip,
7190             "EXPORTER_POD_IP": host_name,
7191             "EXPORTER_POD_PORT": host_port,
7192         }
7193         job_list = parse_job(job_data, variables)
7194         # ensure job_name is using the vnfr_id. Adding the metadata nsr_id
7195         for job in job_list:
7196             if (
7197                 not isinstance(job.get("job_name"), str)
7198                 or vnfr_id not in job["job_name"]
7199             ):
7200                 job["job_name"] = vnfr_id + "_" + str(randint(1, 10000))
7201             job["nsr_id"] = nsr_id
7202             job["vnfr_id"] = vnfr_id
7203         return job_list
7204
7205     def get_vca_cloud_and_credentials(self, vim_account_id: str) -> (str, str):
7206         """
7207         Get VCA Cloud and VCA Cloud Credentials for the VIM account
7208
7209         :param: vim_account_id:     VIM Account ID
7210
7211         :return: (cloud_name, cloud_credential)
7212         """
7213         config = VimAccountDB.get_vim_account_with_id(vim_account_id).get("config", {})
7214         return config.get("vca_cloud"), config.get("vca_cloud_credential")
7215
7216     def get_vca_k8s_cloud_and_credentials(self, vim_account_id: str) -> (str, str):
7217         """
7218         Get VCA K8s Cloud and VCA K8s Cloud Credentials for the VIM account
7219
7220         :param: vim_account_id:     VIM Account ID
7221
7222         :return: (cloud_name, cloud_credential)
7223         """
7224         config = VimAccountDB.get_vim_account_with_id(vim_account_id).get("config", {})
7225         return config.get("vca_k8s_cloud"), config.get("vca_k8s_cloud_credential")
7226
7227     async def migrate(self, nsr_id, nslcmop_id):
7228         """
7229         Migrate VNFs and VDUs instances in a NS
7230
7231         :param: nsr_id: NS Instance ID
7232         :param: nslcmop_id: nslcmop ID of migrate
7233
7234         """
7235         # Try to lock HA task here
7236         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7237         if not task_is_locked_by_me:
7238             return
7239         logging_text = "Task ns={} migrate ".format(nsr_id)
7240         self.logger.debug(logging_text + "Enter")
7241         # get all needed from database
7242         db_nslcmop = None
7243         db_nslcmop_update = {}
7244         nslcmop_operation_state = None
7245         db_nsr_update = {}
7246         target = {}
7247         exc = None
7248         # in case of error, indicates what part of scale was failed to put nsr at error status
7249         start_deploy = time()
7250
7251         try:
7252             # wait for any previous tasks in process
7253             step = "Waiting for previous operations to terminate"
7254             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7255
7256             self._write_ns_status(
7257                 nsr_id=nsr_id,
7258                 ns_state=None,
7259                 current_operation="MIGRATING",
7260                 current_operation_id=nslcmop_id,
7261             )
7262             step = "Getting nslcmop from database"
7263             self.logger.debug(
7264                 step + " after having waited for previous tasks to be completed"
7265             )
7266             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7267             migrate_params = db_nslcmop.get("operationParams")
7268
7269             target = {}
7270             target.update(migrate_params)
7271             desc = await self.RO.migrate(nsr_id, target)
7272             self.logger.debug("RO return > {}".format(desc))
7273             action_id = desc["action_id"]
7274             await self._wait_ng_ro(
7275                 nsr_id, action_id, nslcmop_id, start_deploy, self.timeout_migrate,
7276                 operation="migrate"
7277             )
7278         except (ROclient.ROClientException, DbException, LcmException) as e:
7279             self.logger.error("Exit Exception {}".format(e))
7280             exc = e
7281         except asyncio.CancelledError:
7282             self.logger.error("Cancelled Exception while '{}'".format(step))
7283             exc = "Operation was cancelled"
7284         except Exception as e:
7285             exc = traceback.format_exc()
7286             self.logger.critical(
7287                 "Exit Exception {} {}".format(type(e).__name__, e), exc_info=True
7288             )
7289         finally:
7290             self._write_ns_status(
7291                 nsr_id=nsr_id,
7292                 ns_state=None,
7293                 current_operation="IDLE",
7294                 current_operation_id=None,
7295             )
7296             if exc:
7297                 db_nslcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
7298                 nslcmop_operation_state = "FAILED"
7299             else:
7300                 nslcmop_operation_state = "COMPLETED"
7301                 db_nslcmop_update["detailed-status"] = "Done"
7302                 db_nsr_update["detailed-status"] = "Done"
7303
7304             self._write_op_status(
7305                 op_id=nslcmop_id,
7306                 stage="",
7307                 error_message="",
7308                 operation_state=nslcmop_operation_state,
7309                 other_update=db_nslcmop_update,
7310             )
7311             if nslcmop_operation_state:
7312                 try:
7313                     msg = {
7314                         "nsr_id": nsr_id,
7315                         "nslcmop_id": nslcmop_id,
7316                         "operationState": nslcmop_operation_state,
7317                     }
7318                     await self.msg.aiowrite("ns", "migrated", msg, loop=self.loop)
7319                 except Exception as e:
7320                     self.logger.error(
7321                         logging_text + "kafka_write notification Exception {}".format(e)
7322                     )
7323             self.logger.debug(logging_text + "Exit")
7324             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_migrate")
7325
7326
7327     async def heal(self, nsr_id, nslcmop_id):
7328         """
7329         Heal NS
7330
7331         :param nsr_id: ns instance to heal
7332         :param nslcmop_id: operation to run
7333         :return:
7334         """
7335
7336         # Try to lock HA task here
7337         task_is_locked_by_me = self.lcm_tasks.lock_HA("ns", "nslcmops", nslcmop_id)
7338         if not task_is_locked_by_me:
7339             return
7340
7341         logging_text = "Task ns={} heal={} ".format(nsr_id, nslcmop_id)
7342         stage = ["", "", ""]
7343         tasks_dict_info = {}
7344         # ^ stage, step, VIM progress
7345         self.logger.debug(logging_text + "Enter")
7346         # get all needed from database
7347         db_nsr = None
7348         db_nslcmop_update = {}
7349         db_nsr_update = {}
7350         db_vnfrs = {}  # vnf's info indexed by _id
7351         exc = None
7352         old_operational_status = ""
7353         old_config_status = ""
7354         nsi_id = None
7355         try:
7356             # wait for any previous tasks in process
7357             step = "Waiting for previous operations to terminate"
7358             await self.lcm_tasks.waitfor_related_HA("ns", "nslcmops", nslcmop_id)
7359             self._write_ns_status(
7360                 nsr_id=nsr_id,
7361                 ns_state=None,
7362                 current_operation="HEALING",
7363                 current_operation_id=nslcmop_id,
7364             )
7365
7366             step = "Getting nslcmop from database"
7367             self.logger.debug(
7368                 step + " after having waited for previous tasks to be completed"
7369             )
7370             db_nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
7371
7372             step = "Getting nsr from database"
7373             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
7374             old_operational_status = db_nsr["operational-status"]
7375             old_config_status = db_nsr["config-status"]
7376
7377             db_nsr_update = {
7378                 "_admin.deployed.RO.operational-status": "healing",
7379             }
7380             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7381
7382             step = "Sending heal order to VIM"
7383             task_ro = asyncio.ensure_future(
7384                 self.heal_RO(
7385                     logging_text=logging_text,
7386                     nsr_id=nsr_id,
7387                     db_nslcmop=db_nslcmop,
7388                     stage=stage,
7389                 )
7390             )
7391             self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "heal_RO", task_ro)
7392             tasks_dict_info[task_ro] = "Healing at VIM"
7393
7394             # VCA tasks
7395             # read from db: nsd
7396             stage[1] = "Getting nsd={} from db.".format(db_nsr["nsd-id"])
7397             self.logger.debug(logging_text + stage[1])
7398             nsd = self.db.get_one("nsds", {"_id": db_nsr["nsd-id"]})
7399             self.fs.sync(db_nsr["nsd-id"])
7400             db_nsr["nsd"] = nsd
7401             # read from db: vnfr's of this ns
7402             step = "Getting vnfrs from db"
7403             db_vnfrs_list = self.db.get_list("vnfrs", {"nsr-id-ref": nsr_id})
7404             for vnfr in db_vnfrs_list:
7405                 db_vnfrs[vnfr["_id"]] = vnfr
7406             self.logger.debug("ns.heal db_vnfrs={}".format(db_vnfrs))
7407
7408             # Check for each target VNF
7409             target_list = db_nslcmop.get("operationParams", {}).get("healVnfData", {})
7410             for target_vnf in target_list:
7411                 # Find this VNF in the list from DB
7412                 vnfr_id = target_vnf.get("vnfInstanceId", None)
7413                 if vnfr_id:
7414                     db_vnfr = db_vnfrs[vnfr_id]
7415                     vnfd_id = db_vnfr.get("vnfd-id")
7416                     vnfd_ref = db_vnfr.get("vnfd-ref")
7417                     vnfd = self.db.get_one("vnfds", {"_id": vnfd_id})
7418                     base_folder = vnfd["_admin"]["storage"]
7419                     vdu_id = None
7420                     vdu_index = 0
7421                     vdu_name = None
7422                     kdu_name = None
7423                     nsi_id = None  # TODO put nsi_id when this nsr belongs to a NSI
7424                     member_vnf_index = db_vnfr.get("member-vnf-index-ref")
7425
7426                     # Check each target VDU and deploy N2VC
7427                     for target_vdu in target_vnf["additionalParams"].get("vdu", None):
7428                         deploy_params_vdu = target_vdu
7429                         # Set run-day1 vnf level value if not vdu level value exists
7430                         if not deploy_params_vdu.get("run-day1") and target_vnf["additionalParams"].get("run-day1"):
7431                             deploy_params_vdu["run-day1"] = target_vnf["additionalParams"].get("run-day1")
7432                         vdu_name = target_vdu.get("vdu-id", None)
7433                         # TODO: Get vdu_id from vdud.
7434                         vdu_id = vdu_name
7435                         # For multi instance VDU count-index is mandatory
7436                         # For single session VDU count-indes is 0
7437                         vdu_index = target_vdu.get("count-index",0)
7438
7439                         # n2vc_redesign STEP 3 to 6 Deploy N2VC
7440                         stage[1] = "Deploying Execution Environments."
7441                         self.logger.debug(logging_text + stage[1])
7442
7443                         # VNF Level charm. Normal case when proxy charms.
7444                         # If target instance is management machine continue with actions: recreate EE for native charms or reinject juju key for proxy charms.
7445                         descriptor_config = get_configuration(vnfd, vnfd_ref)
7446                         if descriptor_config:
7447                             # Continue if healed machine is management machine
7448                             vnf_ip_address = db_vnfr.get("ip-address")
7449                             target_instance = None
7450                             for instance in db_vnfr.get("vdur", None):
7451                                 if ( instance["vdu-name"] == vdu_name and instance["count-index"] == vdu_index ):
7452                                     target_instance = instance
7453                                     break
7454                             if vnf_ip_address == target_instance.get("ip-address"):
7455                                 self._heal_n2vc(
7456                                 logging_text=logging_text
7457                                 + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
7458                                     member_vnf_index, vdu_name, vdu_index
7459                                 ),
7460                                 db_nsr=db_nsr,
7461                                 db_vnfr=db_vnfr,
7462                                 nslcmop_id=nslcmop_id,
7463                                 nsr_id=nsr_id,
7464                                 nsi_id=nsi_id,
7465                                 vnfd_id=vnfd_ref,
7466                                 vdu_id=None,
7467                                 kdu_name=None,
7468                                 member_vnf_index=member_vnf_index,
7469                                 vdu_index=0,
7470                                 vdu_name=None,
7471                                 deploy_params=deploy_params_vdu,
7472                                 descriptor_config=descriptor_config,
7473                                 base_folder=base_folder,
7474                                 task_instantiation_info=tasks_dict_info,
7475                                 stage=stage,
7476                             )
7477
7478                         # VDU Level charm. Normal case with native charms.
7479                         descriptor_config = get_configuration(vnfd, vdu_name)
7480                         if descriptor_config:
7481                             self._heal_n2vc(
7482                                 logging_text=logging_text
7483                                 + "member_vnf_index={}, vdu_name={}, vdu_index={} ".format(
7484                                     member_vnf_index, vdu_name, vdu_index
7485                                 ),
7486                                 db_nsr=db_nsr,
7487                                 db_vnfr=db_vnfr,
7488                                 nslcmop_id=nslcmop_id,
7489                                 nsr_id=nsr_id,
7490                                 nsi_id=nsi_id,
7491                                 vnfd_id=vnfd_ref,
7492                                 vdu_id=vdu_id,
7493                                 kdu_name=kdu_name,
7494                                 member_vnf_index=member_vnf_index,
7495                                 vdu_index=vdu_index,
7496                                 vdu_name=vdu_name,
7497                                 deploy_params=deploy_params_vdu,
7498                                 descriptor_config=descriptor_config,
7499                                 base_folder=base_folder,
7500                                 task_instantiation_info=tasks_dict_info,
7501                                 stage=stage,
7502                             )
7503
7504         except (
7505             ROclient.ROClientException,
7506             DbException,
7507             LcmException,
7508             NgRoException,
7509         ) as e:
7510             self.logger.error(logging_text + "Exit Exception {}".format(e))
7511             exc = e
7512         except asyncio.CancelledError:
7513             self.logger.error(
7514                 logging_text + "Cancelled Exception while '{}'".format(step)
7515             )
7516             exc = "Operation was cancelled"
7517         except Exception as e:
7518             exc = traceback.format_exc()
7519             self.logger.critical(
7520                 logging_text + "Exit Exception {} {}".format(type(e).__name__, e),
7521                 exc_info=True,
7522             )
7523         finally:
7524             if tasks_dict_info:
7525                 stage[1] = "Waiting for healing pending tasks."
7526                 self.logger.debug(logging_text + stage[1])
7527                 exc = await self._wait_for_tasks(
7528                     logging_text,
7529                     tasks_dict_info,
7530                     self.timeout_ns_deploy,
7531                     stage,
7532                     nslcmop_id,
7533                     nsr_id=nsr_id,
7534                 )
7535             if exc:
7536                 db_nslcmop_update[
7537                     "detailed-status"
7538                 ] = error_description_nslcmop = "FAILED {}: {}".format(step, exc)
7539                 nslcmop_operation_state = "FAILED"
7540                 if db_nsr:
7541                     db_nsr_update["operational-status"] = old_operational_status
7542                     db_nsr_update["config-status"] = old_config_status
7543                     db_nsr_update[
7544                         "detailed-status"
7545                     ] = "FAILED healing nslcmop={} {}: {}".format(
7546                         nslcmop_id, step, exc
7547                     )
7548                     for task, task_name in tasks_dict_info.items():
7549                         if not task.done() or task.cancelled() or task.exception():
7550                             if task_name.startswith(self.task_name_deploy_vca):
7551                                 # A N2VC task is pending
7552                                 db_nsr_update["config-status"] = "failed"
7553                             else:
7554                                 # RO task is pending
7555                                 db_nsr_update["operational-status"] = "failed"
7556             else:
7557                 error_description_nslcmop = None
7558                 nslcmop_operation_state = "COMPLETED"
7559                 db_nslcmop_update["detailed-status"] = "Done"
7560                 db_nsr_update["detailed-status"] = "Done"
7561                 db_nsr_update["operational-status"] = "running"
7562                 db_nsr_update["config-status"] = "configured"
7563
7564             self._write_op_status(
7565                 op_id=nslcmop_id,
7566                 stage="",
7567                 error_message=error_description_nslcmop,
7568                 operation_state=nslcmop_operation_state,
7569                 other_update=db_nslcmop_update,
7570             )
7571             if db_nsr:
7572                 self._write_ns_status(
7573                     nsr_id=nsr_id,
7574                     ns_state=None,
7575                     current_operation="IDLE",
7576                     current_operation_id=None,
7577                     other_update=db_nsr_update,
7578                 )
7579
7580             if nslcmop_operation_state:
7581                 try:
7582                     msg = {
7583                         "nsr_id": nsr_id,
7584                         "nslcmop_id": nslcmop_id,
7585                         "operationState": nslcmop_operation_state,
7586                     }
7587                     await self.msg.aiowrite("ns", "healed", msg, loop=self.loop)
7588                 except Exception as e:
7589                     self.logger.error(
7590                         logging_text + "kafka_write notification Exception {}".format(e)
7591                     )
7592             self.logger.debug(logging_text + "Exit")
7593             self.lcm_tasks.remove("ns", nsr_id, nslcmop_id, "ns_heal")
7594
7595     async def heal_RO(
7596         self,
7597         logging_text,
7598         nsr_id,
7599         db_nslcmop,
7600         stage,
7601     ):
7602         """
7603         Heal at RO
7604         :param logging_text: preffix text to use at logging
7605         :param nsr_id: nsr identity
7606         :param db_nslcmop: database content of ns operation, in this case, 'instantiate'
7607         :param stage: list with 3 items: [general stage, tasks, vim_specific]. This task will write over vim_specific
7608         :return: None or exception
7609         """
7610         def get_vim_account(vim_account_id):
7611             nonlocal db_vims
7612             if vim_account_id in db_vims:
7613                 return db_vims[vim_account_id]
7614             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account_id})
7615             db_vims[vim_account_id] = db_vim
7616             return db_vim
7617
7618         try:
7619             start_heal = time()
7620             ns_params = db_nslcmop.get("operationParams")
7621             if ns_params and ns_params.get("timeout_ns_heal"):
7622                 timeout_ns_heal = ns_params["timeout_ns_heal"]
7623             else:
7624                 timeout_ns_heal = self.timeout.get(
7625                     "ns_heal", self.timeout_ns_heal
7626                 )
7627
7628             db_vims = {}
7629
7630             nslcmop_id = db_nslcmop["_id"]
7631             target = {
7632                 "action_id": nslcmop_id,
7633             }
7634             self.logger.warning("db_nslcmop={} and timeout_ns_heal={}".format(db_nslcmop,timeout_ns_heal))
7635             target.update(db_nslcmop.get("operationParams", {}))
7636
7637             self.logger.debug("Send to RO > nsr_id={} target={}".format(nsr_id, target))
7638             desc = await self.RO.recreate(nsr_id, target)
7639             self.logger.debug("RO return > {}".format(desc))
7640             action_id = desc["action_id"]
7641             # waits for RO to complete because Reinjecting juju key at ro can find VM in state Deleted
7642             await self._wait_ng_ro(
7643                 nsr_id, action_id, nslcmop_id, start_heal, timeout_ns_heal, stage,
7644                 operation="healing"
7645             )
7646
7647             # Updating NSR
7648             db_nsr_update = {
7649                 "_admin.deployed.RO.operational-status": "running",
7650                 "detailed-status": " ".join(stage),
7651             }
7652             self.update_db_2("nsrs", nsr_id, db_nsr_update)
7653             self._write_op_status(nslcmop_id, stage)
7654             self.logger.debug(
7655                 logging_text + "ns healed at RO. RO_id={}".format(action_id)
7656             )
7657
7658         except Exception as e:
7659             stage[2] = "ERROR healing at VIM"
7660             #self.set_vnfr_at_error(db_vnfrs, str(e))
7661             self.logger.error(
7662                 "Error healing at VIM {}".format(e),
7663                 exc_info=not isinstance(
7664                     e,
7665                     (
7666                         ROclient.ROClientException,
7667                         LcmException,
7668                         DbException,
7669                         NgRoException,
7670                     ),
7671                 ),
7672             )
7673             raise
7674
7675     def _heal_n2vc(
7676         self,
7677         logging_text,
7678         db_nsr,
7679         db_vnfr,
7680         nslcmop_id,
7681         nsr_id,
7682         nsi_id,
7683         vnfd_id,
7684         vdu_id,
7685         kdu_name,
7686         member_vnf_index,
7687         vdu_index,
7688         vdu_name,
7689         deploy_params,
7690         descriptor_config,
7691         base_folder,
7692         task_instantiation_info,
7693         stage,
7694     ):
7695         # launch instantiate_N2VC in a asyncio task and register task object
7696         # Look where information of this charm is at database <nsrs>._admin.deployed.VCA
7697         # if not found, create one entry and update database
7698         # fill db_nsr._admin.deployed.VCA.<index>
7699
7700         self.logger.debug(
7701             logging_text + "_deploy_n2vc vnfd_id={}, vdu_id={}".format(vnfd_id, vdu_id)
7702         )
7703         if "execution-environment-list" in descriptor_config:
7704             ee_list = descriptor_config.get("execution-environment-list", [])
7705         elif "juju" in descriptor_config:
7706             ee_list = [descriptor_config]  # ns charms
7707         else:  # other types as script are not supported
7708             ee_list = []
7709
7710         for ee_item in ee_list:
7711             self.logger.debug(
7712                 logging_text
7713                 + "_deploy_n2vc ee_item juju={}, helm={}".format(
7714                     ee_item.get("juju"), ee_item.get("helm-chart")
7715                 )
7716             )
7717             ee_descriptor_id = ee_item.get("id")
7718             if ee_item.get("juju"):
7719                 vca_name = ee_item["juju"].get("charm")
7720                 vca_type = (
7721                     "lxc_proxy_charm"
7722                     if ee_item["juju"].get("charm") is not None
7723                     else "native_charm"
7724                 )
7725                 if ee_item["juju"].get("cloud") == "k8s":
7726                     vca_type = "k8s_proxy_charm"
7727                 elif ee_item["juju"].get("proxy") is False:
7728                     vca_type = "native_charm"
7729             elif ee_item.get("helm-chart"):
7730                 vca_name = ee_item["helm-chart"]
7731                 if ee_item.get("helm-version") and ee_item.get("helm-version") == "v2":
7732                     vca_type = "helm"
7733                 else:
7734                     vca_type = "helm-v3"
7735             else:
7736                 self.logger.debug(
7737                     logging_text + "skipping non juju neither charm configuration"
7738                 )
7739                 continue
7740
7741             vca_index = -1
7742             for vca_index, vca_deployed in enumerate(
7743                 db_nsr["_admin"]["deployed"]["VCA"]
7744             ):
7745                 if not vca_deployed:
7746                     continue
7747                 if (
7748                     vca_deployed.get("member-vnf-index") == member_vnf_index
7749                     and vca_deployed.get("vdu_id") == vdu_id
7750                     and vca_deployed.get("kdu_name") == kdu_name
7751                     and vca_deployed.get("vdu_count_index", 0) == vdu_index
7752                     and vca_deployed.get("ee_descriptor_id") == ee_descriptor_id
7753                 ):
7754                     break
7755             else:
7756                 # not found, create one.
7757                 target = (
7758                     "ns" if not member_vnf_index else "vnf/{}".format(member_vnf_index)
7759                 )
7760                 if vdu_id:
7761                     target += "/vdu/{}/{}".format(vdu_id, vdu_index or 0)
7762                 elif kdu_name:
7763                     target += "/kdu/{}".format(kdu_name)
7764                 vca_deployed = {
7765                     "target_element": target,
7766                     # ^ target_element will replace member-vnf-index, kdu_name, vdu_id ... in a single string
7767                     "member-vnf-index": member_vnf_index,
7768                     "vdu_id": vdu_id,
7769                     "kdu_name": kdu_name,
7770                     "vdu_count_index": vdu_index,
7771                     "operational-status": "init",  # TODO revise
7772                     "detailed-status": "",  # TODO revise
7773                     "step": "initial-deploy",  # TODO revise
7774                     "vnfd_id": vnfd_id,
7775                     "vdu_name": vdu_name,
7776                     "type": vca_type,
7777                     "ee_descriptor_id": ee_descriptor_id,
7778                 }
7779                 vca_index += 1
7780
7781                 # create VCA and configurationStatus in db
7782                 db_dict = {
7783                     "_admin.deployed.VCA.{}".format(vca_index): vca_deployed,
7784                     "configurationStatus.{}".format(vca_index): dict(),
7785                 }
7786                 self.update_db_2("nsrs", nsr_id, db_dict)
7787
7788                 db_nsr["_admin"]["deployed"]["VCA"].append(vca_deployed)
7789
7790             self.logger.debug("N2VC > NSR_ID > {}".format(nsr_id))
7791             self.logger.debug("N2VC > DB_NSR > {}".format(db_nsr))
7792             self.logger.debug("N2VC > VCA_DEPLOYED > {}".format(vca_deployed))
7793
7794             # Launch task
7795             task_n2vc = asyncio.ensure_future(
7796                 self.heal_N2VC(
7797                     logging_text=logging_text,
7798                     vca_index=vca_index,
7799                     nsi_id=nsi_id,
7800                     db_nsr=db_nsr,
7801                     db_vnfr=db_vnfr,
7802                     vdu_id=vdu_id,
7803                     kdu_name=kdu_name,
7804                     vdu_index=vdu_index,
7805                     deploy_params=deploy_params,
7806                     config_descriptor=descriptor_config,
7807                     base_folder=base_folder,
7808                     nslcmop_id=nslcmop_id,
7809                     stage=stage,
7810                     vca_type=vca_type,
7811                     vca_name=vca_name,
7812                     ee_config_descriptor=ee_item,
7813                 )
7814             )
7815             self.lcm_tasks.register(
7816                 "ns",
7817                 nsr_id,
7818                 nslcmop_id,
7819                 "instantiate_N2VC-{}".format(vca_index),
7820                 task_n2vc,
7821             )
7822             task_instantiation_info[
7823                 task_n2vc
7824             ] = self.task_name_deploy_vca + " {}.{}".format(
7825                 member_vnf_index or "", vdu_id or ""
7826             )
7827
7828     async def heal_N2VC(
7829         self,
7830         logging_text,
7831         vca_index,
7832         nsi_id,
7833         db_nsr,
7834         db_vnfr,
7835         vdu_id,
7836         kdu_name,
7837         vdu_index,
7838         config_descriptor,
7839         deploy_params,
7840         base_folder,
7841         nslcmop_id,
7842         stage,
7843         vca_type,
7844         vca_name,
7845         ee_config_descriptor,
7846     ):
7847         nsr_id = db_nsr["_id"]
7848         db_update_entry = "_admin.deployed.VCA.{}.".format(vca_index)
7849         vca_deployed_list = db_nsr["_admin"]["deployed"]["VCA"]
7850         vca_deployed = db_nsr["_admin"]["deployed"]["VCA"][vca_index]
7851         osm_config = {"osm": {"ns_id": db_nsr["_id"]}}
7852         db_dict = {
7853             "collection": "nsrs",
7854             "filter": {"_id": nsr_id},
7855             "path": db_update_entry,
7856         }
7857         step = ""
7858         try:
7859
7860             element_type = "NS"
7861             element_under_configuration = nsr_id
7862
7863             vnfr_id = None
7864             if db_vnfr:
7865                 vnfr_id = db_vnfr["_id"]
7866                 osm_config["osm"]["vnf_id"] = vnfr_id
7867
7868             namespace = "{nsi}.{ns}".format(nsi=nsi_id if nsi_id else "", ns=nsr_id)
7869
7870             if vca_type == "native_charm":
7871                 index_number = 0
7872             else:
7873                 index_number = vdu_index or 0
7874
7875             if vnfr_id:
7876                 element_type = "VNF"
7877                 element_under_configuration = vnfr_id
7878                 namespace += ".{}-{}".format(vnfr_id, index_number)
7879                 if vdu_id:
7880                     namespace += ".{}-{}".format(vdu_id, index_number)
7881                     element_type = "VDU"
7882                     element_under_configuration = "{}-{}".format(vdu_id, index_number)
7883                     osm_config["osm"]["vdu_id"] = vdu_id
7884                 elif kdu_name:
7885                     namespace += ".{}".format(kdu_name)
7886                     element_type = "KDU"
7887                     element_under_configuration = kdu_name
7888                     osm_config["osm"]["kdu_name"] = kdu_name
7889
7890             # Get artifact path
7891             if base_folder["pkg-dir"]:
7892                 artifact_path = "{}/{}/{}/{}".format(
7893                     base_folder["folder"],
7894                     base_folder["pkg-dir"],
7895                     "charms"
7896                     if vca_type
7897                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
7898                     else "helm-charts",
7899                     vca_name,
7900                 )
7901             else:
7902                 artifact_path = "{}/Scripts/{}/{}/".format(
7903                     base_folder["folder"],
7904                     "charms"
7905                     if vca_type
7906                     in ("native_charm", "lxc_proxy_charm", "k8s_proxy_charm")
7907                     else "helm-charts",
7908                     vca_name,
7909                 )
7910
7911             self.logger.debug("Artifact path > {}".format(artifact_path))
7912
7913             # get initial_config_primitive_list that applies to this element
7914             initial_config_primitive_list = config_descriptor.get(
7915                 "initial-config-primitive"
7916             )
7917
7918             self.logger.debug(
7919                 "Initial config primitive list > {}".format(
7920                     initial_config_primitive_list
7921                 )
7922             )
7923
7924             # add config if not present for NS charm
7925             ee_descriptor_id = ee_config_descriptor.get("id")
7926             self.logger.debug("EE Descriptor > {}".format(ee_descriptor_id))
7927             initial_config_primitive_list = get_ee_sorted_initial_config_primitive_list(
7928                 initial_config_primitive_list, vca_deployed, ee_descriptor_id
7929             )
7930
7931             self.logger.debug(
7932                 "Initial config primitive list #2 > {}".format(
7933                     initial_config_primitive_list
7934                 )
7935             )
7936             # n2vc_redesign STEP 3.1
7937             # find old ee_id if exists
7938             ee_id = vca_deployed.get("ee_id")
7939
7940             vca_id = self.get_vca_id(db_vnfr, db_nsr)
7941             # create or register execution environment in VCA. Only for native charms when healing
7942             if vca_type == "native_charm":
7943                 step = "Waiting to VM being up and getting IP address"
7944                 self.logger.debug(logging_text + step)
7945                 rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
7946                     logging_text,
7947                     nsr_id,
7948                     vnfr_id,
7949                     vdu_id,
7950                     vdu_index,
7951                     user=None,
7952                     pub_key=None,
7953                 )
7954                 credentials = {"hostname": rw_mgmt_ip}
7955                 # get username
7956                 username = deep_get(
7957                     config_descriptor, ("config-access", "ssh-access", "default-user")
7958                 )
7959                 # TODO remove this when changes on IM regarding config-access:ssh-access:default-user were
7960                 #  merged. Meanwhile let's get username from initial-config-primitive
7961                 if not username and initial_config_primitive_list:
7962                     for config_primitive in initial_config_primitive_list:
7963                         for param in config_primitive.get("parameter", ()):
7964                             if param["name"] == "ssh-username":
7965                                 username = param["value"]
7966                                 break
7967                 if not username:
7968                     raise LcmException(
7969                         "Cannot determine the username neither with 'initial-config-primitive' nor with "
7970                         "'config-access.ssh-access.default-user'"
7971                     )
7972                 credentials["username"] = username
7973
7974                 # n2vc_redesign STEP 3.2
7975                 # TODO: Before healing at RO it is needed to destroy native charm units to be deleted.
7976                 self._write_configuration_status(
7977                     nsr_id=nsr_id,
7978                     vca_index=vca_index,
7979                     status="REGISTERING",
7980                     element_under_configuration=element_under_configuration,
7981                     element_type=element_type,
7982                 )
7983
7984                 step = "register execution environment {}".format(credentials)
7985                 self.logger.debug(logging_text + step)
7986                 ee_id = await self.vca_map[vca_type].register_execution_environment(
7987                     credentials=credentials,
7988                     namespace=namespace,
7989                     db_dict=db_dict,
7990                     vca_id=vca_id,
7991                 )
7992
7993                 # update ee_id en db
7994                 db_dict_ee_id = {
7995                     "_admin.deployed.VCA.{}.ee_id".format(vca_index): ee_id,
7996                 }
7997                 self.update_db_2("nsrs", nsr_id, db_dict_ee_id)
7998
7999             # for compatibility with MON/POL modules, the need model and application name at database
8000             # TODO ask MON/POL if needed to not assuming anymore the format "model_name.application_name"
8001             # Not sure if this need to be done when healing
8002             """
8003             ee_id_parts = ee_id.split(".")
8004             db_nsr_update = {db_update_entry + "ee_id": ee_id}
8005             if len(ee_id_parts) >= 2:
8006                 model_name = ee_id_parts[0]
8007                 application_name = ee_id_parts[1]
8008                 db_nsr_update[db_update_entry + "model"] = model_name
8009                 db_nsr_update[db_update_entry + "application"] = application_name
8010             """
8011
8012             # n2vc_redesign STEP 3.3
8013             # Install configuration software. Only for native charms.
8014             step = "Install configuration Software"
8015
8016             self._write_configuration_status(
8017                 nsr_id=nsr_id,
8018                 vca_index=vca_index,
8019                 status="INSTALLING SW",
8020                 element_under_configuration=element_under_configuration,
8021                 element_type=element_type,
8022                 #other_update=db_nsr_update,
8023                 other_update=None,
8024             )
8025
8026             # TODO check if already done
8027             self.logger.debug(logging_text + step)
8028             config = None
8029             if vca_type == "native_charm":
8030                 config_primitive = next(
8031                     (p for p in initial_config_primitive_list if p["name"] == "config"),
8032                     None,
8033                 )
8034                 if config_primitive:
8035                     config = self._map_primitive_params(
8036                         config_primitive, {}, deploy_params
8037                     )
8038                 await self.vca_map[vca_type].install_configuration_sw(
8039                     ee_id=ee_id,
8040                     artifact_path=artifact_path,
8041                     db_dict=db_dict,
8042                     config=config,
8043                     num_units=1,
8044                     vca_id=vca_id,
8045                     vca_type=vca_type,
8046                 )
8047
8048             # write in db flag of configuration_sw already installed
8049             self.update_db_2(
8050                 "nsrs", nsr_id, {db_update_entry + "config_sw_installed": True}
8051             )
8052
8053             # Not sure if this need to be done when healing
8054             """
8055             # add relations for this VCA (wait for other peers related with this VCA)
8056             await self._add_vca_relations(
8057                 logging_text=logging_text,
8058                 nsr_id=nsr_id,
8059                 vca_type=vca_type,
8060                 vca_index=vca_index,
8061             )
8062             """
8063
8064             # if SSH access is required, then get execution environment SSH public
8065             # if native charm we have waited already to VM be UP
8066             if vca_type in ("k8s_proxy_charm", "lxc_proxy_charm", "helm", "helm-v3"):
8067                 pub_key = None
8068                 user = None
8069                 # self.logger.debug("get ssh key block")
8070                 if deep_get(
8071                     config_descriptor, ("config-access", "ssh-access", "required")
8072                 ):
8073                     # self.logger.debug("ssh key needed")
8074                     # Needed to inject a ssh key
8075                     user = deep_get(
8076                         config_descriptor,
8077                         ("config-access", "ssh-access", "default-user"),
8078                     )
8079                     step = "Install configuration Software, getting public ssh key"
8080                     pub_key = await self.vca_map[vca_type].get_ee_ssh_public__key(
8081                         ee_id=ee_id, db_dict=db_dict, vca_id=vca_id
8082                     )
8083
8084                     step = "Insert public key into VM user={} ssh_key={}".format(
8085                         user, pub_key
8086                     )
8087                 else:
8088                     # self.logger.debug("no need to get ssh key")
8089                     step = "Waiting to VM being up and getting IP address"
8090                 self.logger.debug(logging_text + step)
8091
8092                 # n2vc_redesign STEP 5.1
8093                 # wait for RO (ip-address) Insert pub_key into VM
8094                 # IMPORTANT: We need do wait for RO to complete healing operation.
8095                 await self._wait_heal_ro(nsr_id,self.timeout_ns_heal)
8096                 if vnfr_id:
8097                     if kdu_name:
8098                         rw_mgmt_ip = await self.wait_kdu_up(
8099                             logging_text, nsr_id, vnfr_id, kdu_name
8100                         )
8101                     else:
8102                         rw_mgmt_ip = await self.wait_vm_up_insert_key_ro(
8103                             logging_text,
8104                             nsr_id,
8105                             vnfr_id,
8106                             vdu_id,
8107                             vdu_index,
8108                             user=user,
8109                             pub_key=pub_key,
8110                         )
8111                 else:
8112                     rw_mgmt_ip = None  # This is for a NS configuration
8113
8114                 self.logger.debug(logging_text + " VM_ip_address={}".format(rw_mgmt_ip))
8115
8116             # store rw_mgmt_ip in deploy params for later replacement
8117             deploy_params["rw_mgmt_ip"] = rw_mgmt_ip
8118
8119             # Day1 operations.
8120             # get run-day1 operation parameter
8121             runDay1 =  deploy_params.get("run-day1",False)
8122             self.logger.debug(" Healing vnf={}, vdu={}, runDay1 ={}".format(vnfr_id,vdu_id,runDay1))
8123             if runDay1:
8124                 # n2vc_redesign STEP 6  Execute initial config primitive
8125                 step = "execute initial config primitive"
8126
8127                 # wait for dependent primitives execution (NS -> VNF -> VDU)
8128                 if initial_config_primitive_list:
8129                     await self._wait_dependent_n2vc(nsr_id, vca_deployed_list, vca_index)
8130
8131                 # stage, in function of element type: vdu, kdu, vnf or ns
8132                 my_vca = vca_deployed_list[vca_index]
8133                 if my_vca.get("vdu_id") or my_vca.get("kdu_name"):
8134                     # VDU or KDU
8135                     stage[0] = "Stage 3/5: running Day-1 primitives for VDU."
8136                 elif my_vca.get("member-vnf-index"):
8137                     # VNF
8138                     stage[0] = "Stage 4/5: running Day-1 primitives for VNF."
8139                 else:
8140                     # NS
8141                     stage[0] = "Stage 5/5: running Day-1 primitives for NS."
8142
8143                 self._write_configuration_status(
8144                     nsr_id=nsr_id, vca_index=vca_index, status="EXECUTING PRIMITIVE"
8145                 )
8146
8147                 self._write_op_status(op_id=nslcmop_id, stage=stage)
8148
8149                 check_if_terminated_needed = True
8150                 for initial_config_primitive in initial_config_primitive_list:
8151                     # adding information on the vca_deployed if it is a NS execution environment
8152                     if not vca_deployed["member-vnf-index"]:
8153                         deploy_params["ns_config_info"] = json.dumps(
8154                             self._get_ns_config_info(nsr_id)
8155                         )
8156                     # TODO check if already done
8157                     primitive_params_ = self._map_primitive_params(
8158                         initial_config_primitive, {}, deploy_params
8159                     )
8160
8161                     step = "execute primitive '{}' params '{}'".format(
8162                         initial_config_primitive["name"], primitive_params_
8163                     )
8164                     self.logger.debug(logging_text + step)
8165                     await self.vca_map[vca_type].exec_primitive(
8166                         ee_id=ee_id,
8167                         primitive_name=initial_config_primitive["name"],
8168                         params_dict=primitive_params_,
8169                         db_dict=db_dict,
8170                         vca_id=vca_id,
8171                         vca_type=vca_type,
8172                     )
8173                     # Once some primitive has been exec, check and write at db if it needs to exec terminated primitives
8174                     if check_if_terminated_needed:
8175                         if config_descriptor.get("terminate-config-primitive"):
8176                             self.update_db_2(
8177                                 "nsrs", nsr_id, {db_update_entry + "needed_terminate": True}
8178                             )
8179                         check_if_terminated_needed = False
8180
8181                     # TODO register in database that primitive is done
8182
8183             # STEP 7 Configure metrics
8184             # Not sure if this need to be done when healing
8185             """
8186             if vca_type == "helm" or vca_type == "helm-v3":
8187                 prometheus_jobs = await self.extract_prometheus_scrape_jobs(
8188                     ee_id=ee_id,
8189                     artifact_path=artifact_path,
8190                     ee_config_descriptor=ee_config_descriptor,
8191                     vnfr_id=vnfr_id,
8192                     nsr_id=nsr_id,
8193                     target_ip=rw_mgmt_ip,
8194                 )
8195                 if prometheus_jobs:
8196                     self.update_db_2(
8197                         "nsrs",
8198                         nsr_id,
8199                         {db_update_entry + "prometheus_jobs": prometheus_jobs},
8200                     )
8201
8202                     for job in prometheus_jobs:
8203                         self.db.set_one(
8204                             "prometheus_jobs",
8205                             {"job_name": job["job_name"]},
8206                             job,
8207                             upsert=True,
8208                             fail_on_empty=False,
8209                         )
8210
8211             """
8212             step = "instantiated at VCA"
8213             self.logger.debug(logging_text + step)
8214
8215             self._write_configuration_status(
8216                 nsr_id=nsr_id, vca_index=vca_index, status="READY"
8217             )
8218
8219         except Exception as e:  # TODO not use Exception but N2VC exception
8220             # self.update_db_2("nsrs", nsr_id, {db_update_entry + "instantiation": "FAILED"})
8221             if not isinstance(
8222                 e, (DbException, N2VCException, LcmException, asyncio.CancelledError)
8223             ):
8224                 self.logger.error(
8225                     "Exception while {} : {}".format(step, e), exc_info=True
8226                 )
8227             self._write_configuration_status(
8228                 nsr_id=nsr_id, vca_index=vca_index, status="BROKEN"
8229             )
8230             raise LcmException("{} {}".format(step, e)) from e
8231
8232     async def _wait_heal_ro(
8233         self,
8234         nsr_id,
8235         timeout=600,
8236     ):
8237         start_time = time()
8238         while time() <= start_time + timeout:
8239             db_nsr = self.db.get_one("nsrs", {"_id": nsr_id})
8240             operational_status_ro = db_nsr["_admin"]["deployed"]["RO"]["operational-status"]
8241             self.logger.debug("Wait Heal RO > {}".format(operational_status_ro))
8242             if operational_status_ro != "healing":
8243                 break
8244             await asyncio.sleep(15, loop=self.loop)
8245         else:  # timeout_ns_deploy
8246             raise NgRoException("Timeout waiting ns to deploy")