osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import asyncio
  17 import logging
  18 import logging.handlers
  19 import traceback
  20 from osm_lcm import ROclient
  21 from osm_lcm.lcm_utils import (
  22     LcmException,
  23     LcmBase,
  24     populate_dict,
  25     get_iterable,
  26     deep_get,
  27 )
  28 from osm_common.dbbase import DbException
  29 from time import time
  30 from copy import deepcopy
  31
  32
  33 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  34
  35
  36 class NetsliceLcm(LcmBase):
  37     def __init__(self, msg, lcm_tasks, config, loop, ns):
  38         """
  39         Init, Connect to database, filesystem storage, and messaging
  40         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  41         :return: None
  42         """
  43         # logging
  44         self.logger = logging.getLogger("lcm.netslice")
  45         self.loop = loop
  46         self.lcm_tasks = lcm_tasks
  47         self.ns = ns
  48         self.ro_config = config["RO"]
  49         self.timeout = config["timeout"]
  50
  51         super().__init__(msg, self.logger)
  52
  53     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  54         """
  55         Updates database nsir with the RO info for the created vld
  56         :param nsi_update_nsir: dictionary to be filled with the updated info
  57         :param db_nsir: content of db_nsir. This is also modified
  58         :param nsir_desc_RO: nsir descriptor from RO
  59         :return: Nothing, LcmException is raised on errors
  60         """
  61
  62         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  63             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  64                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  65                     continue
  66                 vld["vim-id"] = net_RO.get("vim_net_id")
  67                 vld["name"] = net_RO.get("vim_name")
  68                 vld["status"] = net_RO.get("status")
  69                 vld["status-detailed"] = net_RO.get("error_msg")
  70                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  71                 break
  72             else:
  73                 raise LcmException(
  74                     "ns_update_nsir: Not found vld={} at RO info".format(vld["id"])
  75                 )
  76
  77     async def instantiate(self, nsir_id, nsilcmop_id):
  78
  79         # Try to lock HA task here
  80         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
  81         if not task_is_locked_by_me:
  82             return
  83
  84         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  85         self.logger.debug(logging_text + "Enter")
  86         # get all needed from database
  87         exc = None
  88         db_nsir = None
  89         db_nsilcmop = None
  90         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  91         db_nsilcmop_update = {}
  92         nsilcmop_operation_state = None
  93         vim_2_RO = {}
  94         RO = ROclient.ROClient(self.loop, **self.ro_config)
  95         nsi_vld_instantiationi_params = {}
  96
  97         def ip_profile_2_RO(ip_profile):
  98             RO_ip_profile = deepcopy((ip_profile))
  99             if "dns-server" in RO_ip_profile:
 100                 if isinstance(RO_ip_profile["dns-server"], list):
 101                     RO_ip_profile["dns-address"] = []
 102                     for ds in RO_ip_profile.pop("dns-server"):
 103                         RO_ip_profile["dns-address"].append(ds["address"])
 104                 else:
 105                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 106             if RO_ip_profile.get("ip-version") == "ipv4":
 107                 RO_ip_profile["ip-version"] = "IPv4"
 108             if RO_ip_profile.get("ip-version") == "ipv6":
 109                 RO_ip_profile["ip-version"] = "IPv6"
 110             if "dhcp-params" in RO_ip_profile:
 111                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 112             return RO_ip_profile
 113
 114         def vim_account_2_RO(vim_account):
 115             """
 116             Translate a RO vim_account from OSM vim_account params
 117             :param ns_params: OSM instantiate params
 118             :return: The RO ns descriptor
 119             """
 120             if vim_account in vim_2_RO:
 121                 return vim_2_RO[vim_account]
 122
 123             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 124             if db_vim["_admin"]["operationalState"] != "ENABLED":
 125                 raise LcmException(
 126                     "VIM={} is not available. operationalState={}".format(
 127                         vim_account, db_vim["_admin"]["operationalState"]
 128                     )
 129                 )
 130             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 131             vim_2_RO[vim_account] = RO_vim_id
 132             return RO_vim_id
 133
 134         async def netslice_scenario_create(
 135             self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 136         ):
 137             """
 138             Create a network slice VLD through RO Scenario
 139             :param vld_id The VLD id inside nsir to be created
 140             :param nsir_id The nsir id
 141             """
 142             nonlocal nsi_vld_instantiationi_params
 143             ip_vld = None
 144             mgmt_network = False
 145             RO_vld_sites = []
 146             vld_id = vld_item["id"]
 147             netslice_vld = vld_item
 148             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 149             # self.logger.debug(logging_text + "Enter")
 150
 151             vld_shared = None
 152             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 153                 _filter = {
 154                     "_id.ne": nsir_id,
 155                     "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item,
 156                 }
 157                 shared_nsi = self.db.get_one(
 158                     "nsis", _filter, fail_on_empty=False, fail_on_more=False
 159                 )
 160                 if shared_nsi:
 161                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 162                         if vld_id == vlds["vld_id"]:
 163                             vld_shared = {
 164                                 "instance_scenario_id": vlds["netslice_scenario_id"],
 165                                 "osm_id": vld_id,
 166                             }
 167                             break
 168                     break
 169
 170             # Creating netslice-vld at RO
 171             RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
 172
 173             if vld_id in RO_nsir:
 174                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 175
 176             # If netslice-vld doesn't exists then create it
 177             else:
 178                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 179                 # Updating network names and datacenters from instantiation parameters for each VLD
 180                 for instantiation_params_vld in get_iterable(
 181                     db_nsir["instantiation_parameters"], "netslice-vld"
 182                 ):
 183                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 184                         ip_vld = deepcopy(instantiation_params_vld)
 185                         ip_vld.pop("name")
 186                         nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
 187
 188                 db_nsir_update_RO = {}
 189                 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
 190                 if self.ro_config["ng"]:
 191                     db_nsir_update_RO["netslice_scenario_id"] = (
 192                         vld_shared.get("instance_scenario_id")
 193                         if vld_shared
 194                         else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
 195                     )
 196                 else:  # if not self.ro_config["ng"]:
 197                     if netslice_vld.get("mgmt-network"):
 198                         mgmt_network = True
 199                     RO_ns_params = {}
 200                     RO_ns_params["name"] = netslice_vld["name"]
 201                     RO_ns_params["datacenter"] = vim_account_2_RO(
 202                         db_nsir["instantiation_parameters"]["vimAccountId"]
 203                     )
 204
 205                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 206                     # Use vim-network-id instantiation parameter
 207                     vim_network_option = None
 208                     if ip_vld:
 209                         if ip_vld.get("vim-network-id"):
 210                             vim_network_option = "vim-network-id"
 211                         elif ip_vld.get("vim-network-name"):
 212                             vim_network_option = "vim-network-name"
 213                         if ip_vld.get("ip-profile"):
 214                             populate_dict(
 215                                 RO_ns_params,
 216                                 ("networks", netslice_vld["name"], "ip-profile"),
 217                                 ip_profile_2_RO(ip_vld["ip-profile"]),
 218                             )
 219
 220                     if vim_network_option:
 221                         if ip_vld.get(vim_network_option):
 222                             if isinstance(ip_vld.get(vim_network_option), list):
 223                                 for vim_net_id in ip_vld.get(vim_network_option):
 224                                     for vim_account, vim_net in vim_net_id.items():
 225                                         RO_vld_sites.append(
 226                                             {
 227                                                 "netmap-use": vim_net,
 228                                                 "datacenter": vim_account_2_RO(
 229                                                     vim_account
 230                                                 ),
 231                                             }
 232                                         )
 233                             elif isinstance(ip_vld.get(vim_network_option), dict):
 234                                 for vim_account, vim_net in ip_vld.get(
 235                                     vim_network_option
 236                                 ).items():
 237                                     RO_vld_sites.append(
 238                                         {
 239                                             "netmap-use": vim_net,
 240                                             "datacenter": vim_account_2_RO(vim_account),
 241                                         }
 242                                     )
 243                             else:
 244                                 RO_vld_sites.append(
 245                                     {
 246                                         "netmap-use": ip_vld[vim_network_option],
 247                                         "datacenter": vim_account_2_RO(
 248                                             netslice_vld["vimAccountId"]
 249                                         ),
 250                                     }
 251                                 )
 252
 253                     # Use default netslice vim-network-name from template
 254                     else:
 255                         for nss_conn_point_ref in get_iterable(
 256                             netslice_vld, "nss-connection-point-ref"
 257                         ):
 258                             if nss_conn_point_ref.get("vimAccountId"):
 259                                 if (
 260                                     nss_conn_point_ref["vimAccountId"]
 261                                     != netslice_vld["vimAccountId"]
 262                                 ):
 263                                     RO_vld_sites.append(
 264                                         {
 265                                             "netmap-create": None,
 266                                             "datacenter": vim_account_2_RO(
 267                                                 nss_conn_point_ref["vimAccountId"]
 268                                             ),
 269                                         }
 270                                     )
 271
 272                     if vld_shared:
 273                         populate_dict(
 274                             RO_ns_params,
 275                             ("networks", netslice_vld["name"], "use-network"),
 276                             vld_shared,
 277                         )
 278
 279                     if RO_vld_sites:
 280                         populate_dict(
 281                             RO_ns_params,
 282                             ("networks", netslice_vld["name"], "sites"),
 283                             RO_vld_sites,
 284                         )
 285
 286                     RO_ns_params["scenario"] = {
 287                         "nets": [
 288                             {
 289                                 "name": netslice_vld["name"],
 290                                 "external": mgmt_network,
 291                                 "type": "bridge",
 292                             }
 293                         ]
 294                     }
 295
 296                     # self.logger.debug(logging_text + step)
 297                     desc = await RO.create("ns", descriptor=RO_ns_params)
 298                     db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 299                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 300
 301         def overwrite_nsd_params(self, db_nsir, nslcmop):
 302             nonlocal nsi_vld_instantiationi_params
 303             nonlocal db_nsir_update
 304             vld_op_list = []
 305             vld = None
 306             nsr_id = nslcmop.get("nsInstanceId")
 307             # Overwrite instantiation parameters in netslice runtime
 308             RO_list = db_nsir_admin["deployed"]["RO"]
 309
 310             for ro_item_index, RO_item in enumerate(RO_list):
 311                 netslice_vld = next(
 312                     (
 313                         n
 314                         for n in get_iterable(db_nsir["_admin"], "netslice-vld")
 315                         if RO_item.get("vld_id") == n.get("id")
 316                     ),
 317                     None,
 318                 )
 319                 if not netslice_vld:
 320                     continue
 321                 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 322                 # Search the cp of netslice-vld that match with nst:netslice-subnet
 323                 for nss_cp_item in get_iterable(
 324                     netslice_vld, "nss-connection-point-ref"
 325                 ):
 326                     # Search the netslice-subnet of nst that match
 327                     nss = next(
 328                         (
 329                             nss
 330                             for nss in get_iterable(
 331                                 db_nsir["_admin"], "netslice-subnet"
 332                             )
 333                             if nss_cp_item["nss-ref"] == nss["nss-id"]
 334                         ),
 335                         None,
 336                     )
 337                     # Compare nss-ref equal nss from nst
 338                     if not nss:
 339                         continue
 340                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 341                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 342                     for cp_nsd in db_nsds.get("sapd", ()):
 343                         if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
 344                             if nslcmop.get("operationParams"):
 345                                 if (
 346                                     nslcmop["operationParams"].get("nsName")
 347                                     == nss["nsName"]
 348                                 ):
 349                                     vld_id = RO_item["vld_id"]
 350                                     netslice_scenario_id = RO_item[
 351                                         "netslice_scenario_id"
 352                                     ]
 353                                     nslcmop_vld = {}
 354                                     nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
 355                                     for vld in get_iterable(
 356                                         nslcmop["operationParams"], "vld"
 357                                     ):
 358                                         if vld["name"] == cp_nsd["virtual-link-desc"]:
 359                                             nslcmop_vld.update(vld)
 360                                     if self.ro_config["ng"]:
 361                                         nslcmop_vld["common_id"] = netslice_scenario_id
 362                                         nslcmop_vld.update(
 363                                             nsi_vld_instantiationi_params.get(
 364                                                 RO_item["vld_id"], {}
 365                                             )
 366                                         )
 367                                     else:
 368                                         nslcmop_vld["ns-net"] = {
 369                                             vld_id: netslice_scenario_id
 370                                         }
 371                                     vld_op_list.append(nslcmop_vld)
 372             nslcmop["operationParams"]["vld"] = vld_op_list
 373             self.update_db_2(
 374                 "nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list}
 375             )
 376             return nsr_id, nslcmop
 377
 378         try:
 379             # wait for any previous tasks in process
 380             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 381
 382             step = "Getting nsir={} from db".format(nsir_id)
 383             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 384             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 385             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 386
 387             start_deploy = time()
 388             nsi_params = db_nsilcmop.get("operationParams")
 389             if nsi_params and nsi_params.get("timeout_nsi_deploy"):
 390                 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
 391             else:
 392                 timeout_nsi_deploy = self.timeout.get("nsi_deploy")
 393
 394             # Empty list to keep track of network service records status in the netslice
 395             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 396
 397             step = "Creating slice operational-status init"
 398             # Slice status Creating
 399             db_nsir_update["detailed-status"] = "creating"
 400             db_nsir_update["operational-status"] = "init"
 401             db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 402
 403             step = "Instantiating netslice VLDs before NS instantiation"
 404             # Creating netslice VLDs networking before NS instantiation
 405             db_nsir_update["detailed-status"] = step
 406             self.update_db_2("nsis", nsir_id, db_nsir_update)
 407             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 408             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 409                 await netslice_scenario_create(
 410                     self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 411                 )
 412
 413             step = "Instantiating netslice subnets"
 414             db_nsir_update["detailed-status"] = step
 415             self.update_db_2("nsis", nsir_id, db_nsir_update)
 416
 417             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 418
 419             # Check status of the VLDs and wait for creation
 420             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 421             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 422             # for netslice_scenario in netslice_scenarios:
 423             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 424             #                                  nsir_id, db_nsir_update_RO)
 425
 426             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 427             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 428
 429             # Iterate over the network services operation ids to instantiate NSs
 430             step = "Instantiating Netslice Subnets"
 431             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 432             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 433             for nslcmop_id in nslcmop_ids:
 434                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 435                 # Overwriting netslice-vld vim-net-id to ns
 436                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 437                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 438                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 439                 self.lcm_tasks.register(
 440                     "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 441                 )
 442
 443             # Wait until Network Slice is ready
 444             step = " Waiting nsi ready."
 445             nsrs_detailed_list_old = None
 446             self.logger.debug(logging_text + step)
 447
 448             # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
 449             while time() <= start_deploy + timeout_nsi_deploy:
 450                 # Check ns instantiation status
 451                 nsi_ready = True
 452                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 453                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 454                 nsrs_detailed_list_new = []
 455                 for nslcmop_item in nslcmop_ids:
 456                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 457                     status = nslcmop.get("operationState")
 458                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 459                     for nss in nsrs_detailed_list:
 460                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 461                             nss.update(
 462                                 {
 463                                     "nsrId": nslcmop["nsInstanceId"],
 464                                     "status": nslcmop["operationState"],
 465                                     "detailed-status": nslcmop.get("detailed-status"),
 466                                     "instantiated": True,
 467                                 }
 468                             )
 469                             nsrs_detailed_list_new.append(nss)
 470                     if status not in [
 471                         "COMPLETED",
 472                         "PARTIALLY_COMPLETED",
 473                         "FAILED",
 474                         "FAILED_TEMP",
 475                     ]:
 476                         nsi_ready = False
 477
 478                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 479                     nsrs_detailed_list_old = nsrs_detailed_list_new
 480                     self.update_db_2(
 481                         "nsis",
 482                         nsir_id,
 483                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 484                     )
 485
 486                 if nsi_ready:
 487                     error_list = []
 488                     step = "Network Slice Instance instantiated"
 489                     for nss in nsrs_detailed_list:
 490                         if nss["status"] in ("FAILED", "FAILED_TEMP"):
 491                             error_list.append(
 492                                 "NS {} {}: {}".format(
 493                                     nss["nsrId"], nss["status"], nss["detailed-status"]
 494                                 )
 495                             )
 496                     if error_list:
 497                         step = "instantiating"
 498                         raise LcmException("; ".join(error_list))
 499                     break
 500
 501                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 502                 await asyncio.sleep(5, loop=self.loop)
 503
 504             else:  # timeout_nsi_deploy reached:
 505                 raise LcmException("Timeout waiting nsi to be ready.")
 506
 507             db_nsir_update["operational-status"] = "running"
 508             db_nsir_update["detailed-status"] = "done"
 509             db_nsir_update["config-status"] = "configured"
 510             db_nsilcmop_update[
 511                 "operationState"
 512             ] = nsilcmop_operation_state = "COMPLETED"
 513             db_nsilcmop_update["statusEnteredTime"] = time()
 514             db_nsilcmop_update["detailed-status"] = "done"
 515             return
 516
 517         except (LcmException, DbException) as e:
 518             self.logger.error(
 519                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 520             )
 521             exc = e
 522         except asyncio.CancelledError:
 523             self.logger.error(
 524                 logging_text + "Cancelled Exception while '{}'".format(step)
 525             )
 526             exc = "Operation was cancelled"
 527         except Exception as e:
 528             exc = traceback.format_exc()
 529             self.logger.critical(
 530                 logging_text
 531                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 532                 exc_info=True,
 533             )
 534         finally:
 535             if exc:
 536                 if db_nsir:
 537                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 538                     db_nsir_update["operational-status"] = "failed"
 539                     db_nsir_update["config-status"] = "configured"
 540                 if db_nsilcmop:
 541                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 542                         step, exc
 543                     )
 544                     db_nsilcmop_update[
 545                         "operationState"
 546                     ] = nsilcmop_operation_state = "FAILED"
 547                     db_nsilcmop_update["statusEnteredTime"] = time()
 548             try:
 549                 if db_nsir:
 550                     db_nsir_update["_admin.nsilcmop"] = None
 551                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 552                 if db_nsilcmop:
 553                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 554             except DbException as e:
 555                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 556             if nsilcmop_operation_state:
 557                 try:
 558                     await self.msg.aiowrite(
 559                         "nsi",
 560                         "instantiated",
 561                         {
 562                             "nsir_id": nsir_id,
 563                             "nsilcmop_id": nsilcmop_id,
 564                             "operationState": nsilcmop_operation_state,
 565                         },
 566                     )
 567                 except Exception as e:
 568                     self.logger.error(
 569                         logging_text + "kafka_write notification Exception {}".format(e)
 570                     )
 571             self.logger.debug(logging_text + "Exit")
 572             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 573
 574     async def terminate(self, nsir_id, nsilcmop_id):
 575
 576         # Try to lock HA task here
 577         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
 578         if not task_is_locked_by_me:
 579             return
 580
 581         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 582         self.logger.debug(logging_text + "Enter")
 583         exc = None
 584         db_nsir = None
 585         db_nsilcmop = None
 586         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 587         db_nsilcmop_update = {}
 588         RO = ROclient.ROClient(self.loop, **self.ro_config)
 589         nsir_deployed = None
 590         failed_detail = []  # annotates all failed error messages
 591         nsilcmop_operation_state = None
 592         autoremove = False  # autoremove after terminated
 593         try:
 594             # wait for any previous tasks in process
 595             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 596
 597             step = "Getting nsir={} from db".format(nsir_id)
 598             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 599             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 600             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 601             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 602
 603             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 604             # CASE: Instance was terminated but there is a second request to terminate the instance
 605             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 606                 return
 607
 608             # Slice status Terminating
 609             db_nsir_update["operational-status"] = "terminating"
 610             db_nsir_update["config-status"] = "terminating"
 611             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 612             self.update_db_2("nsis", nsir_id, db_nsir_update)
 613
 614             # Gets the list to keep track of network service records status in the netslice
 615             nsrs_detailed_list = []
 616
 617             # Iterate over the network services operation ids to terminate NSs
 618             # TODO: (future improvement) look another way check the tasks instead of keep asking
 619             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 620             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 621             step = "Terminating Netslice Subnets"
 622             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 623             nslcmop_new = []
 624             for nslcmop_id in nslcmop_ids:
 625                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 626                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 627                 nss_in_use = self.db.get_list(
 628                     "nsis",
 629                     {
 630                         "_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 631                         "operational-status": {"$nin": ["terminated", "failed"]},
 632                     },
 633                 )
 634                 if len(nss_in_use) < 2:
 635                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 636                     self.lcm_tasks.register(
 637                         "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 638                     )
 639                     nslcmop_new.append(nslcmop_id)
 640                 else:
 641                     # Update shared nslcmop shared with active nsi
 642                     netsliceInstanceId = db_nsir["_id"]
 643                     for nsis_item in nss_in_use:
 644                         if db_nsir["_id"] != nsis_item["_id"]:
 645                             netsliceInstanceId = nsis_item["_id"]
 646                             break
 647                     self.db.set_one(
 648                         "nslcmops",
 649                         {"_id": nslcmop_id},
 650                         {"operationParams.netsliceInstanceId": netsliceInstanceId},
 651                     )
 652             self.db.set_one(
 653                 "nsilcmops",
 654                 {"_id": nsilcmop_id},
 655                 {"operationParams.nslcmops_ids": nslcmop_new},
 656             )
 657
 658             # Wait until Network Slice is terminated
 659             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(
 660                 nsir_id
 661             )
 662             nsrs_detailed_list_old = None
 663             self.logger.debug(logging_text + step)
 664
 665             termination_timeout = 2 * 3600  # Two hours
 666             while termination_timeout > 0:
 667                 # Check ns termination status
 668                 nsi_ready = True
 669                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 670                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 671                 nsrs_detailed_list_new = []
 672                 for nslcmop_item in nslcmop_ids:
 673                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 674                     status = nslcmop["operationState"]
 675                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 676                     for nss in nsrs_detailed_list:
 677                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 678                             nss.update(
 679                                 {
 680                                     "nsrId": nslcmop["nsInstanceId"],
 681                                     "status": nslcmop["operationState"],
 682                                     "detailed-status": nsir_status_detailed
 683                                     + "; {}".format(nslcmop.get("detailed-status")),
 684                                 }
 685                             )
 686                             nsrs_detailed_list_new.append(nss)
 687                     if status not in [
 688                         "COMPLETED",
 689                         "PARTIALLY_COMPLETED",
 690                         "FAILED",
 691                         "FAILED_TEMP",
 692                     ]:
 693                         nsi_ready = False
 694
 695                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 696                     nsrs_detailed_list_old = nsrs_detailed_list_new
 697                     self.update_db_2(
 698                         "nsis",
 699                         nsir_id,
 700                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 701                     )
 702
 703                 if nsi_ready:
 704                     # Check if it is the last used nss and mark isinstantiate: False
 705                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 706                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 707                     for nss in nsrs_detailed_list:
 708                         _filter = {
 709                             "_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 710                             "operational-status.ne": "terminated",
 711                             "_id.ne": nsir_id,
 712                         }
 713                         nsis_list = self.db.get_one(
 714                             "nsis", _filter, fail_on_empty=False, fail_on_more=False
 715                         )
 716                         if not nsis_list:
 717                             nss.update({"instantiated": False})
 718
 719                     step = "Network Slice Instance is terminated. nsi_id={}".format(
 720                         nsir_id
 721                     )
 722                     for items in nsrs_detailed_list:
 723                         if "FAILED" in items.values():
 724                             raise LcmException(
 725                                 "Error terminating NSI: {}".format(nsir_id)
 726                             )
 727                     break
 728
 729                 await asyncio.sleep(5, loop=self.loop)
 730                 termination_timeout -= 5
 731
 732             if termination_timeout <= 0:
 733                 raise LcmException(
 734                     "Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id)
 735                 )
 736
 737             # Delete netslice-vlds
 738             RO_nsir_id = RO_delete_action = None
 739             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 740                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 741                 try:
 742                     if not self.ro_config["ng"]:
 743                         step = db_nsir_update[
 744                             "detailed-status"
 745                         ] = "Deleting netslice-vld at RO"
 746                         db_nsilcmop_update[
 747                             "detailed-status"
 748                         ] = "Deleting netslice-vld at RO"
 749                         self.logger.debug(logging_text + step)
 750                         desc = await RO.delete("ns", RO_nsir_id)
 751                         RO_delete_action = desc["action_id"]
 752                         nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 753                         nsir_deployed_RO["vld_status"] = "DELETING"
 754                         db_nsir_update["_admin.deployed"] = nsir_deployed
 755                         self.update_db_2("nsis", nsir_id, db_nsir_update)
 756                         if RO_delete_action:
 757                             # wait until NS is deleted from VIM
 758                             step = "Waiting ns deleted from VIM. RO_id={}".format(
 759                                 RO_nsir_id
 760                             )
 761                             self.logger.debug(logging_text + step)
 762                 except ROclient.ROClientException as e:
 763                     if e.http_code == 404:  # not found
 764                         nsir_deployed_RO["vld_id"] = None
 765                         nsir_deployed_RO["vld_status"] = "DELETED"
 766                         self.logger.debug(
 767                             logging_text
 768                             + "RO_ns_id={} already deleted".format(RO_nsir_id)
 769                         )
 770                     elif e.http_code == 409:  # conflict
 771                         failed_detail.append(
 772                             "RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e)
 773                         )
 774                         self.logger.debug(logging_text + failed_detail[-1])
 775                     else:
 776                         failed_detail.append(
 777                             "RO_ns_id={} delete error: {}".format(RO_nsir_id, e)
 778                         )
 779                         self.logger.error(logging_text + failed_detail[-1])
 780
 781                 if failed_detail:
 782                     self.logger.error(logging_text + " ;".join(failed_detail))
 783                     db_nsir_update["operational-status"] = "failed"
 784                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(
 785                         failed_detail
 786                     )
 787                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 788                     db_nsilcmop_update[
 789                         "operationState"
 790                     ] = nsilcmop_operation_state = "FAILED"
 791                     db_nsilcmop_update["statusEnteredTime"] = time()
 792                 else:
 793                     db_nsir_update["operational-status"] = "terminating"
 794                     db_nsir_update["config-status"] = "terminating"
 795                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 796                     db_nsilcmop_update[
 797                         "operationState"
 798                     ] = nsilcmop_operation_state = "COMPLETED"
 799                     db_nsilcmop_update["statusEnteredTime"] = time()
 800                     if db_nsilcmop["operationParams"].get("autoremove"):
 801                         autoremove = True
 802
 803             db_nsir_update["detailed-status"] = "done"
 804             db_nsir_update["operational-status"] = "terminated"
 805             db_nsir_update["config-status"] = "terminated"
 806             db_nsilcmop_update["statusEnteredTime"] = time()
 807             db_nsilcmop_update["detailed-status"] = "done"
 808             return
 809
 810         except (LcmException, DbException) as e:
 811             self.logger.error(
 812                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 813             )
 814             exc = e
 815         except asyncio.CancelledError:
 816             self.logger.error(
 817                 logging_text + "Cancelled Exception while '{}'".format(step)
 818             )
 819             exc = "Operation was cancelled"
 820         except Exception as e:
 821             exc = traceback.format_exc()
 822             self.logger.critical(
 823                 logging_text
 824                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 825                 exc_info=True,
 826             )
 827         finally:
 828             if exc:
 829                 if db_nsir:
 830                     db_nsir_update["_admin.deployed"] = nsir_deployed
 831                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 832                     db_nsir_update["operational-status"] = "failed"
 833                 if db_nsilcmop:
 834                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 835                         step, exc
 836                     )
 837                     db_nsilcmop_update[
 838                         "operationState"
 839                     ] = nsilcmop_operation_state = "FAILED"
 840                     db_nsilcmop_update["statusEnteredTime"] = time()
 841             try:
 842                 if db_nsir:
 843                     db_nsir_update["_admin.deployed"] = nsir_deployed
 844                     db_nsir_update["_admin.nsilcmop"] = None
 845                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 846                 if db_nsilcmop:
 847                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 848             except DbException as e:
 849                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 850
 851             if nsilcmop_operation_state:
 852                 try:
 853                     await self.msg.aiowrite(
 854                         "nsi",
 855                         "terminated",
 856                         {
 857                             "nsir_id": nsir_id,
 858                             "nsilcmop_id": nsilcmop_id,
 859                             "operationState": nsilcmop_operation_state,
 860                             "autoremove": autoremove,
 861                         },
 862                         loop=self.loop,
 863                     )
 864                 except Exception as e:
 865                     self.logger.error(
 866                         logging_text + "kafka_write notification Exception {}".format(e)
 867                     )
 868             self.logger.debug(logging_text + "Exit")
 869             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")