osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import asyncio
  17 import logging
  18 import logging.handlers
  19 import traceback
  20 from osm_lcm import ROclient
  21 from osm_lcm.lcm_utils import LcmException, LcmBase, populate_dict, get_iterable, deep_get
  22 from osm_common.dbbase import DbException
  23 from time import time
  24 from copy import deepcopy
  25
  26
  27 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  28
  29
  30 class NetsliceLcm(LcmBase):
  31
  32     timeout_nsi_deploy = 2 * 3600  # default global timeout for deployment a nsi
  33
  34     def __init__(self, msg, lcm_tasks, config, loop, ns):
  35         """
  36         Init, Connect to database, filesystem storage, and messaging
  37         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  38         :return: None
  39         """
  40         # logging
  41         self.logger = logging.getLogger('lcm.netslice')
  42         self.loop = loop
  43         self.lcm_tasks = lcm_tasks
  44         self.ns = ns
  45         self.ro_config = config["ro_config"]
  46         self.timeout = config["timeout"]
  47
  48         super().__init__(msg, self.logger)
  49
  50     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  51         """
  52         Updates database nsir with the RO info for the created vld
  53         :param nsi_update_nsir: dictionary to be filled with the updated info
  54         :param db_nsir: content of db_nsir. This is also modified
  55         :param nsir_desc_RO: nsir descriptor from RO
  56         :return: Nothing, LcmException is raised on errors
  57         """
  58
  59         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  60             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  61                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  62                     continue
  63                 vld["vim-id"] = net_RO.get("vim_net_id")
  64                 vld["name"] = net_RO.get("vim_name")
  65                 vld["status"] = net_RO.get("status")
  66                 vld["status-detailed"] = net_RO.get("error_msg")
  67                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  68                 break
  69             else:
  70                 raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld["id"]))
  71
  72     async def instantiate(self, nsir_id, nsilcmop_id):
  73
  74         # Try to lock HA task here
  75         task_is_locked_by_me = self.lcm_tasks.lock_HA('nsi', 'nsilcmops', nsilcmop_id)
  76         if not task_is_locked_by_me:
  77             return
  78
  79         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  80         self.logger.debug(logging_text + "Enter")
  81         # get all needed from database
  82         exc = None
  83         db_nsir = None
  84         db_nsilcmop = None
  85         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  86         db_nsilcmop_update = {}
  87         nsilcmop_operation_state = None
  88         vim_2_RO = {}
  89         RO = ROclient.ROClient(self.loop, **self.ro_config)
  90         nsi_vld_instantiationi_params = {}
  91
  92         def ip_profile_2_RO(ip_profile):
  93             RO_ip_profile = deepcopy((ip_profile))
  94             if "dns-server" in RO_ip_profile:
  95                 if isinstance(RO_ip_profile["dns-server"], list):
  96                     RO_ip_profile["dns-address"] = []
  97                     for ds in RO_ip_profile.pop("dns-server"):
  98                         RO_ip_profile["dns-address"].append(ds['address'])
  99                 else:
 100                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 101             if RO_ip_profile.get("ip-version") == "ipv4":
 102                 RO_ip_profile["ip-version"] = "IPv4"
 103             if RO_ip_profile.get("ip-version") == "ipv6":
 104                 RO_ip_profile["ip-version"] = "IPv6"
 105             if "dhcp-params" in RO_ip_profile:
 106                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 107             return RO_ip_profile
 108
 109         def vim_account_2_RO(vim_account):
 110             """
 111             Translate a RO vim_account from OSM vim_account params
 112             :param ns_params: OSM instantiate params
 113             :return: The RO ns descriptor
 114             """
 115             if vim_account in vim_2_RO:
 116                 return vim_2_RO[vim_account]
 117
 118             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 119             if db_vim["_admin"]["operationalState"] != "ENABLED":
 120                 raise LcmException("VIM={} is not available. operationalState={}".format(
 121                     vim_account, db_vim["_admin"]["operationalState"]))
 122             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 123             vim_2_RO[vim_account] = RO_vim_id
 124             return RO_vim_id
 125
 126         async def netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update):
 127             """
 128             Create a network slice VLD through RO Scenario
 129             :param vld_id The VLD id inside nsir to be created
 130             :param nsir_id The nsir id
 131             """
 132             nonlocal nsi_vld_instantiationi_params
 133             ip_vld = None
 134             mgmt_network = False
 135             RO_vld_sites = []
 136             vld_id = vld_item["id"]
 137             netslice_vld = vld_item
 138             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 139             # self.logger.debug(logging_text + "Enter")
 140
 141             vld_shared = None
 142             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 143                 _filter = {"_id.ne": nsir_id, "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item}
 144                 shared_nsi = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 145                 if shared_nsi:
 146                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 147                         if vld_id == vlds["vld_id"]:
 148                             vld_shared = {"instance_scenario_id": vlds["netslice_scenario_id"], "osm_id": vld_id}
 149                             break
 150                     break
 151
 152             # Creating netslice-vld at RO
 153             RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
 154
 155             if vld_id in RO_nsir:
 156                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 157
 158             # If netslice-vld doesn't exists then create it
 159             else:
 160                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 161                 # Updating network names and datacenters from instantiation parameters for each VLD
 162                 for instantiation_params_vld in get_iterable(db_nsir["instantiation_parameters"], "netslice-vld"):
 163                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 164                         ip_vld = deepcopy(instantiation_params_vld)
 165                         ip_vld.pop("name")
 166                         nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
 167
 168                 db_nsir_update_RO = {}
 169                 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
 170                 if self.ro_config["ng"]:
 171                     db_nsir_update_RO["netslice_scenario_id"] = vld_shared.get("instance_scenario_id") if vld_shared \
 172                         else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
 173                 else:  # if not self.ro_config["ng"]:
 174                     if netslice_vld.get("mgmt-network"):
 175                         mgmt_network = True
 176                     RO_ns_params = {}
 177                     RO_ns_params["name"] = netslice_vld["name"]
 178                     RO_ns_params["datacenter"] = vim_account_2_RO(db_nsir["instantiation_parameters"]["vimAccountId"])
 179
 180                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 181                     # Use vim-network-id instantiation parameter
 182                     vim_network_option = None
 183                     if ip_vld:
 184                         if ip_vld.get("vim-network-id"):
 185                             vim_network_option = "vim-network-id"
 186                         elif ip_vld.get("vim-network-name"):
 187                             vim_network_option = "vim-network-name"
 188                         if ip_vld.get("ip-profile"):
 189                             populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "ip-profile"),
 190                                           ip_profile_2_RO(ip_vld["ip-profile"]))
 191
 192                     if vim_network_option:
 193                         if ip_vld.get(vim_network_option):
 194                             if isinstance(ip_vld.get(vim_network_option), list):
 195                                 for vim_net_id in ip_vld.get(vim_network_option):
 196                                     for vim_account, vim_net in vim_net_id.items():
 197                                         RO_vld_sites.append({
 198                                             "netmap-use": vim_net,
 199                                             "datacenter": vim_account_2_RO(vim_account)
 200                                         })
 201                             elif isinstance(ip_vld.get(vim_network_option), dict):
 202                                 for vim_account, vim_net in ip_vld.get(vim_network_option).items():
 203                                     RO_vld_sites.append({
 204                                         "netmap-use": vim_net,
 205                                         "datacenter": vim_account_2_RO(vim_account)
 206                                     })
 207                             else:
 208                                 RO_vld_sites.append({
 209                                     "netmap-use": ip_vld[vim_network_option],
 210                                     "datacenter": vim_account_2_RO(netslice_vld["vimAccountId"])})
 211
 212                     # Use default netslice vim-network-name from template
 213                     else:
 214                         for nss_conn_point_ref in get_iterable(netslice_vld, "nss-connection-point-ref"):
 215                             if nss_conn_point_ref.get("vimAccountId"):
 216                                 if nss_conn_point_ref["vimAccountId"] != netslice_vld["vimAccountId"]:
 217                                     RO_vld_sites.append({
 218                                         "netmap-create": None,
 219                                         "datacenter": vim_account_2_RO(nss_conn_point_ref["vimAccountId"])})
 220
 221                     if vld_shared:
 222                         populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "use-network"), vld_shared)
 223
 224                     if RO_vld_sites:
 225                         populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "sites"), RO_vld_sites)
 226
 227                     RO_ns_params["scenario"] = {"nets": [{"name": netslice_vld["name"],
 228                                                 "external": mgmt_network, "type": "bridge"}]}
 229
 230                     # self.logger.debug(logging_text + step)
 231                     desc = await RO.create("ns", descriptor=RO_ns_params)
 232                     db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 233                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 234
 235         def overwrite_nsd_params(self, db_nsir, nslcmop):
 236             nonlocal nsi_vld_instantiationi_params
 237             nonlocal db_nsir_update
 238             vld_op_list = []
 239             vld = None
 240             nsr_id = nslcmop.get("nsInstanceId")
 241             # Overwrite instantiation parameters in netslice runtime
 242             RO_list = db_nsir_admin["deployed"]["RO"]
 243
 244             for ro_item_index, RO_item in enumerate(RO_list):
 245                 netslice_vld = next((n for n in get_iterable(db_nsir["_admin"], "netslice-vld")
 246                                      if RO_item.get("vld_id") == n.get("id")), None)
 247                 if not netslice_vld:
 248                     continue
 249                 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 250                 # Search the cp of netslice-vld that match with nst:netslice-subnet
 251                 for nss_cp_item in get_iterable(netslice_vld, "nss-connection-point-ref"):
 252                     # Search the netslice-subnet of nst that match
 253                     nss = next((nss for nss in get_iterable(db_nsir["_admin"], "netslice-subnet")
 254                                 if nss_cp_item["nss-ref"] == nss["nss-id"]), None)
 255                     # Compare nss-ref equal nss from nst
 256                     if not nss:
 257                         continue
 258                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 259                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 260                     for cp_nsd in db_nsds.get("sapd", ()):
 261                         if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
 262                             if nslcmop.get("operationParams"):
 263                                 if nslcmop["operationParams"].get("nsName") == nss["nsName"]:
 264                                     vld_id = RO_item["vld_id"]
 265                                     netslice_scenario_id = RO_item["netslice_scenario_id"]
 266                                     nslcmop_vld = {}
 267                                     nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
 268                                     for vld in get_iterable(nslcmop["operationParams"], "vld"):
 269                                         if vld["name"] == cp_nsd["virtual-link-desc"]:
 270                                             nslcmop_vld.update(vld)
 271                                     if self.ro_config["ng"]:
 272                                         nslcmop_vld["common_id"] = netslice_scenario_id
 273                                         nslcmop_vld.update(nsi_vld_instantiationi_params.get(RO_item["vld_id"], {}))
 274                                     else:
 275                                         nslcmop_vld["ns-net"] = {vld_id: netslice_scenario_id}
 276                                     vld_op_list.append(nslcmop_vld)
 277             nslcmop["operationParams"]["vld"] = vld_op_list
 278             self.update_db_2("nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list})
 279             return nsr_id, nslcmop
 280
 281         try:
 282             # wait for any previous tasks in process
 283             await self.lcm_tasks.waitfor_related_HA('nsi', 'nsilcmops', nsilcmop_id)
 284
 285             step = "Getting nsir={} from db".format(nsir_id)
 286             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 287             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 288             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 289
 290             start_deploy = time()
 291             nsi_params = db_nsilcmop.get("operationParams")
 292             if nsi_params and nsi_params.get("timeout_nsi_deploy"):
 293                 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
 294             else:
 295                 timeout_nsi_deploy = self.timeout.get("nsi_deploy", self.timeout_nsi_deploy)
 296
 297             # Empty list to keep track of network service records status in the netslice
 298             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 299
 300             step = "Creating slice operational-status init"
 301             # Slice status Creating
 302             db_nsir_update["detailed-status"] = "creating"
 303             db_nsir_update["operational-status"] = "init"
 304             db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 305
 306             step = "Instantiating netslice VLDs before NS instantiation"
 307             # Creating netslice VLDs networking before NS instantiation
 308             db_nsir_update["detailed-status"] = step
 309             self.update_db_2("nsis", nsir_id, db_nsir_update)
 310             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 311             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 312                 await netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update)
 313
 314             step = "Instantiating netslice subnets"
 315             db_nsir_update["detailed-status"] = step
 316             self.update_db_2("nsis", nsir_id, db_nsir_update)
 317
 318             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 319
 320             # Check status of the VLDs and wait for creation
 321             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 322             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 323             # for netslice_scenario in netslice_scenarios:
 324             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 325             #                                  nsir_id, db_nsir_update_RO)
 326
 327             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 328             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 329
 330             # Iterate over the network services operation ids to instantiate NSs
 331             step = "Instantiating Netslice Subnets"
 332             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 333             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 334             for nslcmop_id in nslcmop_ids:
 335                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 336                 # Overwriting netslice-vld vim-net-id to ns
 337                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 338                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 339                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 340                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 341
 342             # Wait until Network Slice is ready
 343             step = " Waiting nsi ready."
 344             nsrs_detailed_list_old = None
 345             self.logger.debug(logging_text + step)
 346
 347             # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
 348             while time() <= start_deploy + timeout_nsi_deploy:
 349                 # Check ns instantiation status
 350                 nsi_ready = True
 351                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 352                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 353                 nsrs_detailed_list_new = []
 354                 for nslcmop_item in nslcmop_ids:
 355                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 356                     status = nslcmop.get("operationState")
 357                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 358                     for nss in nsrs_detailed_list:
 359                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 360                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 361                                         "detailed-status": nslcmop.get("detailed-status"),
 362                                         "instantiated": True})
 363                             nsrs_detailed_list_new.append(nss)
 364                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 365                         nsi_ready = False
 366
 367                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 368                     nsrs_detailed_list_old = nsrs_detailed_list_new
 369                     self.update_db_2("nsis", nsir_id, {"_admin.nsrs-detailed-list": nsrs_detailed_list_new})
 370
 371                 if nsi_ready:
 372                     error_list = []
 373                     step = "Network Slice Instance instantiated"
 374                     for nss in nsrs_detailed_list:
 375                         if nss["status"] in ("FAILED", "FAILED_TEMP"):
 376                             error_list.append("NS {} {}: {}".format(nss["nsrId"], nss["status"],
 377                                                                     nss["detailed-status"]))
 378                     if error_list:
 379                         step = "instantiating"
 380                         raise LcmException("; ".join(error_list))
 381                     break
 382
 383                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 384                 await asyncio.sleep(5, loop=self.loop)
 385
 386             else:   # timeout_nsi_deploy reached:
 387                 raise LcmException("Timeout waiting nsi to be ready.")
 388
 389             db_nsir_update["operational-status"] = "running"
 390             db_nsir_update["detailed-status"] = "done"
 391             db_nsir_update["config-status"] = "configured"
 392             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 393             db_nsilcmop_update["statusEnteredTime"] = time()
 394             db_nsilcmop_update["detailed-status"] = "done"
 395             return
 396
 397         except (LcmException, DbException) as e:
 398             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 399             exc = e
 400         except asyncio.CancelledError:
 401             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 402             exc = "Operation was cancelled"
 403         except Exception as e:
 404             exc = traceback.format_exc()
 405             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 406                                  exc_info=True)
 407         finally:
 408             if exc:
 409                 if db_nsir:
 410                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 411                     db_nsir_update["operational-status"] = "failed"
 412                     db_nsir_update["config-status"] = "configured"
 413                 if db_nsilcmop:
 414                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 415                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 416                     db_nsilcmop_update["statusEnteredTime"] = time()
 417             try:
 418                 if db_nsir:
 419                     db_nsir_update["_admin.nsilcmop"] = None
 420                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 421                 if db_nsilcmop:
 422                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 423             except DbException as e:
 424                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 425             if nsilcmop_operation_state:
 426                 try:
 427                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 428                                                                     "operationState": nsilcmop_operation_state})
 429                 except Exception as e:
 430                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 431             self.logger.debug(logging_text + "Exit")
 432             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 433
 434     async def terminate(self, nsir_id, nsilcmop_id):
 435
 436         # Try to lock HA task here
 437         task_is_locked_by_me = self.lcm_tasks.lock_HA('nsi', 'nsilcmops', nsilcmop_id)
 438         if not task_is_locked_by_me:
 439             return
 440
 441         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 442         self.logger.debug(logging_text + "Enter")
 443         exc = None
 444         db_nsir = None
 445         db_nsilcmop = None
 446         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 447         db_nsilcmop_update = {}
 448         RO = ROclient.ROClient(self.loop, **self.ro_config)
 449         nsir_deployed = None
 450         failed_detail = []   # annotates all failed error messages
 451         nsilcmop_operation_state = None
 452         autoremove = False  # autoremove after terminated
 453         try:
 454             # wait for any previous tasks in process
 455             await self.lcm_tasks.waitfor_related_HA('nsi', 'nsilcmops', nsilcmop_id)
 456
 457             step = "Getting nsir={} from db".format(nsir_id)
 458             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 459             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 460             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 461             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 462
 463             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 464             # CASE: Instance was terminated but there is a second request to terminate the instance
 465             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 466                 return
 467
 468             # Slice status Terminating
 469             db_nsir_update["operational-status"] = "terminating"
 470             db_nsir_update["config-status"] = "terminating"
 471             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 472             self.update_db_2("nsis", nsir_id, db_nsir_update)
 473
 474             # Gets the list to keep track of network service records status in the netslice
 475             nsrs_detailed_list = []
 476
 477             # Iterate over the network services operation ids to terminate NSs
 478             # TODO: (future improvement) look another way check the tasks instead of keep asking
 479             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 480             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 481             step = "Terminating Netslice Subnets"
 482             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 483             nslcmop_new = []
 484             for nslcmop_id in nslcmop_ids:
 485                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 486                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 487                 nss_in_use = self.db.get_list("nsis", {"_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 488                                                        "operational-status": {"$nin": ["terminated", "failed"]}})
 489                 if len(nss_in_use) < 2:
 490                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 491                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 492                     nslcmop_new.append(nslcmop_id)
 493                 else:
 494                     # Update shared nslcmop shared with active nsi
 495                     netsliceInstanceId = db_nsir["_id"]
 496                     for nsis_item in nss_in_use:
 497                         if db_nsir["_id"] != nsis_item["_id"]:
 498                             netsliceInstanceId = nsis_item["_id"]
 499                             break
 500                     self.db.set_one("nslcmops", {"_id": nslcmop_id},
 501                                     {"operationParams.netsliceInstanceId": netsliceInstanceId})
 502             self.db.set_one("nsilcmops", {"_id": nsilcmop_id}, {"operationParams.nslcmops_ids": nslcmop_new})
 503
 504             # Wait until Network Slice is terminated
 505             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 506             nsrs_detailed_list_old = None
 507             self.logger.debug(logging_text + step)
 508
 509             termination_timeout = 2 * 3600   # Two hours
 510             while termination_timeout > 0:
 511                 # Check ns termination status
 512                 nsi_ready = True
 513                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 514                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 515                 nsrs_detailed_list_new = []
 516                 for nslcmop_item in nslcmop_ids:
 517                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 518                     status = nslcmop["operationState"]
 519                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 520                     for nss in nsrs_detailed_list:
 521                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 522                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 523                                         "detailed-status":
 524                                         nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 525                             nsrs_detailed_list_new.append(nss)
 526                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 527                         nsi_ready = False
 528
 529                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 530                     nsrs_detailed_list_old = nsrs_detailed_list_new
 531                     self.update_db_2("nsis", nsir_id, {"_admin.nsrs-detailed-list": nsrs_detailed_list_new})
 532
 533                 if nsi_ready:
 534                     # Check if it is the last used nss and mark isinstantiate: False
 535                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 536                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 537                     for nss in nsrs_detailed_list:
 538                         _filter = {"_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 539                                    "operational-status.ne": "terminated",
 540                                    "_id.ne": nsir_id}
 541                         nsis_list = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 542                         if not nsis_list:
 543                             nss.update({"instantiated": False})
 544
 545                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 546                     for items in nsrs_detailed_list:
 547                         if "FAILED" in items.values():
 548                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 549                     break
 550
 551                 await asyncio.sleep(5, loop=self.loop)
 552                 termination_timeout -= 5
 553
 554             if termination_timeout <= 0:
 555                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 556
 557             # Delete netslice-vlds
 558             RO_nsir_id = RO_delete_action = None
 559             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 560                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 561                 try:
 562                     if not self.ro_config["ng"]:
 563                         step = db_nsir_update["detailed-status"] = "Deleting netslice-vld at RO"
 564                         db_nsilcmop_update["detailed-status"] = "Deleting netslice-vld at RO"
 565                         self.logger.debug(logging_text + step)
 566                         desc = await RO.delete("ns", RO_nsir_id)
 567                         RO_delete_action = desc["action_id"]
 568                         nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 569                         nsir_deployed_RO["vld_status"] = "DELETING"
 570                         db_nsir_update["_admin.deployed"] = nsir_deployed
 571                         self.update_db_2("nsis", nsir_id, db_nsir_update)
 572                         if RO_delete_action:
 573                             # wait until NS is deleted from VIM
 574                             step = "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id)
 575                             self.logger.debug(logging_text + step)
 576                 except ROclient.ROClientException as e:
 577                     if e.http_code == 404:  # not found
 578                         nsir_deployed_RO["vld_id"] = None
 579                         nsir_deployed_RO["vld_status"] = "DELETED"
 580                         self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsir_id))
 581                     elif e.http_code == 409:   # conflict
 582                         failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e))
 583                         self.logger.debug(logging_text + failed_detail[-1])
 584                     else:
 585                         failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsir_id, e))
 586                         self.logger.error(logging_text + failed_detail[-1])
 587
 588                 if failed_detail:
 589                     self.logger.error(logging_text + " ;".join(failed_detail))
 590                     db_nsir_update["operational-status"] = "failed"
 591                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
 592                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 593                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 594                     db_nsilcmop_update["statusEnteredTime"] = time()
 595                 else:
 596                     db_nsir_update["operational-status"] = "terminating"
 597                     db_nsir_update["config-status"] = "terminating"
 598                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 599                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 600                     db_nsilcmop_update["statusEnteredTime"] = time()
 601                     if db_nsilcmop["operationParams"].get("autoremove"):
 602                         autoremove = True
 603
 604             db_nsir_update["detailed-status"] = "done"
 605             db_nsir_update["operational-status"] = "terminated"
 606             db_nsir_update["config-status"] = "terminated"
 607             db_nsilcmop_update["statusEnteredTime"] = time()
 608             db_nsilcmop_update["detailed-status"] = "done"
 609             return
 610
 611         except (LcmException, DbException) as e:
 612             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 613             exc = e
 614         except asyncio.CancelledError:
 615             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 616             exc = "Operation was cancelled"
 617         except Exception as e:
 618             exc = traceback.format_exc()
 619             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 620                                  exc_info=True)
 621         finally:
 622             if exc:
 623                 if db_nsir:
 624                     db_nsir_update["_admin.deployed"] = nsir_deployed
 625                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 626                     db_nsir_update["operational-status"] = "failed"
 627                 if db_nsilcmop:
 628                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 629                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 630                     db_nsilcmop_update["statusEnteredTime"] = time()
 631             try:
 632                 if db_nsir:
 633                     db_nsir_update["_admin.deployed"] = nsir_deployed
 634                     db_nsir_update["_admin.nsilcmop"] = None
 635                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 636                 if db_nsilcmop:
 637                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 638             except DbException as e:
 639                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 640
 641             if nsilcmop_operation_state:
 642                 try:
 643                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 644                                                                   "operationState": nsilcmop_operation_state,
 645                                                                   "autoremove": autoremove},
 646                                             loop=self.loop)
 647                 except Exception as e:
 648                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 649             self.logger.debug(logging_text + "Exit")
 650             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")