osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2
   3 import asyncio
   4 import logging
   5 import logging.handlers
   6 import traceback
   7 import ns
   8 from ns import populate_dict as populate_dict
   9 import ROclient
  10 from lcm_utils import LcmException, LcmBase
  11 from osm_common.dbbase import DbException
  12 from time import time
  13 from http import HTTPStatus
  14 from copy import deepcopy
  15
  16
  17 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  18
  19
  20 def get_iterable(in_dict, in_key):
  21     """
  22     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  23     :param in_dict: a dictionary
  24     :param in_key: the key to look for at in_dict
  25     :return: in_dict[in_var] or () if it is None or not present
  26     """
  27     if not in_dict.get(in_key):
  28         return ()
  29     return in_dict[in_key]
  30
  31
  32 class NetsliceLcm(LcmBase):
  33
  34     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  35
  36     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  37         """
  38         Init, Connect to database, filesystem storage, and messaging
  39         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  40         :return: None
  41         """
  42         # logging
  43         self.logger = logging.getLogger('lcm.netslice')
  44         self.loop = loop
  45         self.lcm_tasks = lcm_tasks
  46         self.ns = ns.NsLcm(db, msg, fs, lcm_tasks, ro_config, vca_config, loop)
  47         self.ro_config = ro_config
  48
  49         super().__init__(db, msg, fs, self.logger)
  50
  51     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  52         """
  53         Updates database nsir with the RO info for the created vld
  54         :param nsi_update_nsir: dictionary to be filled with the updated info
  55         :param db_nsir: content of db_nsir. This is also modified
  56         :param nsir_desc_RO: nsir descriptor from RO
  57         :return: Nothing, LcmException is raised on errors
  58         """
  59
  60         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  61             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  62                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  63                     continue
  64                 vld["vim-id"] = net_RO.get("vim_net_id")
  65                 vld["name"] = net_RO.get("vim_name")
  66                 vld["status"] = net_RO.get("status")
  67                 vld["status-detailed"] = net_RO.get("error_msg")
  68                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  69                 break
  70             else:
  71                 raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld["id"]))
  72
  73     async def instantiate(self, nsir_id, nsilcmop_id):
  74         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  75         self.logger.debug(logging_text + "Enter")
  76         # get all needed from database
  77         exc = None
  78         RO_nsir_id = None
  79         db_nsir = None
  80         db_nsilcmop = None
  81         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  82         db_nsilcmop_update = {}
  83         nsilcmop_operation_state = None
  84         vim_2_RO = {}
  85         RO = ROclient.ROClient(self.loop, **self.ro_config)
  86         start_deploy = time()
  87
  88         def vim_account_2_RO(vim_account):
  89             """
  90             Translate a RO vim_account from OSM vim_account params
  91             :param ns_params: OSM instantiate params
  92             :return: The RO ns descriptor
  93             """
  94             if vim_account in vim_2_RO:
  95                 return vim_2_RO[vim_account]
  96
  97             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
  98             if db_vim["_admin"]["operationalState"] != "ENABLED":
  99                 raise LcmException("VIM={} is not available. operationalState={}".format(
 100                     vim_account, db_vim["_admin"]["operationalState"]))
 101             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 102             vim_2_RO[vim_account] = RO_vim_id
 103             return RO_vim_id
 104
 105         try:
 106             step = "Getting nsir={} from db".format(nsir_id)
 107             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 108             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 109             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 110
 111             # look if previous tasks is in process
 112             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 113             if task_dependency:
 114                 step = db_nsilcmop_update["detailed-status"] = \
 115                     "Waiting for related tasks to be completed: {}".format(task_name)
 116                 self.logger.debug(logging_text + step)
 117                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 118                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 119                 if pending:
 120                     raise LcmException("Timeout waiting related tasks to be completed")
 121
 122             # Empty list to keep track of network service records status in the netslice
 123             nsir_admin = db_nsir["_admin"]
 124             nsir_admin["nsrs-detailed-list"] = []
 125
 126             # Slice status Creating
 127             db_nsir_update["detailed-status"] = "creating"
 128             db_nsir_update["operational-status"] = "init"
 129             self.update_db_2("nsis", nsir_id, db_nsir_update)
 130
 131             # TODO: Check Multiple VIMs networks: datacenters
 132
 133             # Creating netslice VLDs networking before NS instantiation
 134             for netslice_subnet in get_iterable(nsir_admin, "netslice-subnet"):
 135                 db_nsd = self.db.get_one("nsds", {"_id": netslice_subnet["nsdId"]})
 136
 137                 # Fist operate with VLDs inside netslice_subnet
 138                 for vld_item in get_iterable(netslice_subnet, "vld"):
 139                     RO_ns_params = {}
 140                     RO_ns_params["name"] = vld_item["name"]
 141                     RO_ns_params["datacenter"] = vim_account_2_RO(db_nsir["datacenter"])
 142
 143                     # TODO: Enable in the ns fake scenario the ip-profile
 144                     # if "ip-profile" in netslice-subnet:
 145                     #     populate_dict(RO_ns_params, ("networks", vld_params["name"], "ip-profile"),
 146                     #                 ip_profile_2_RO(vld_params["ip-profile"]))
 147                     # TODO: Check VDU type in all descriptors finding SRIOV / PT
 148                     # Updating network names and datacenters from instantiation parameters for each VLD
 149                     mgmt_network = False
 150                     for nsd_vld in get_iterable(db_nsd, "vld"):
 151                         if nsd_vld["name"] == vld_item["name"]:
 152                             if nsd_vld.get("mgmt-network"):
 153                                 mgmt_network = True
 154                                 break
 155
 156                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 157                     # Use vim-network-id instantiation parameter
 158                     vim_network_option = None
 159                     if vld_item.get("vim-network-id"):
 160                         vim_network_option = "vim-network-id"
 161                     elif vld_item.get("vim-network-name"):
 162                         vim_network_option = "vim-network-name"
 163
 164                     if vim_network_option:
 165                         RO_vld_sites = []
 166                         if vld_item.get(vim_network_option):
 167                             if isinstance(vld_item[vim_network_option], dict):
 168                                 for vim_account, vim_net in vld_item[vim_network_option].items():
 169                                     RO_vld_sites.append({
 170                                         "netmap-use": vim_net,
 171                                         "datacenter": vim_account_2_RO(vim_account)
 172                                     })
 173                             else:
 174                                 RO_vld_sites.append({"netmap-use": vld_item[vim_network_option],
 175                                                     "datacenter": vim_account_2_RO(netslice_subnet["vimAccountId"])})
 176                         if RO_vld_sites:
 177                             populate_dict(RO_ns_params, ("networks", vld_item["name"], "sites"), RO_vld_sites)
 178
 179                         if mgmt_network:
 180                             RO_ns_params["scenario"] = {"nets": [{"name": vld_item["name"],
 181                                                         "external": True, "type": "bridge"}]}
 182                         else:
 183                             RO_ns_params["scenario"] = {"nets": [{"name": vld_item["name"],
 184                                                         "external": False, "type": "bridge"}]}
 185
 186                     # Use default netslice vim-network-name from template
 187                     else:
 188                         if mgmt_network:
 189                             RO_ns_params["scenario"] = {"nets": [{"name": vld_item["name"],
 190                                                         "external": True, "type": "bridge"}]}
 191                         else:
 192                             RO_ns_params["scenario"] = {"nets": [{"name": vld_item["name"],
 193                                                         "external": False, "type": "bridge"}]}
 194
 195                     # Creating netslice-vld at RO
 196                     RO_nsir_id = db_nsir["_admin"].get("deployed", {}).get("RO", {}).get("nsir_id")
 197
 198                     # if RO vlds are present use it unless in error status
 199                     if RO_nsir_id:
 200                         try:
 201                             step = db_nsir_update["detailed-status"] = "Looking for existing ns at RO"
 202                             self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsir_id))
 203                             desc = await RO.show("ns", RO_nsir_id)
 204                         except ROclient.ROClientException as e:
 205                             if e.http_code != HTTPStatus.NOT_FOUND:
 206                                 raise
 207                             RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = None
 208                         if RO_nsir_id:
 209                             ns_status, ns_status_info = RO.check_ns_status(desc)
 210                             db_nsir_update["_admin.deployed.RO.nsir_status"] = ns_status
 211                             if ns_status == "ERROR":
 212                                 step = db_nsir_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}"\
 213                                                                            .format(RO_nsir_id)
 214                                 self.logger.debug(logging_text + step)
 215                                 await RO.delete("ns", RO_nsir_id)
 216                                 RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = None
 217
 218                     # If network doesn't exists then create it
 219                     else:
 220                         step = db_nsir_update["detailed-status"] = "Checking dependencies"
 221                         self.logger.debug(logging_text + step)
 222                         # check if VIM is creating and wait  look if previous tasks in process
 223                         # TODO: Check the case for multiple datacenters specified in instantiation parameter
 224                         for vimAccountId_unit in RO_ns_params["datacenter"]:
 225                             task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account",
 226                                                                                         vimAccountId_unit)
 227                             if task_dependency:
 228                                 step = "Waiting for related tasks to be completed: {}".format(task_name)
 229                                 self.logger.debug(logging_text + step)
 230                                 await asyncio.wait(task_dependency, timeout=3600)
 231
 232                         step = db_nsir_update["detailed-status"] = "Creating netslice-vld at RO"
 233                         desc = await RO.create("ns", descriptor=RO_ns_params)
 234                         RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = desc["uuid"]
 235                         db_nsir_update["_admin.nsState"] = "INSTANTIATED"
 236                         db_nsir_update["_admin.deployed.RO.nsir_status"] = "BUILD"
 237                         self.logger.debug(logging_text + "netslice-vld created at RO. RO_id={}".format(desc["uuid"]))
 238                         self.update_db_2("nsis", nsir_id, db_nsir_update)
 239
 240                 if RO_nsir_id:
 241                     # wait until NS scenario for netslice-vld is ready
 242                     step = ns_status_detailed = detailed_status = "Waiting netslice-vld ready at RO. RO_id={}"\
 243                                                                   .format(RO_nsir_id)
 244                     detailed_status_old = None
 245                     self.logger.debug(logging_text + step)
 246
 247                     while time() <= start_deploy + self.total_deploy_timeout:
 248                         desc = await RO.show("ns", RO_nsir_id)
 249                         ns_status, ns_status_info = RO.check_ns_status(desc)
 250                         db_nsir_update["admin.deployed.RO.nsir_status"] = ns_status
 251                         db_nsir_update["admin.deployed.RO.netslice_scenario_id"] = desc.get("uuid")
 252                         netROinfo_list = []
 253                         name = desc.get("name")
 254                         if ns_status == "ERROR":
 255                             raise ROclient.ROClientException(ns_status_info)
 256                         elif ns_status == "BUILD":
 257                             detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 258                         elif ns_status == "ACTIVE":
 259                             for nets in get_iterable(desc, "nets"):
 260                                 netROinfo = {"name": name, "vim_net_id": nets.get("vim_net_id"),
 261                                              "datacenter_id": nets.get("datacenter_id"),
 262                                              "vim_name": nets.get("vim_name")}
 263                                 netROinfo_list.append(netROinfo)
 264                             db_nsir_update["admin.deployed.RO.vim_network_info"] = netROinfo_list
 265                             self.update_db_2("nsis", nsir_id, db_nsir_update)
 266                             break
 267                         else:
 268                             assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 269                         if detailed_status != detailed_status_old:
 270                             detailed_status_old = db_nsir_update["detailed-status"] = detailed_status
 271                             self.update_db_2("nsis", nsir_id, db_nsir_update)
 272                         await asyncio.sleep(5, loop=self.loop)
 273                     else:   # total_deploy_timeout
 274                         raise ROclient.ROClientException("Timeout waiting netslice-vld to be ready")
 275
 276                     step = "Updating NSIR"
 277                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 278                     self.nsi_update_nsir(db_nsir_update, db_nsir, desc)
 279
 280             # Iterate over the network services operation ids to instantiate NSs
 281             # TODO: (future improvement) look another way check the tasks instead of keep asking
 282             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 283             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 284             # ns_tasks = []
 285
 286             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 287             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 288             for nslcmop_id in nslcmop_ids:
 289                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 290                 nsr_id = nslcmop.get("nsInstanceId")
 291                 # Overwrite instantiation parameters in netslice runtime
 292                 if db_nsir.get("admin"):
 293                     if db_nsir["admin"].get("deployed"):
 294                         db_admin_deployed_nsir = db_nsir["admin"].get("deployed")
 295                         if db_admin_deployed_nsir.get("RO"):
 296                             RO_item = db_admin_deployed_nsir["RO"]
 297                             if RO_item.get("vim_network_info"):
 298                                 for vim_network_info_item in RO_item["vim_network_info"]:
 299                                     if nslcmop.get("operationParams"):
 300                                         if nslcmop["operationParams"].get("vld"):
 301                                             for vld in nslcmop["operationParams"]["vld"]:
 302                                                 if vld["name"] == vim_network_info_item.get("name"):
 303                                                     vld["vim-network-id"] = vim_network_info_item.get("vim_net_id")
 304                                                     if vld.get("vim-network-name"):
 305                                                         del vld["vim-network-name"]
 306                                             self.update_db_2("nslcmops", nslcmop_id, nslcmop)
 307                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop)
 308                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 309                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 310
 311             # Wait until Network Slice is ready
 312             step = nsir_status_detailed = " Waiting nsi ready. nsi_id={}".format(nsir_id)
 313             nsrs_detailed_list_old = None
 314             self.logger.debug(logging_text + step)
 315
 316             # TODO: substitute while for await (all task to be done or not)
 317             deployment_timeout = 2 * 3600   # Two hours
 318             while deployment_timeout > 0:
 319                 # Check ns instantiation status
 320                 nsi_ready = True
 321                 nsrs_detailed_list = []
 322                 for nslcmop_item in nslcmop_ids:
 323                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 324                     status = nslcmop.get("operationState")
 325                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 326                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 327                                                "detailed-status":
 328                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 329                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 330                         nsi_ready = False
 331
 332                 # TODO: Check admin and _admin
 333                 if nsrs_detailed_list != nsrs_detailed_list_old:
 334                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 335                     nsrs_detailed_list_old = nsrs_detailed_list
 336                     db_nsir_update["_admin"] = nsir_admin
 337                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 338
 339                 if nsi_ready:
 340                     step = "Network Slice Instance is ready. nsi_id={}".format(nsir_id)
 341                     for items in nsrs_detailed_list:
 342                         if "FAILED" in items.values():
 343                             raise LcmException("Error deploying NSI: {}".format(nsir_id))
 344                     break
 345
 346                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 347                 await asyncio.sleep(5, loop=self.loop)
 348                 deployment_timeout -= 5
 349
 350             if deployment_timeout <= 0:
 351                 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id))
 352
 353             db_nsir_update["operational-status"] = "running"
 354             db_nsir_update["detailed-status"] = "done"
 355             db_nsir_update["config-status"] = "configured"
 356             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 357             db_nsilcmop_update["statusEnteredTime"] = time()
 358             db_nsilcmop_update["detailed-status"] = "done"
 359             return
 360
 361         except (LcmException, DbException) as e:
 362             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 363             exc = e
 364         except asyncio.CancelledError:
 365             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 366             exc = "Operation was cancelled"
 367         except Exception as e:
 368             exc = traceback.format_exc()
 369             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 370                                  exc_info=True)
 371         finally:
 372             if exc:
 373                 if db_nsir:
 374                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 375                     db_nsir_update["operational-status"] = "failed"
 376                 if db_nsilcmop:
 377                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 378                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 379                     db_nsilcmop_update["statusEnteredTime"] = time()
 380             try:
 381                 if db_nsir:
 382                     db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 383                     db_nsir_update["_admin.nsilcmop"] = None
 384                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 385                 if db_nsilcmop:
 386                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 387             except DbException as e:
 388                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 389             if nsilcmop_operation_state:
 390                 try:
 391                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 392                                                                     "operationState": nsilcmop_operation_state})
 393                 except Exception as e:
 394                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 395             self.logger.debug(logging_text + "Exit")
 396             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 397
 398     async def terminate(self, nsir_id, nsilcmop_id):
 399         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 400         self.logger.debug(logging_text + "Enter")
 401         exc = None
 402         db_nsir = None
 403         db_nsilcmop = None
 404         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 405         db_nsilcmop_update = {}
 406         RO = ROclient.ROClient(self.loop, **self.ro_config)
 407         failed_detail = []   # annotates all failed error messages
 408         nsilcmop_operation_state = None
 409         autoremove = False  # autoremove after terminated
 410         try:
 411             step = "Getting nsir={} from db".format(nsir_id)
 412             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 413             nsir_deployed = deepcopy(db_nsir["admin"].get("deployed"))
 414             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 415             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 416
 417             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 418             # CASE: Instance was terminated but there is a second request to terminate the instance
 419             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 420                 return
 421
 422             # Slice status Terminating
 423             db_nsir_update["operational-status"] = "terminating"
 424             db_nsir_update["config-status"] = "terminating"
 425             self.update_db_2("nsis", nsir_id, db_nsir_update)
 426
 427             # look if previous tasks is in process
 428             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 429             if task_dependency:
 430                 step = db_nsilcmop_update["detailed-status"] = \
 431                     "Waiting for related tasks to be completed: {}".format(task_name)
 432                 self.logger.debug(logging_text + step)
 433                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 434                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 435                 if pending:
 436                     raise LcmException("Timeout waiting related tasks to be completed")
 437
 438             # Gets the list to keep track of network service records status in the netslice
 439             nsir_admin = db_nsir["_admin"]
 440             nsrs_detailed_list = []
 441
 442             # Iterate over the network services operation ids to terminate NSs
 443             # TODO: (future improvement) look another way check the tasks instead of keep asking
 444             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 445             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 446             # ns_tasks = []
 447             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 448             for nslcmop_id in nslcmop_ids:
 449                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 450                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 451                 task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 452                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 453
 454             # Wait until Network Slice is terminated
 455             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 456             nsrs_detailed_list_old = None
 457             self.logger.debug(logging_text + step)
 458
 459             termination_timeout = 2 * 3600   # Two hours
 460             while termination_timeout > 0:
 461                 # Check ns termination status
 462                 nsi_ready = True
 463                 nsrs_detailed_list = []
 464                 for nslcmop_item in nslcmop_ids:
 465                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 466                     status = nslcmop["operationState"]
 467                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 468                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 469                                                "detailed-status":
 470                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 471                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 472                         nsi_ready = False
 473
 474                 if nsrs_detailed_list != nsrs_detailed_list_old:
 475                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 476                     nsrs_detailed_list_old = nsrs_detailed_list
 477                     db_nsir_update["_admin"] = nsir_admin
 478                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 479
 480                 if nsi_ready:
 481                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 482                     for items in nsrs_detailed_list:
 483                         if "FAILED" in items.values():
 484                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 485                     break
 486
 487                 await asyncio.sleep(5, loop=self.loop)
 488                 termination_timeout -= 5
 489
 490             if termination_timeout <= 0:
 491                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 492
 493             # Delete ns
 494             RO_nsir_id = RO_delete_action = None
 495             if nsir_deployed and nsir_deployed.get("RO"):
 496                 RO_nsir_id = nsir_deployed["RO"].get("nsr_id")
 497                 RO_delete_action = nsir_deployed["RO"].get("nsr_delete_action_id")
 498             try:
 499                 if RO_nsir_id:
 500                     step = db_nsir_update["detailed-status"] = "Deleting ns at RO"
 501                     db_nsilcmop_update["detailed-status"] = "Deleting ns at RO"
 502                     self.logger.debug(logging_text + step)
 503                     desc = await RO.delete("ns", RO_nsir_id)
 504                     RO_delete_action = desc["action_id"]
 505                     db_nsir_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
 506                     db_nsir_update["_admin.deployed.RO.nsr_id"] = None
 507                     db_nsir_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 508                 if RO_delete_action:
 509                     # wait until NS is deleted from VIM
 510                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id)
 511                     detailed_status_old = None
 512                     self.logger.debug(logging_text + step)
 513
 514                     delete_timeout = 20 * 60   # 20 minutes
 515                     while delete_timeout > 0:
 516                         desc = await RO.show("ns", item_id_name=RO_nsir_id, extra_item="action",
 517                                              extra_item_id=RO_delete_action)
 518                         ns_status, ns_status_info = RO.check_action_status(desc)
 519                         if ns_status == "ERROR":
 520                             raise ROclient.ROClientException(ns_status_info)
 521                         elif ns_status == "BUILD":
 522                             detailed_status = step + "; {}".format(ns_status_info)
 523                         elif ns_status == "ACTIVE":
 524                             break
 525                         else:
 526                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
 527                         await asyncio.sleep(5, loop=self.loop)
 528                         delete_timeout -= 5
 529                         if detailed_status != detailed_status_old:
 530                             detailed_status_old = db_nsilcmop_update["detailed-status"] = detailed_status
 531                             self.update_db_2("nslcmops", nslcmop_id, db_nsilcmop_update)
 532                     else:  # delete_timeout <= 0:
 533                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
 534
 535             except ROclient.ROClientException as e:
 536                 if e.http_code == 404:  # not found
 537                     db_nsir_update["_admin.deployed.RO.nsr_id"] = None
 538                     db_nsir_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 539                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsir_id))
 540                 elif e.http_code == 409:   # conflict
 541                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e))
 542                     self.logger.debug(logging_text + failed_detail[-1])
 543                 else:
 544                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsir_id, e))
 545                     self.logger.error(logging_text + failed_detail[-1])
 546
 547             if failed_detail:
 548                 self.logger.error(logging_text + " ;".join(failed_detail))
 549                 db_nsir_update["operational-status"] = "failed"
 550                 db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
 551                 db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 552                 db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 553                 db_nsilcmop_update["statusEnteredTime"] = time()
 554             else:
 555                 db_nsir_update["operational-status"] = "terminated"
 556                 db_nsir_update["detailed-status"] = "done"
 557                 db_nsir_update["config-status"] = "configured"
 558                 db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 559                 db_nsilcmop_update["detailed-status"] = "Done"
 560                 db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 561                 db_nsilcmop_update["statusEnteredTime"] = time()
 562                 if db_nsilcmop["operationParams"].get("autoremove"):
 563                     autoremove = True
 564
 565             db_nsir_update["operational-status"] = "terminated"
 566             db_nsir_update["config-status"] = "configured"
 567             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 568             db_nsilcmop_update["statusEnteredTime"] = time()
 569             db_nsilcmop_update["detailed-status"] = "done"
 570             return
 571
 572         except (LcmException, DbException) as e:
 573             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 574             exc = e
 575         except asyncio.CancelledError:
 576             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 577             exc = "Operation was cancelled"
 578         except Exception as e:
 579             exc = traceback.format_exc()
 580             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 581                                  exc_info=True)
 582         finally:
 583             if exc:
 584                 if db_nsir:
 585                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 586                     db_nsir_update["operational-status"] = "failed"
 587                 if db_nsilcmop:
 588                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 589                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 590                     db_nsilcmop_update["statusEnteredTime"] = time()
 591             try:
 592                 if db_nsir:
 593                     db_nsir_update["_admin.nsilcmop"] = None
 594                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 595                 if db_nsilcmop:
 596                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 597             except DbException as e:
 598                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 599
 600             if nsilcmop_operation_state:
 601                 try:
 602                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 603                                                                   "operationState": nsilcmop_operation_state,
 604                                                                   "autoremove": autoremove},
 605                                             loop=self.loop)
 606                 except Exception as e:
 607                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 608             self.logger.debug(logging_text + "Exit")
 609             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")