osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2
   3 import asyncio
   4 import logging
   5 import logging.handlers
   6 import traceback
   7 import ns
   8 import ROclient
   9 from lcm_utils import LcmException, LcmBase
  10 from osm_common.dbbase import DbException
  11 from time import time
  12 from http import HTTPStatus
  13 from copy import deepcopy
  14
  15
  16 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  17
  18
  19 def get_iterable(in_dict, in_key):
  20     """
  21     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  22     :param in_dict: a dictionary
  23     :param in_key: the key to look for at in_dict
  24     :return: in_dict[in_var] or () if it is None or not present
  25     """
  26     if not in_dict.get(in_key):
  27         return ()
  28     return in_dict[in_key]
  29
  30
  31 class NetsliceLcm(LcmBase):
  32
  33     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  34
  35     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  36         """
  37         Init, Connect to database, filesystem storage, and messaging
  38         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  39         :return: None
  40         """
  41         # logging
  42         self.logger = logging.getLogger('lcm.netslice')
  43         self.loop = loop
  44         self.lcm_tasks = lcm_tasks
  45         self.ns = ns.NsLcm(db, msg, fs, lcm_tasks, ro_config, vca_config, loop)
  46         self.ro_config = ro_config
  47
  48         super().__init__(db, msg, fs, self.logger)
  49
  50     async def instantiate(self, nsir_id, nsilcmop_id):
  51         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  52         self.logger.debug(logging_text + "Enter")
  53         # get all needed from database
  54         exc = None
  55         db_nsir = None
  56         db_nsilcmop = None
  57         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  58         db_nsilcmop_update = {}
  59         nsilcmop_operation_state = None
  60         vim_2_RO = {}
  61         RO = ROclient.ROClient(self.loop, **self.ro_config)
  62         start_deploy = time()
  63
  64         def vim_account_2_RO(vim_account):
  65             """
  66             Translate a RO vim_account from OSM vim_account params
  67             :param ns_params: OSM instantiate params
  68             :return: The RO ns descriptor
  69             """
  70             if vim_account in vim_2_RO:
  71                 return vim_2_RO[vim_account]
  72
  73             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
  74             if db_vim["_admin"]["operationalState"] != "ENABLED":
  75                 raise LcmException("VIM={} is not available. operationalState={}".format(
  76                     vim_account, db_vim["_admin"]["operationalState"]))
  77             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
  78             vim_2_RO[vim_account] = RO_vim_id
  79             return RO_vim_id
  80
  81         def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  82             """
  83             Updates database nsir with the RO info for the created vld
  84             :param nsi_update_nsir: dictionary to be filled with the updated info
  85             :param db_nsir: content of db_nsir. This is also modified
  86             :param nsir_desc_RO: nsir descriptor from RO
  87             :return: Nothing, LcmException is raised on errors
  88             """
  89
  90             for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  91                 for net_RO in get_iterable(nsir_desc_RO, "nets"):
  92                     if vld["id"] != net_RO.get("ns_net_osm_id"):
  93                         continue
  94                     vld["vim-id"] = net_RO.get("vim_net_id")
  95                     vld["name"] = net_RO.get("vim_name")
  96                     vld["status"] = net_RO.get("status")
  97                     vld["status-detailed"] = net_RO.get("error_msg")
  98                     nsi_update_nsir["vld.{}".format(vld_index)] = vld
  99                     break
 100                 else:
 101                     raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld["id"]))
 102
 103         try:
 104             step = "Getting nsir={} from db".format(nsir_id)
 105             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 106             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 107             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 108
 109             # look if previous tasks is in process
 110             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 111             if task_dependency:
 112                 step = db_nsilcmop_update["detailed-status"] = \
 113                     "Waiting for related tasks to be completed: {}".format(task_name)
 114                 self.logger.debug(logging_text + step)
 115                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 116                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 117                 if pending:
 118                     raise LcmException("Timeout waiting related tasks to be completed")
 119
 120             # Empty list to keep track of network service records status in the netslice
 121             nsir_admin = db_nsir["_admin"]
 122             nsir_vlds = nsir_admin["netslice-vld"]
 123             nsir_admin["nsrs-detailed-list"] = []
 124
 125             # Slice status Creating
 126             db_nsir_update["detailed-status"] = "creating"
 127             db_nsir_update["operational-status"] = "init"
 128             self.update_db_2("nsis", nsir_id, db_nsir_update)
 129
 130             # TODO: Multiple VIMs networks: datacenters
 131             nets = []
 132             for netslice_vld in nsir_vlds:
 133                 RO_ns_params = {}
 134                 RO_ns_params["name"] = db_nsir["name"]
 135                 if netslice_vld.get("vim-network-name"):
 136                     if not isinstance(netslice_vld["vim-network-name"], str):
 137                         name = netslice_vld["vim-network-name"][netslice_vld["vimAccountId"]]
 138                         nets.append({"name": name, "external": False, "type": "bridge"})
 139                     else:
 140                         nets.append({"name": netslice_vld["vim-network-name"], "external": False, "type": "bridge"})
 141                 else:
 142                     nets.append({"name": "default", "external": False, "type": "bridge"})
 143                 RO_ns_params["scenario"] = {"nets": nets}
 144                 RO_ns_params["datacenter"] = vim_account_2_RO(netslice_vld["vimAccountId"])
 145
 146             # Crate ns at RO
 147             # if present use it unless in error status
 148             RO_nsir_id = db_nsir["_admin"].get("deployed", {}).get("RO", {}).get("nsir_id")
 149             if RO_nsir_id:
 150                 try:
 151                     step = db_nsir_update["detailed-status"] = "Looking for existing ns at RO"
 152                     # self.logger.debug(logging_text + step + " RO_ns_id={}".format(RO_nsir_id))
 153                     desc = await RO.show("ns", RO_nsir_id)
 154                 except ROclient.ROClientException as e:
 155                     if e.http_code != HTTPStatus.NOT_FOUND:
 156                         raise
 157                     RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = None
 158                 if RO_nsir_id:
 159                     ns_status, ns_status_info = RO.check_ns_status(desc)
 160                     db_nsir_update["_admin.deployed.RO.nsir_status"] = ns_status
 161                     if ns_status == "ERROR":
 162                         step = db_nsir_update["detailed-status"] = "Deleting ns at RO. RO_ns_id={}".format(RO_nsir_id)
 163                         self.logger.debug(logging_text + step)
 164                         await RO.delete("ns", RO_nsir_id)
 165                         RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = None
 166             if not RO_nsir_id:
 167                 step = db_nsir_update["detailed-status"] = "Checking dependencies"
 168                 # self.logger.debug(logging_text + step)
 169                 # check if VIM is creating and wait  look if previous tasks in process
 170                 for vimAccountId_unit in RO_ns_params["datacenter"]:
 171                     task_name, task_dependency = self.lcm_tasks.lookfor_related("vim_account", vimAccountId_unit)
 172                     if task_dependency:
 173                         step = "Waiting for related tasks to be completed: {}".format(task_name)
 174                         self.logger.debug(logging_text + step)
 175                         await asyncio.wait(task_dependency, timeout=3600)
 176
 177                 step = db_nsir_update["detailed-status"] = "Creating ns at RO"
 178                 desc = await RO.create("ns", descriptor=RO_ns_params)
 179                 RO_nsir_id = db_nsir_update["_admin.deployed.RO.nsir_id"] = desc["uuid"]
 180                 db_nsir_update["_admin.nsState"] = "INSTANTIATED"
 181                 db_nsir_update["_admin.deployed.RO.nsir_status"] = "BUILD"
 182                 self.logger.debug(logging_text + "ns created at RO. RO_id={}".format(desc["uuid"]))
 183             self.update_db_2("nsis", nsir_id, db_nsir_update)
 184
 185             # wait until NS scenario for netslice-vld is ready
 186             step = ns_status_detailed = detailed_status = "Waiting ns ready at RO. RO_id={}".format(RO_nsir_id)
 187             detailed_status_old = None
 188             self.logger.debug(logging_text + step)
 189
 190             while time() <= start_deploy + self.total_deploy_timeout:
 191                 desc = await RO.show("ns", RO_nsir_id)
 192                 ns_status, ns_status_info = RO.check_ns_status(desc)
 193                 db_nsir_update["admin.deployed.RO.nsir_status"] = ns_status
 194                 db_nsir_update["admin.deployed.RO.nsr_id"] = desc.get("uuid")
 195                 if ns_status == "ERROR":
 196                     raise ROclient.ROClientException(ns_status_info)
 197                 elif ns_status == "BUILD":
 198                     detailed_status = ns_status_detailed + "; {}".format(ns_status_info)
 199                 elif ns_status == "ACTIVE":
 200                     break
 201                 else:
 202                     assert False, "ROclient.check_ns_status returns unknown {}".format(ns_status)
 203                 if detailed_status != detailed_status_old:
 204                     detailed_status_old = db_nsir_update["detailed-status"] = detailed_status
 205                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 206                 await asyncio.sleep(5, loop=self.loop)
 207             else:   # total_deploy_timeout
 208                 raise ROclient.ROClientException("Timeout waiting ns to be ready")
 209
 210             step = "Updating NSIR"
 211             nsi_update_nsir(self, db_nsir_update, db_nsir, desc)
 212
 213             # Iterate over the network services operation ids to instantiate NSs
 214             # TODO: (future improvement) look another way check the tasks instead of keep asking
 215             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 216             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 217             # ns_tasks = []
 218             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 219             for nslcmop_id in nslcmop_ids:
 220                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 221                 nsr_id = nslcmop.get("nsInstanceId")
 222                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop)
 223                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 224                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 225
 226             # Wait until Network Slice is ready
 227             step = nsir_status_detailed = " Waiting nsi ready. nsi_id={}".format(nsir_id)
 228             nsrs_detailed_list_old = None
 229             self.logger.debug(logging_text + step)
 230
 231             # TODO: substitute while for await (all task to be done or not)
 232             deployment_timeout = 2 * 3600   # Two hours
 233             while deployment_timeout > 0:
 234                 # Check ns instantiation status
 235                 nsi_ready = True
 236                 nsrs_detailed_list = []
 237                 for nslcmop_item in nslcmop_ids:
 238                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 239                     status = nslcmop.get("operationState")
 240                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 241                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 242                                                "detailed-status":
 243                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 244                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 245                         nsi_ready = False
 246
 247                 # TODO: Check admin and _admin
 248                 if nsrs_detailed_list != nsrs_detailed_list_old:
 249                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 250                     nsrs_detailed_list_old = nsrs_detailed_list
 251                     db_nsir_update["_admin"] = nsir_admin
 252                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 253
 254                 if nsi_ready:
 255                     step = "Network Slice Instance is ready. nsi_id={}".format(nsir_id)
 256                     for items in nsrs_detailed_list:
 257                         if "FAILED" in items.values():
 258                             raise LcmException("Error deploying NSI: {}".format(nsir_id))
 259                     break
 260
 261                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 262                 await asyncio.sleep(5, loop=self.loop)
 263                 deployment_timeout -= 5
 264
 265             if deployment_timeout <= 0:
 266                 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id))
 267
 268             db_nsir_update["operational-status"] = "running"
 269             db_nsir_update["detailed-status"] = "done"
 270             db_nsir_update["config-status"] = "configured"
 271             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 272             db_nsilcmop_update["statusEnteredTime"] = time()
 273             db_nsilcmop_update["detailed-status"] = "done"
 274             return
 275
 276         except (LcmException, DbException) as e:
 277             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 278             exc = e
 279         except asyncio.CancelledError:
 280             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 281             exc = "Operation was cancelled"
 282         except Exception as e:
 283             exc = traceback.format_exc()
 284             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 285                                  exc_info=True)
 286         finally:
 287             if exc:
 288                 if db_nsir:
 289                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 290                     db_nsir_update["operational-status"] = "failed"
 291                 if db_nsilcmop:
 292                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 293                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 294                     db_nsilcmop_update["statusEnteredTime"] = time()
 295             if db_nsir:
 296                 db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 297                 db_nsir_update["_admin.nsilcmop"] = None
 298                 self.update_db_2("nsis", nsir_id, db_nsir_update)
 299             if db_nsilcmop:
 300
 301                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 302             if nsilcmop_operation_state:
 303                 try:
 304                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 305                                                                     "operationState": nsilcmop_operation_state})
 306                 except Exception as e:
 307                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 308             self.logger.debug(logging_text + "Exit")
 309             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 310
 311     async def terminate(self, nsir_id, nsilcmop_id):
 312         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 313         self.logger.debug(logging_text + "Enter")
 314         exc = None
 315         db_nsir = None
 316         db_nsilcmop = None
 317         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 318         db_nsilcmop_update = {}
 319         RO = ROclient.ROClient(self.loop, **self.ro_config)
 320         failed_detail = []   # annotates all failed error messages
 321         nsilcmop_operation_state = None
 322         try:
 323             step = "Getting nsir={} from db".format(nsir_id)
 324             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 325             nsir_deployed = deepcopy(db_nsir["admin"].get("deployed"))
 326             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 327             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 328
 329             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 330             # CASE: Instance was terminated but there is a second request to terminate the instance
 331             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 332                 return
 333
 334             # Slice status Terminating
 335             db_nsir_update["operational-status"] = "terminating"
 336             db_nsir_update["config-status"] = "terminating"
 337             self.update_db_2("nsis", nsir_id, db_nsir_update)
 338
 339             # look if previous tasks is in process
 340             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 341             if task_dependency:
 342                 step = db_nsilcmop_update["detailed-status"] = \
 343                     "Waiting for related tasks to be completed: {}".format(task_name)
 344                 self.logger.debug(logging_text + step)
 345                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 346                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 347                 if pending:
 348                     raise LcmException("Timeout waiting related tasks to be completed")
 349
 350             # Gets the list to keep track of network service records status in the netslice
 351             nsir_admin = db_nsir["_admin"]
 352             nsrs_detailed_list = []
 353
 354             # Iterate over the network services operation ids to terminate NSs
 355             # TODO: (future improvement) look another way check the tasks instead of keep asking
 356             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 357             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 358             # ns_tasks = []
 359             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 360             for nslcmop_id in nslcmop_ids:
 361                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 362                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 363                 task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 364                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 365
 366             # Wait until Network Slice is terminated
 367             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 368             nsrs_detailed_list_old = None
 369             self.logger.debug(logging_text + step)
 370
 371             termination_timeout = 2 * 3600   # Two hours
 372             while termination_timeout > 0:
 373                 # Check ns termination status
 374                 nsi_ready = True
 375                 nsrs_detailed_list = []
 376                 for nslcmop_item in nslcmop_ids:
 377                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 378                     status = nslcmop["operationState"]
 379                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 380                     nsrs_detailed_list.append({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 381                                                "detailed-status":
 382                                                nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 383                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 384                         nsi_ready = False
 385
 386                 if nsrs_detailed_list != nsrs_detailed_list_old:
 387                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list
 388                     nsrs_detailed_list_old = nsrs_detailed_list
 389                     db_nsir_update["_admin"] = nsir_admin
 390                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 391
 392                 if nsi_ready:
 393                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 394                     for items in nsrs_detailed_list:
 395                         if "FAILED" in items.values():
 396                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 397                     break
 398
 399                 await asyncio.sleep(5, loop=self.loop)
 400                 termination_timeout -= 5
 401
 402             if termination_timeout <= 0:
 403                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 404
 405             # Delete ns
 406             RO_nsir_id = RO_delete_action = None
 407             if nsir_deployed and nsir_deployed.get("RO"):
 408                 RO_nsir_id = nsir_deployed["RO"].get("nsr_id")
 409                 RO_delete_action = nsir_deployed["RO"].get("nsr_delete_action_id")
 410             try:
 411                 if RO_nsir_id:
 412                     step = db_nsir_update["detailed-status"] = "Deleting ns at RO"
 413                     db_nsilcmop_update["detailed-status"] = "Deleting ns at RO"
 414                     self.logger.debug(logging_text + step)
 415                     desc = await RO.delete("ns", RO_nsir_id)
 416                     RO_delete_action = desc["action_id"]
 417                     db_nsir_update["_admin.deployed.RO.nsr_delete_action_id"] = RO_delete_action
 418                     db_nsir_update["_admin.deployed.RO.nsr_id"] = None
 419                     db_nsir_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 420                 if RO_delete_action:
 421                     # wait until NS is deleted from VIM
 422                     step = detailed_status = "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id)
 423                     detailed_status_old = None
 424                     self.logger.debug(logging_text + step)
 425
 426                     delete_timeout = 20 * 60   # 20 minutes
 427                     while delete_timeout > 0:
 428                         desc = await RO.show("ns", item_id_name=RO_nsir_id, extra_item="action",
 429                                              extra_item_id=RO_delete_action)
 430                         ns_status, ns_status_info = RO.check_action_status(desc)
 431                         if ns_status == "ERROR":
 432                             raise ROclient.ROClientException(ns_status_info)
 433                         elif ns_status == "BUILD":
 434                             detailed_status = step + "; {}".format(ns_status_info)
 435                         elif ns_status == "ACTIVE":
 436                             break
 437                         else:
 438                             assert False, "ROclient.check_action_status returns unknown {}".format(ns_status)
 439                         await asyncio.sleep(5, loop=self.loop)
 440                         delete_timeout -= 5
 441                         if detailed_status != detailed_status_old:
 442                             detailed_status_old = db_nsilcmop_update["detailed-status"] = detailed_status
 443                             self.update_db_2("nslcmops", nslcmop_id, db_nsilcmop_update)
 444                     else:  # delete_timeout <= 0:
 445                         raise ROclient.ROClientException("Timeout waiting ns deleted from VIM")
 446
 447             except ROclient.ROClientException as e:
 448                 if e.http_code == 404:  # not found
 449                     db_nsir_update["_admin.deployed.RO.nsr_id"] = None
 450                     db_nsir_update["_admin.deployed.RO.nsr_status"] = "DELETED"
 451                     self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsir_id))
 452                 elif e.http_code == 409:   # conflict
 453                     failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e))
 454                     self.logger.debug(logging_text + failed_detail[-1])
 455                 else:
 456                     failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsir_id, e))
 457                     self.logger.error(logging_text + failed_detail[-1])
 458
 459             db_nsir_update["operational-status"] = "terminated"
 460             db_nsir_update["config-status"] = "configured"
 461             db_nsir_update["detailed-status"] = "done"
 462             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 463             db_nsilcmop_update["statusEnteredTime"] = time()
 464             db_nsilcmop_update["detailed-status"] = "done"
 465             return
 466
 467         except (LcmException, DbException) as e:
 468             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 469             exc = e
 470         except asyncio.CancelledError:
 471             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 472             exc = "Operation was cancelled"
 473         except Exception as e:
 474             exc = traceback.format_exc()
 475             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 476                                  exc_info=True)
 477         finally:
 478             if exc:
 479                 if db_nsir:
 480                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 481                     db_nsir_update["operational-status"] = "failed"
 482                 if db_nsilcmop:
 483                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 484                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 485                     db_nsilcmop_update["statusEnteredTime"] = time()
 486             if db_nsir:
 487                 db_nsir_update["_admin.nsilcmop"] = None
 488                 db_nsir_update["_admin.nsiState"] = "TERMINATED"
 489                 self.update_db_2("nsis", nsir_id, db_nsir_update)
 490             if db_nsilcmop:
 491                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 492
 493             if nsilcmop_operation_state:
 494                 try:
 495                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 496                                                                   "operationState": nsilcmop_operation_state})
 497                 except Exception as e:
 498                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 499             self.logger.debug(logging_text + "Exit")
 500             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")