osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import asyncio
  17 import logging
  18 import logging.handlers
  19 import traceback
  20 from osm_lcm.ns import populate_dict as populate_dict
  21 from osm_lcm import ROclient, ns
  22 from osm_lcm.lcm_utils import LcmException, LcmBase
  23 from osm_common.dbbase import DbException
  24 from time import time
  25 from copy import deepcopy
  26
  27
  28 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  29
  30
  31 def get_iterable(in_dict, in_key):
  32     """
  33     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  34     :param in_dict: a dictionary
  35     :param in_key: the key to look for at in_dict
  36     :return: in_dict[in_var] or () if it is None or not present
  37     """
  38     if not in_dict.get(in_key):
  39         return ()
  40     return in_dict[in_key]
  41
  42
  43 class NetsliceLcm(LcmBase):
  44
  45     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  46
  47     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  48         """
  49         Init, Connect to database, filesystem storage, and messaging
  50         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  51         :return: None
  52         """
  53         # logging
  54         self.logger = logging.getLogger('lcm.netslice')
  55         self.loop = loop
  56         self.lcm_tasks = lcm_tasks
  57         self.ns = ns.NsLcm(db, msg, fs, lcm_tasks, ro_config, vca_config, loop)
  58         self.ro_config = ro_config
  59
  60         super().__init__(db, msg, fs, self.logger)
  61
  62     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  63         """
  64         Updates database nsir with the RO info for the created vld
  65         :param nsi_update_nsir: dictionary to be filled with the updated info
  66         :param db_nsir: content of db_nsir. This is also modified
  67         :param nsir_desc_RO: nsir descriptor from RO
  68         :return: Nothing, LcmException is raised on errors
  69         """
  70
  71         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  72             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  73                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  74                     continue
  75                 vld["vim-id"] = net_RO.get("vim_net_id")
  76                 vld["name"] = net_RO.get("vim_name")
  77                 vld["status"] = net_RO.get("status")
  78                 vld["status-detailed"] = net_RO.get("error_msg")
  79                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  80                 break
  81             else:
  82                 raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld["id"]))
  83
  84     async def instantiate(self, nsir_id, nsilcmop_id):
  85
  86         # Try to lock HA task here
  87         task_is_locked_by_me = self.lcm_tasks.lock_HA('nsi', 'nsilcmops', nsilcmop_id)
  88         if not task_is_locked_by_me:
  89             return
  90
  91         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  92         self.logger.debug(logging_text + "Enter")
  93         # get all needed from database
  94         exc = None
  95         db_nsir = None
  96         db_nsilcmop = None
  97         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  98         db_nsilcmop_update = {}
  99         nsilcmop_operation_state = None
 100         vim_2_RO = {}
 101         RO = ROclient.ROClient(self.loop, **self.ro_config)
 102
 103         def ip_profile_2_RO(ip_profile):
 104             RO_ip_profile = deepcopy((ip_profile))
 105             if "dns-server" in RO_ip_profile:
 106                 if isinstance(RO_ip_profile["dns-server"], list):
 107                     RO_ip_profile["dns-address"] = []
 108                     for ds in RO_ip_profile.pop("dns-server"):
 109                         RO_ip_profile["dns-address"].append(ds['address'])
 110                 else:
 111                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 112             if RO_ip_profile.get("ip-version") == "ipv4":
 113                 RO_ip_profile["ip-version"] = "IPv4"
 114             if RO_ip_profile.get("ip-version") == "ipv6":
 115                 RO_ip_profile["ip-version"] = "IPv6"
 116             if "dhcp-params" in RO_ip_profile:
 117                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 118             return RO_ip_profile
 119
 120         def vim_account_2_RO(vim_account):
 121             """
 122             Translate a RO vim_account from OSM vim_account params
 123             :param ns_params: OSM instantiate params
 124             :return: The RO ns descriptor
 125             """
 126             if vim_account in vim_2_RO:
 127                 return vim_2_RO[vim_account]
 128
 129             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 130             if db_vim["_admin"]["operationalState"] != "ENABLED":
 131                 raise LcmException("VIM={} is not available. operationalState={}".format(
 132                     vim_account, db_vim["_admin"]["operationalState"]))
 133             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 134             vim_2_RO[vim_account] = RO_vim_id
 135             return RO_vim_id
 136
 137         async def netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update):
 138             """
 139             Create a network slice VLD through RO Scenario
 140             :param vld_id The VLD id inside nsir to be created
 141             :param nsir_id The nsir id
 142             """
 143             ip_vld = None
 144             mgmt_network = False
 145             RO_vld_sites = []
 146             vld_id = vld_item["id"]
 147             netslice_vld = vld_item
 148             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 149             # self.logger.debug(logging_text + "Enter")
 150
 151             vld_shared = None
 152             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 153                 _filter = {"_id.ne": nsir_id, "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item}
 154                 shared_nsi = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 155                 if shared_nsi:
 156                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 157                         if vld_id == vlds["vld_id"]:
 158                             vld_shared = {"instance_scenario_id": vlds["netslice_scenario_id"], "osm_id": vld_id}
 159                             break
 160                     break
 161
 162             # Creating netslice-vld at RO
 163             RO_nsir = db_nsir["_admin"].get("deployed", {}).get("RO", [])
 164
 165             if vld_id in RO_nsir:
 166                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 167
 168             # If netslice-vld doesn't exists then create it
 169             else:
 170                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 171                 # Updating network names and datacenters from instantiation parameters for each VLD
 172                 RO_ns_params = {}
 173                 RO_ns_params["name"] = netslice_vld["name"]
 174                 RO_ns_params["datacenter"] = vim_account_2_RO(db_nsir["instantiation_parameters"]["vimAccountId"])
 175                 for instantiation_params_vld in get_iterable(db_nsir["instantiation_parameters"], "netslice-vld"):
 176                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 177                         ip_vld = deepcopy(instantiation_params_vld)
 178
 179                 if netslice_vld.get("mgmt-network"):
 180                     mgmt_network = True
 181
 182                 # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 183                 # Use vim-network-id instantiation parameter
 184                 vim_network_option = None
 185                 if ip_vld:
 186                     if ip_vld.get("vim-network-id"):
 187                         vim_network_option = "vim-network-id"
 188                     elif ip_vld.get("vim-network-name"):
 189                         vim_network_option = "vim-network-name"
 190                     if ip_vld.get("ip-profile"):
 191                         populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "ip-profile"),
 192                                       ip_profile_2_RO(ip_vld["ip-profile"]))
 193
 194                 if vim_network_option:
 195                     if ip_vld.get(vim_network_option):
 196                         if isinstance(ip_vld.get(vim_network_option), list):
 197                             for vim_net_id in ip_vld.get(vim_network_option):
 198                                 for vim_account, vim_net in vim_net_id.items():
 199                                     RO_vld_sites.append({
 200                                         "netmap-use": vim_net,
 201                                         "datacenter": vim_account_2_RO(vim_account)
 202                                     })
 203                         elif isinstance(ip_vld.get(vim_network_option), dict):
 204                             for vim_account, vim_net in ip_vld.get(vim_network_option).items():
 205                                 RO_vld_sites.append({
 206                                     "netmap-use": vim_net,
 207                                     "datacenter": vim_account_2_RO(vim_account)
 208                                 })
 209                         else:
 210                             RO_vld_sites.append({
 211                                 "netmap-use": ip_vld[vim_network_option],
 212                                 "datacenter": vim_account_2_RO(netslice_vld["vimAccountId"])})
 213
 214                 # Use default netslice vim-network-name from template
 215                 else:
 216                     for nss_conn_point_ref in get_iterable(netslice_vld, "nss-connection-point-ref"):
 217                         if nss_conn_point_ref.get("vimAccountId"):
 218                             if nss_conn_point_ref["vimAccountId"] != netslice_vld["vimAccountId"]:
 219                                 RO_vld_sites.append({
 220                                     "netmap-create": None,
 221                                     "datacenter": vim_account_2_RO(nss_conn_point_ref["vimAccountId"])})
 222
 223                 if vld_shared:
 224                     populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "use-network"), vld_shared)
 225
 226                 if RO_vld_sites:
 227                     populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "sites"), RO_vld_sites)
 228
 229                 RO_ns_params["scenario"] = {"nets": [{"name": netslice_vld["name"],
 230                                             "external": mgmt_network, "type": "bridge"}]}
 231
 232                 # self.logger.debug(logging_text + step)
 233                 db_nsir_update["detailed-status"] = "Creating netslice-vld at RO"
 234                 desc = await RO.create("ns", descriptor=RO_ns_params)
 235                 db_nsir_update_RO = {}
 236                 db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 237                 db_nsir_update_RO["vld_id"] = RO_ns_params["name"]
 238                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 239
 240         def overwrite_nsd_params(self, db_nsir, nslcmop):
 241             RO_list = []
 242             vld_op_list = []
 243             vld = None
 244             nsr_id = nslcmop.get("nsInstanceId")
 245             # Overwrite instantiation parameters in netslice runtime
 246             if db_nsir.get("_admin"):
 247                 if db_nsir["_admin"].get("deployed"):
 248                     db_admin_deployed_nsir = db_nsir["_admin"].get("deployed")
 249                     if db_admin_deployed_nsir.get("RO"):
 250                         RO_list = db_admin_deployed_nsir["RO"]
 251
 252             for RO_item in RO_list:
 253                 for netslice_vld in get_iterable(db_nsir["_admin"], "netslice-vld"):
 254                     # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 255                     if RO_item.get("vld_id") == netslice_vld.get("id"):
 256                         # Search the cp of netslice-vld that match with nst:netslice-subnet
 257                         for nss_cp_item in get_iterable(netslice_vld, "nss-connection-point-ref"):
 258                             # Search the netslice-subnet of nst that match
 259                             for nss in get_iterable(db_nsir["_admin"], "netslice-subnet"):
 260                                 # Compare nss-ref equal nss from nst
 261                                 if nss_cp_item["nss-ref"] == nss["nss-id"]:
 262                                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 263                                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 264                                     for cp_nsd in db_nsds["connection-point"]:
 265                                         if cp_nsd["name"] == nss_cp_item["nsd-connection-point-ref"]:
 266                                             if nslcmop.get("operationParams"):
 267                                                 if nslcmop["operationParams"].get("nsName") == nss["nsName"]:
 268                                                     vld_id = RO_item["vld_id"]
 269                                                     netslice_scenario_id = RO_item["netslice_scenario_id"]
 270                                                     nslcmop_vld = {}
 271                                                     nslcmop_vld["ns-net"] = {vld_id: netslice_scenario_id}
 272                                                     nslcmop_vld["name"] = cp_nsd["vld-id-ref"]
 273                                                     for vld in get_iterable(nslcmop["operationParams"], "vld"):
 274                                                         if vld["name"] == cp_nsd["vld-id-ref"]:
 275                                                             nslcmop_vld.update(vld)
 276                                                     vld_op_list.append(nslcmop_vld)
 277             nslcmop["operationParams"]["vld"] = vld_op_list
 278             self.update_db_2("nslcmops", nslcmop["_id"], nslcmop)
 279             return nsr_id, nslcmop
 280
 281         try:
 282             # wait for any previous tasks in process
 283             await self.lcm_tasks.waitfor_related_HA('nsi', 'nsilcmops', nsilcmop_id)
 284
 285             step = "Getting nsir={} from db".format(nsir_id)
 286             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 287             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 288             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 289
 290             # Empty list to keep track of network service records status in the netslice
 291             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 292
 293             # Slice status Creating
 294             db_nsir_update["detailed-status"] = "creating"
 295             db_nsir_update["operational-status"] = "init"
 296             self.update_db_2("nsis", nsir_id, db_nsir_update)
 297
 298             # Creating netslice VLDs networking before NS instantiation
 299             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 300             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 301                 await netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update)
 302             self.update_db_2("nsis", nsir_id, db_nsir_update)
 303
 304             db_nsir_update["detailed-status"] = "Creating netslice subnets at RO"
 305             self.update_db_2("nsis", nsir_id, db_nsir_update)
 306
 307             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 308
 309             # Check status of the VLDs and wait for creation
 310             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 311             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 312             # for netslice_scenario in netslice_scenarios:
 313             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 314             #                                  nsir_id, db_nsir_update_RO)
 315
 316             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 317             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 318
 319             # Iterate over the network services operation ids to instantiate NSs
 320             # TODO: (future improvement) look another way check the tasks instead of keep asking
 321             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 322             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 323             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 324             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 325             for nslcmop_id in nslcmop_ids:
 326                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 327                 # Overwriting netslice-vld vim-net-id to ns
 328                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 329                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 330                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 331                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 332
 333             # Wait until Network Slice is ready
 334             step = nsir_status_detailed = " Waiting nsi ready. nsi_id={}".format(nsir_id)
 335             nsrs_detailed_list_old = None
 336             self.logger.debug(logging_text + step)
 337
 338             # TODO: substitute while for await (all task to be done or not)
 339             deployment_timeout = 2 * 3600   # Two hours
 340             while deployment_timeout > 0:
 341                 # Check ns instantiation status
 342                 nsi_ready = True
 343                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 344                 nsir_admin = nsir["_admin"]
 345                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 346                 nsrs_detailed_list_new = []
 347                 for nslcmop_item in nslcmop_ids:
 348                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 349                     status = nslcmop.get("operationState")
 350                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 351                     for nss in nsrs_detailed_list:
 352                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 353                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 354                                         "detailed-status":
 355                                         nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status")),
 356                                         "instantiated": True})
 357                             nsrs_detailed_list_new.append(nss)
 358                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 359                         nsi_ready = False
 360
 361                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 362                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list_new
 363                     nsrs_detailed_list_old = nsrs_detailed_list_new
 364                     db_nsir_update["_admin"] = nsir_admin
 365                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 366
 367                 if nsi_ready:
 368                     step = "Network Slice Instance is ready. nsi_id={}".format(nsir_id)
 369                     for items in nsrs_detailed_list:
 370                         if "FAILED" in items.values():
 371                             raise LcmException("Error deploying NSI: {}".format(nsir_id))
 372                     break
 373
 374                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 375                 await asyncio.sleep(5, loop=self.loop)
 376                 deployment_timeout -= 5
 377
 378             if deployment_timeout <= 0:
 379                 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id))
 380
 381             db_nsir_update["operational-status"] = "running"
 382             db_nsir_update["detailed-status"] = "done"
 383             db_nsir_update["config-status"] = "configured"
 384             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 385             db_nsilcmop_update["statusEnteredTime"] = time()
 386             db_nsilcmop_update["detailed-status"] = "done"
 387             return
 388
 389         except (LcmException, DbException) as e:
 390             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 391             exc = e
 392         except asyncio.CancelledError:
 393             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 394             exc = "Operation was cancelled"
 395         except Exception as e:
 396             exc = traceback.format_exc()
 397             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 398                                  exc_info=True)
 399         finally:
 400             if exc:
 401                 if db_nsir:
 402                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 403                     db_nsir_update["operational-status"] = "failed"
 404                 if db_nsilcmop:
 405                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 406                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 407                     db_nsilcmop_update["statusEnteredTime"] = time()
 408             try:
 409                 if db_nsir:
 410                     db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 411                     db_nsir_update["_admin.nsilcmop"] = None
 412                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 413                 if db_nsilcmop:
 414                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 415             except DbException as e:
 416                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 417             if nsilcmop_operation_state:
 418                 try:
 419                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 420                                                                     "operationState": nsilcmop_operation_state})
 421                 except Exception as e:
 422                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 423             self.logger.debug(logging_text + "Exit")
 424             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 425
 426     async def terminate(self, nsir_id, nsilcmop_id):
 427
 428         # Try to lock HA task here
 429         task_is_locked_by_me = self.lcm_tasks.lock_HA('nsi', 'nsilcmops', nsilcmop_id)
 430         if not task_is_locked_by_me:
 431             return
 432
 433         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 434         self.logger.debug(logging_text + "Enter")
 435         exc = None
 436         db_nsir = None
 437         db_nsilcmop = None
 438         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 439         db_nsilcmop_update = {}
 440         RO = ROclient.ROClient(self.loop, **self.ro_config)
 441         nsir_deployed = None
 442         failed_detail = []   # annotates all failed error messages
 443         nsilcmop_operation_state = None
 444         autoremove = False  # autoremove after terminated
 445         try:
 446             # wait for any previous tasks in process
 447             await self.lcm_tasks.waitfor_related_HA('nsi', 'nsilcmops', nsilcmop_id)
 448
 449             step = "Getting nsir={} from db".format(nsir_id)
 450             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 451             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 452             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 453             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 454
 455             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 456             # CASE: Instance was terminated but there is a second request to terminate the instance
 457             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 458                 return
 459
 460             # Slice status Terminating
 461             db_nsir_update["operational-status"] = "terminating"
 462             db_nsir_update["config-status"] = "terminating"
 463             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 464             self.update_db_2("nsis", nsir_id, db_nsir_update)
 465
 466             # Gets the list to keep track of network service records status in the netslice
 467             nsir_admin = db_nsir["_admin"]
 468             nsrs_detailed_list = []
 469
 470             # Iterate over the network services operation ids to terminate NSs
 471             # TODO: (future improvement) look another way check the tasks instead of keep asking
 472             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 473             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 474             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 475             nslcmop_new = []
 476             for nslcmop_id in nslcmop_ids:
 477                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 478                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 479                 nss_in_use = self.db.get_list("nsis", {"_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 480                                                        "operational-status": {"$nin": ["terminated", "failed"]}})
 481                 if len(nss_in_use) < 2:
 482                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 483                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 484                     nslcmop_new.append(nslcmop_id)
 485                 else:
 486                     # Update shared nslcmop shared with active nsi
 487                     netsliceInstanceId = db_nsir["_id"]
 488                     for nsis_item in nss_in_use:
 489                         if db_nsir["_id"] != nsis_item["_id"]:
 490                             netsliceInstanceId = nsis_item["_id"]
 491                             break
 492                     self.db.set_one("nslcmops", {"_id": nslcmop_id},
 493                                     {"operationParams.netsliceInstanceId": netsliceInstanceId})
 494             self.db.set_one("nsilcmops", {"_id": nsilcmop_id}, {"operationParams.nslcmops_ids": nslcmop_new})
 495
 496             # Wait until Network Slice is terminated
 497             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 498             nsrs_detailed_list_old = None
 499             self.logger.debug(logging_text + step)
 500
 501             termination_timeout = 2 * 3600   # Two hours
 502             while termination_timeout > 0:
 503                 # Check ns termination status
 504                 nsi_ready = True
 505                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 506                 nsir_admin = db_nsir["_admin"]
 507                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 508                 nsrs_detailed_list_new = []
 509                 for nslcmop_item in nslcmop_ids:
 510                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 511                     status = nslcmop["operationState"]
 512                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 513                     for nss in nsrs_detailed_list:
 514                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 515                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 516                                         "detailed-status":
 517                                         nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 518                             nsrs_detailed_list_new.append(nss)
 519                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 520                         nsi_ready = False
 521
 522                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 523                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list_new
 524                     nsrs_detailed_list_old = nsrs_detailed_list_new
 525                     db_nsir_update["_admin"] = nsir_admin
 526                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 527
 528                 if nsi_ready:
 529                     # Check if it is the last used nss and mark isinstantiate: False
 530                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 531                     nsir_admin = db_nsir["_admin"]
 532                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 533                     for nss in nsrs_detailed_list:
 534                         _filter = {"_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 535                                    "operational-status.ne": "terminated",
 536                                    "_id.ne": nsir_id}
 537                         nsis_list = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 538                         if not nsis_list:
 539                             nss.update({"instantiated": False})
 540                     db_nsir_update["_admin"] = nsir_admin
 541                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 542
 543                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 544                     for items in nsrs_detailed_list:
 545                         if "FAILED" in items.values():
 546                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 547                     break
 548
 549                 await asyncio.sleep(5, loop=self.loop)
 550                 termination_timeout -= 5
 551
 552             if termination_timeout <= 0:
 553                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 554
 555             # Delete netslice-vlds
 556             RO_nsir_id = RO_delete_action = None
 557             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 558                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 559                 try:
 560                     step = db_nsir_update["detailed-status"] = "Deleting netslice-vld at RO"
 561                     db_nsilcmop_update["detailed-status"] = "Deleting netslice-vld at RO"
 562                     self.logger.debug(logging_text + step)
 563                     desc = await RO.delete("ns", RO_nsir_id)
 564                     RO_delete_action = desc["action_id"]
 565                     nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 566                     nsir_deployed_RO["vld_status"] = "DELETING"
 567                     db_nsir_update["_admin.deployed"] = nsir_deployed
 568                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 569                     if RO_delete_action:
 570                         # wait until NS is deleted from VIM
 571                         step = "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id)
 572                         self.logger.debug(logging_text + step)
 573                 except ROclient.ROClientException as e:
 574                     if e.http_code == 404:  # not found
 575                         nsir_deployed_RO["vld_id"] = None
 576                         nsir_deployed_RO["vld_status"] = "DELETED"
 577                         self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsir_id))
 578                     elif e.http_code == 409:   # conflict
 579                         failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e))
 580                         self.logger.debug(logging_text + failed_detail[-1])
 581                     else:
 582                         failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsir_id, e))
 583                         self.logger.error(logging_text + failed_detail[-1])
 584
 585                 if failed_detail:
 586                     self.logger.error(logging_text + " ;".join(failed_detail))
 587                     db_nsir_update["operational-status"] = "failed"
 588                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
 589                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 590                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 591                     db_nsilcmop_update["statusEnteredTime"] = time()
 592                 else:
 593                     db_nsir_update["operational-status"] = "terminating"
 594                     db_nsir_update["config-status"] = "terminating"
 595                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 596                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 597                     db_nsilcmop_update["statusEnteredTime"] = time()
 598                     if db_nsilcmop["operationParams"].get("autoremove"):
 599                         autoremove = True
 600
 601             db_nsir_update["detailed-status"] = "done"
 602             db_nsir_update["operational-status"] = "terminated"
 603             db_nsir_update["config-status"] = "terminated"
 604             db_nsilcmop_update["statusEnteredTime"] = time()
 605             db_nsilcmop_update["detailed-status"] = "done"
 606             return
 607
 608         except (LcmException, DbException) as e:
 609             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 610             exc = e
 611         except asyncio.CancelledError:
 612             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 613             exc = "Operation was cancelled"
 614         except Exception as e:
 615             exc = traceback.format_exc()
 616             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 617                                  exc_info=True)
 618         finally:
 619             if exc:
 620                 if db_nsir:
 621                     db_nsir_update["_admin.deployed"] = nsir_deployed
 622                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 623                     db_nsir_update["operational-status"] = "failed"
 624                 if db_nsilcmop:
 625                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 626                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 627                     db_nsilcmop_update["statusEnteredTime"] = time()
 628             try:
 629                 if db_nsir:
 630                     db_nsir_update["_admin.deployed"] = nsir_deployed
 631                     db_nsir_update["_admin.nsilcmop"] = None
 632                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 633                 if db_nsilcmop:
 634                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 635             except DbException as e:
 636                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 637
 638             if nsilcmop_operation_state:
 639                 try:
 640                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 641                                                                   "operationState": nsilcmop_operation_state,
 642                                                                   "autoremove": autoremove},
 643                                             loop=self.loop)
 644                 except Exception as e:
 645                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 646             self.logger.debug(logging_text + "Exit")
 647             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")