osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2
   3 import asyncio
   4 import logging
   5 import logging.handlers
   6 import traceback
   7 import ns
   8 from ns import populate_dict as populate_dict
   9 import ROclient
  10 from lcm_utils import LcmException, LcmBase
  11 from osm_common.dbbase import DbException
  12 from time import time
  13 from copy import deepcopy
  14
  15
  16 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  17
  18
  19 def get_iterable(in_dict, in_key):
  20     """
  21     Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
  22     :param in_dict: a dictionary
  23     :param in_key: the key to look for at in_dict
  24     :return: in_dict[in_var] or () if it is None or not present
  25     """
  26     if not in_dict.get(in_key):
  27         return ()
  28     return in_dict[in_key]
  29
  30
  31 class NetsliceLcm(LcmBase):
  32
  33     total_deploy_timeout = 2 * 3600   # global timeout for deployment
  34
  35     def __init__(self, db, msg, fs, lcm_tasks, ro_config, vca_config, loop):
  36         """
  37         Init, Connect to database, filesystem storage, and messaging
  38         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  39         :return: None
  40         """
  41         # logging
  42         self.logger = logging.getLogger('lcm.netslice')
  43         self.loop = loop
  44         self.lcm_tasks = lcm_tasks
  45         self.ns = ns.NsLcm(db, msg, fs, lcm_tasks, ro_config, vca_config, loop)
  46         self.ro_config = ro_config
  47
  48         super().__init__(db, msg, fs, self.logger)
  49
  50     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  51         """
  52         Updates database nsir with the RO info for the created vld
  53         :param nsi_update_nsir: dictionary to be filled with the updated info
  54         :param db_nsir: content of db_nsir. This is also modified
  55         :param nsir_desc_RO: nsir descriptor from RO
  56         :return: Nothing, LcmException is raised on errors
  57         """
  58
  59         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  60             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  61                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  62                     continue
  63                 vld["vim-id"] = net_RO.get("vim_net_id")
  64                 vld["name"] = net_RO.get("vim_name")
  65                 vld["status"] = net_RO.get("status")
  66                 vld["status-detailed"] = net_RO.get("error_msg")
  67                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  68                 break
  69             else:
  70                 raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld["id"]))
  71
  72     async def instantiate(self, nsir_id, nsilcmop_id):
  73         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  74         self.logger.debug(logging_text + "Enter")
  75         # get all needed from database
  76         exc = None
  77         db_nsir = None
  78         db_nsilcmop = None
  79         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  80         db_nsilcmop_update = {}
  81         nsilcmop_operation_state = None
  82         vim_2_RO = {}
  83         RO = ROclient.ROClient(self.loop, **self.ro_config)
  84
  85         def ip_profile_2_RO(ip_profile):
  86             RO_ip_profile = deepcopy((ip_profile))
  87             if "dns-server" in RO_ip_profile:
  88                 if isinstance(RO_ip_profile["dns-server"], list):
  89                     RO_ip_profile["dns-address"] = []
  90                     for ds in RO_ip_profile.pop("dns-server"):
  91                         RO_ip_profile["dns-address"].append(ds['address'])
  92                 else:
  93                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
  94             if RO_ip_profile.get("ip-version") == "ipv4":
  95                 RO_ip_profile["ip-version"] = "IPv4"
  96             if RO_ip_profile.get("ip-version") == "ipv6":
  97                 RO_ip_profile["ip-version"] = "IPv6"
  98             if "dhcp-params" in RO_ip_profile:
  99                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 100             return RO_ip_profile
 101
 102         def vim_account_2_RO(vim_account):
 103             """
 104             Translate a RO vim_account from OSM vim_account params
 105             :param ns_params: OSM instantiate params
 106             :return: The RO ns descriptor
 107             """
 108             if vim_account in vim_2_RO:
 109                 return vim_2_RO[vim_account]
 110
 111             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 112             if db_vim["_admin"]["operationalState"] != "ENABLED":
 113                 raise LcmException("VIM={} is not available. operationalState={}".format(
 114                     vim_account, db_vim["_admin"]["operationalState"]))
 115             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 116             vim_2_RO[vim_account] = RO_vim_id
 117             return RO_vim_id
 118
 119         async def netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update):
 120             """
 121             Create a network slice VLD through RO Scenario
 122             :param vld_id The VLD id inside nsir to be created
 123             :param nsir_id The nsir id
 124             """
 125             ip_vld = None
 126             mgmt_network = False
 127             RO_vld_sites = []
 128             vld_id = vld_item["id"]
 129             netslice_vld = vld_item
 130             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 131             # self.logger.debug(logging_text + "Enter")
 132
 133             vld_shared = None
 134             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 135                 _filter = {"_id.ne": nsir_id, "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item}
 136                 shared_nsi = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 137                 if shared_nsi:
 138                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 139                         if vld_id == vlds["vld_id"]:
 140                             vld_shared = {"instance_scenario_id": vlds["netslice_scenario_id"], "osm_id": vld_id}
 141                             break
 142                     break
 143
 144             # Creating netslice-vld at RO
 145             RO_nsir = db_nsir["_admin"].get("deployed", {}).get("RO", [])
 146
 147             if vld_id in RO_nsir:
 148                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 149
 150             # If netslice-vld doesn't exists then create it
 151             else:
 152                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 153                 # Updating network names and datacenters from instantiation parameters for each VLD
 154                 RO_ns_params = {}
 155                 RO_ns_params["name"] = netslice_vld["name"]
 156                 RO_ns_params["datacenter"] = vim_account_2_RO(db_nsir["instantiation_parameters"]["vimAccountId"])
 157                 for instantiation_params_vld in get_iterable(db_nsir["instantiation_parameters"], "netslice-vld"):
 158                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 159                         ip_vld = deepcopy(instantiation_params_vld)
 160
 161                 if netslice_vld.get("mgmt-network"):
 162                     mgmt_network = True
 163
 164                 # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 165                 # Use vim-network-id instantiation parameter
 166                 vim_network_option = None
 167                 if ip_vld:
 168                     if ip_vld.get("vim-network-id"):
 169                         vim_network_option = "vim-network-id"
 170                     elif ip_vld.get("vim-network-name"):
 171                         vim_network_option = "vim-network-name"
 172                     if ip_vld.get("ip-profile"):
 173                         populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "ip-profile"),
 174                                       ip_profile_2_RO(ip_vld["ip-profile"]))
 175
 176                 if vim_network_option:
 177                     if ip_vld.get(vim_network_option):
 178                         if isinstance(ip_vld.get(vim_network_option), list):
 179                             for vim_net_id in ip_vld.get(vim_network_option):
 180                                 for vim_account, vim_net in vim_net_id.items():
 181                                     RO_vld_sites.append({
 182                                         "netmap-use": vim_net,
 183                                         "datacenter": vim_account_2_RO(vim_account)
 184                                     })
 185                         elif isinstance(ip_vld.get(vim_network_option), dict):
 186                             for vim_account, vim_net in ip_vld.get(vim_network_option).items():
 187                                 RO_vld_sites.append({
 188                                     "netmap-use": vim_net,
 189                                     "datacenter": vim_account_2_RO(vim_account)
 190                                 })
 191                         else:
 192                             RO_vld_sites.append({
 193                                 "netmap-use": ip_vld[vim_network_option],
 194                                 "datacenter": vim_account_2_RO(netslice_vld["vimAccountId"])})
 195
 196                 # Use default netslice vim-network-name from template
 197                 else:
 198                     for nss_conn_point_ref in get_iterable(netslice_vld, "nss-connection-point-ref"):
 199                         if nss_conn_point_ref.get("vimAccountId"):
 200                             if nss_conn_point_ref["vimAccountId"] != netslice_vld["vimAccountId"]:
 201                                 RO_vld_sites.append({
 202                                     "netmap-create": None,
 203                                     "datacenter": vim_account_2_RO(nss_conn_point_ref["vimAccountId"])})
 204
 205                 if vld_shared:
 206                     populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "use-network"), vld_shared)
 207
 208                 if RO_vld_sites:
 209                     populate_dict(RO_ns_params, ("networks", netslice_vld["name"], "sites"), RO_vld_sites)
 210
 211                 RO_ns_params["scenario"] = {"nets": [{"name": netslice_vld["name"],
 212                                             "external": mgmt_network, "type": "bridge"}]}
 213
 214                 # self.logger.debug(logging_text + step)
 215                 db_nsir_update["detailed-status"] = "Creating netslice-vld at RO"
 216                 desc = await RO.create("ns", descriptor=RO_ns_params)
 217                 db_nsir_update_RO = {}
 218                 db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 219                 db_nsir_update_RO["vld_id"] = RO_ns_params["name"]
 220                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 221
 222         def overwrite_nsd_params(self, db_nsir, nslcmop):
 223             RO_list = []
 224             vld_op_list = []
 225             vld = None
 226             nsr_id = nslcmop.get("nsInstanceId")
 227             # Overwrite instantiation parameters in netslice runtime
 228             if db_nsir.get("_admin"):
 229                 if db_nsir["_admin"].get("deployed"):
 230                     db_admin_deployed_nsir = db_nsir["_admin"].get("deployed")
 231                     if db_admin_deployed_nsir.get("RO"):
 232                         RO_list = db_admin_deployed_nsir["RO"]
 233
 234             for RO_item in RO_list:
 235                 for netslice_vld in get_iterable(db_nsir["_admin"], "netslice-vld"):
 236                     # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 237                     if RO_item.get("vld_id") == netslice_vld.get("id"):
 238                         # Search the cp of netslice-vld that match with nst:netslice-subnet
 239                         for nss_cp_item in get_iterable(netslice_vld, "nss-connection-point-ref"):
 240                             # Search the netslice-subnet of nst that match
 241                             for nss in get_iterable(db_nsir["_admin"], "netslice-subnet"):
 242                                 # Compare nss-ref equal nss from nst
 243                                 if nss_cp_item["nss-ref"] == nss["nss-id"]:
 244                                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 245                                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 246                                     for cp_nsd in db_nsds["connection-point"]:
 247                                         if cp_nsd["name"] == nss_cp_item["nsd-connection-point-ref"]:
 248                                             if nslcmop.get("operationParams"):
 249                                                 if nslcmop["operationParams"].get("nsName") == nss["nsName"]:
 250                                                     vld_id = RO_item["vld_id"]
 251                                                     netslice_scenario_id = RO_item["netslice_scenario_id"]
 252                                                     nslcmop_vld = {}
 253                                                     nslcmop_vld["ns-net"] = {vld_id: netslice_scenario_id}
 254                                                     nslcmop_vld["name"] = cp_nsd["vld-id-ref"]
 255                                                     for vld in get_iterable(nslcmop["operationParams"], "vld"):
 256                                                         if vld["name"] == cp_nsd["vld-id-ref"]:
 257                                                             nslcmop_vld.update(vld)
 258                                                     vld_op_list.append(nslcmop_vld)
 259             nslcmop["operationParams"]["vld"] = vld_op_list
 260             self.update_db_2("nslcmops", nslcmop["_id"], nslcmop)
 261             return nsr_id, nslcmop
 262
 263         try:
 264             step = "Getting nsir={} from db".format(nsir_id)
 265             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 266             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 267             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 268
 269             # look if previous tasks is in process
 270             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 271             if task_dependency:
 272                 step = db_nsilcmop_update["detailed-status"] = \
 273                     "Waiting for related tasks to be completed: {}".format(task_name)
 274                 self.logger.debug(logging_text + step)
 275                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 276                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 277                 if pending:
 278                     raise LcmException("Timeout waiting related tasks to be completed")
 279
 280             # Empty list to keep track of network service records status in the netslice
 281             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 282
 283             # Slice status Creating
 284             db_nsir_update["detailed-status"] = "creating"
 285             db_nsir_update["operational-status"] = "init"
 286             self.update_db_2("nsis", nsir_id, db_nsir_update)
 287
 288             # Creating netslice VLDs networking before NS instantiation
 289             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 290             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 291                 await netslice_scenario_create(self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update)
 292             self.update_db_2("nsis", nsir_id, db_nsir_update)
 293
 294             db_nsir_update["detailed-status"] = "Creating netslice subnets at RO"
 295             self.update_db_2("nsis", nsir_id, db_nsir_update)
 296
 297             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 298
 299             # Check status of the VLDs and wait for creation
 300             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 301             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 302             # for netslice_scenario in netslice_scenarios:
 303             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 304             #                                  nsir_id, db_nsir_update_RO)
 305
 306             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 307             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 308
 309             # Iterate over the network services operation ids to instantiate NSs
 310             # TODO: (future improvement) look another way check the tasks instead of keep asking
 311             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 312             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 313             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 314             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 315             for nslcmop_id in nslcmop_ids:
 316                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 317                 # Overwriting netslice-vld vim-net-id to ns
 318                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 319                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 320                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 321                 self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 322
 323             # Wait until Network Slice is ready
 324             step = nsir_status_detailed = " Waiting nsi ready. nsi_id={}".format(nsir_id)
 325             nsrs_detailed_list_old = None
 326             self.logger.debug(logging_text + step)
 327
 328             # TODO: substitute while for await (all task to be done or not)
 329             deployment_timeout = 2 * 3600   # Two hours
 330             while deployment_timeout > 0:
 331                 # Check ns instantiation status
 332                 nsi_ready = True
 333                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 334                 nsir_admin = nsir["_admin"]
 335                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 336                 nsrs_detailed_list_new = []
 337                 for nslcmop_item in nslcmop_ids:
 338                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 339                     status = nslcmop.get("operationState")
 340                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 341                     for nss in nsrs_detailed_list:
 342                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 343                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 344                                         "detailed-status":
 345                                         nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status")),
 346                                         "instantiated": True})
 347                             nsrs_detailed_list_new.append(nss)
 348                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 349                         nsi_ready = False
 350
 351                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 352                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list_new
 353                     nsrs_detailed_list_old = nsrs_detailed_list_new
 354                     db_nsir_update["_admin"] = nsir_admin
 355                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 356
 357                 if nsi_ready:
 358                     step = "Network Slice Instance is ready. nsi_id={}".format(nsir_id)
 359                     for items in nsrs_detailed_list:
 360                         if "FAILED" in items.values():
 361                             raise LcmException("Error deploying NSI: {}".format(nsir_id))
 362                     break
 363
 364                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 365                 await asyncio.sleep(5, loop=self.loop)
 366                 deployment_timeout -= 5
 367
 368             if deployment_timeout <= 0:
 369                 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id))
 370
 371             db_nsir_update["operational-status"] = "running"
 372             db_nsir_update["detailed-status"] = "done"
 373             db_nsir_update["config-status"] = "configured"
 374             db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 375             db_nsilcmop_update["statusEnteredTime"] = time()
 376             db_nsilcmop_update["detailed-status"] = "done"
 377             return
 378
 379         except (LcmException, DbException) as e:
 380             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 381             exc = e
 382         except asyncio.CancelledError:
 383             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 384             exc = "Operation was cancelled"
 385         except Exception as e:
 386             exc = traceback.format_exc()
 387             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 388                                  exc_info=True)
 389         finally:
 390             if exc:
 391                 if db_nsir:
 392                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 393                     db_nsir_update["operational-status"] = "failed"
 394                 if db_nsilcmop:
 395                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 396                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 397                     db_nsilcmop_update["statusEnteredTime"] = time()
 398             try:
 399                 if db_nsir:
 400                     db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 401                     db_nsir_update["_admin.nsilcmop"] = None
 402                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 403                 if db_nsilcmop:
 404                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 405             except DbException as e:
 406                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 407             if nsilcmop_operation_state:
 408                 try:
 409                     await self.msg.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 410                                                                     "operationState": nsilcmop_operation_state})
 411                 except Exception as e:
 412                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 413             self.logger.debug(logging_text + "Exit")
 414             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 415
 416     async def terminate(self, nsir_id, nsilcmop_id):
 417         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 418         self.logger.debug(logging_text + "Enter")
 419         exc = None
 420         db_nsir = None
 421         db_nsilcmop = None
 422         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 423         db_nsilcmop_update = {}
 424         RO = ROclient.ROClient(self.loop, **self.ro_config)
 425         nsir_deployed = None
 426         failed_detail = []   # annotates all failed error messages
 427         nsilcmop_operation_state = None
 428         autoremove = False  # autoremove after terminated
 429         try:
 430             step = "Getting nsir={} from db".format(nsir_id)
 431             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 432             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 433             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 434             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 435
 436             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 437             # CASE: Instance was terminated but there is a second request to terminate the instance
 438             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 439                 return
 440
 441             # Slice status Terminating
 442             db_nsir_update["operational-status"] = "terminating"
 443             db_nsir_update["config-status"] = "terminating"
 444             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 445             self.update_db_2("nsis", nsir_id, db_nsir_update)
 446
 447             # look if previous tasks is in process
 448             task_name, task_dependency = self.lcm_tasks.lookfor_related("nsi", nsir_id, nsilcmop_id)
 449             if task_dependency:
 450                 step = db_nsilcmop_update["detailed-status"] = \
 451                     "Waiting for related tasks to be completed: {}".format(task_name)
 452                 self.logger.debug(logging_text + step)
 453                 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 454                 _, pending = await asyncio.wait(task_dependency, timeout=3600)
 455                 if pending:
 456                     raise LcmException("Timeout waiting related tasks to be completed")
 457
 458             # Gets the list to keep track of network service records status in the netslice
 459             nsir_admin = db_nsir["_admin"]
 460             nsrs_detailed_list = []
 461
 462             # Iterate over the network services operation ids to terminate NSs
 463             # TODO: (future improvement) look another way check the tasks instead of keep asking
 464             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 465             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 466             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 467             nslcmop_new = []
 468             for nslcmop_id in nslcmop_ids:
 469                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 470                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 471                 nss_in_use = self.db.get_list("nsis", {"_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 472                                                        "operational-status": {"$nin": ["terminated", "failed"]}})
 473                 if len(nss_in_use) < 2:
 474                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 475                     self.lcm_tasks.register("ns", nsr_id, nslcmop_id, "ns_instantiate", task)
 476                     nslcmop_new.append(nslcmop_id)
 477                 else:
 478                     # Update shared nslcmop shared with active nsi
 479                     netsliceInstanceId = db_nsir["_id"]
 480                     for nsis_item in nss_in_use:
 481                         if db_nsir["_id"] != nsis_item["_id"]:
 482                             netsliceInstanceId = nsis_item["_id"]
 483                             break
 484                     self.db.set_one("nslcmops", {"_id": nslcmop_id},
 485                                     {"operationParams.netsliceInstanceId": netsliceInstanceId})
 486             self.db.set_one("nsilcmops", {"_id": nsilcmop_id}, {"operationParams.nslcmops_ids": nslcmop_new})
 487
 488             # Wait until Network Slice is terminated
 489             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(nsir_id)
 490             nsrs_detailed_list_old = None
 491             self.logger.debug(logging_text + step)
 492
 493             termination_timeout = 2 * 3600   # Two hours
 494             while termination_timeout > 0:
 495                 # Check ns termination status
 496                 nsi_ready = True
 497                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 498                 nsir_admin = db_nsir["_admin"]
 499                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 500                 nsrs_detailed_list_new = []
 501                 for nslcmop_item in nslcmop_ids:
 502                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 503                     status = nslcmop["operationState"]
 504                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 505                     for nss in nsrs_detailed_list:
 506                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 507                             nss.update({"nsrId": nslcmop["nsInstanceId"], "status": nslcmop["operationState"],
 508                                         "detailed-status":
 509                                         nsir_status_detailed + "; {}".format(nslcmop.get("detailed-status"))})
 510                             nsrs_detailed_list_new.append(nss)
 511                     if status not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
 512                         nsi_ready = False
 513
 514                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 515                     nsir_admin["nsrs-detailed-list"] = nsrs_detailed_list_new
 516                     nsrs_detailed_list_old = nsrs_detailed_list_new
 517                     db_nsir_update["_admin"] = nsir_admin
 518                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 519
 520                 if nsi_ready:
 521                     # Check if it is the last used nss and mark isinstantiate: False
 522                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 523                     nsir_admin = db_nsir["_admin"]
 524                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 525                     for nss in nsrs_detailed_list:
 526                         _filter = {"_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 527                                    "operational-status.ne": "terminated",
 528                                    "_id.ne": nsir_id}
 529                         nsis_list = self.db.get_one("nsis", _filter, fail_on_empty=False, fail_on_more=False)
 530                         if not nsis_list:
 531                             nss.update({"instantiated": False})
 532                     db_nsir_update["_admin"] = nsir_admin
 533                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 534
 535                     step = "Network Slice Instance is terminated. nsi_id={}".format(nsir_id)
 536                     for items in nsrs_detailed_list:
 537                         if "FAILED" in items.values():
 538                             raise LcmException("Error terminating NSI: {}".format(nsir_id))
 539                     break
 540
 541                 await asyncio.sleep(5, loop=self.loop)
 542                 termination_timeout -= 5
 543
 544             if termination_timeout <= 0:
 545                 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id))
 546
 547             # Delete netslice-vlds
 548             RO_nsir_id = RO_delete_action = None
 549             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 550                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 551                 try:
 552                     step = db_nsir_update["detailed-status"] = "Deleting netslice-vld at RO"
 553                     db_nsilcmop_update["detailed-status"] = "Deleting netslice-vld at RO"
 554                     self.logger.debug(logging_text + step)
 555                     desc = await RO.delete("ns", RO_nsir_id)
 556                     RO_delete_action = desc["action_id"]
 557                     nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 558                     nsir_deployed_RO["vld_status"] = "DELETING"
 559                     db_nsir_update["_admin.deployed"] = nsir_deployed
 560                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 561                     if RO_delete_action:
 562                         # wait until NS is deleted from VIM
 563                         step = "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id)
 564                         self.logger.debug(logging_text + step)
 565                 except ROclient.ROClientException as e:
 566                     if e.http_code == 404:  # not found
 567                         nsir_deployed_RO["vld_id"] = None
 568                         nsir_deployed_RO["vld_status"] = "DELETED"
 569                         self.logger.debug(logging_text + "RO_ns_id={} already deleted".format(RO_nsir_id))
 570                     elif e.http_code == 409:   # conflict
 571                         failed_detail.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e))
 572                         self.logger.debug(logging_text + failed_detail[-1])
 573                     else:
 574                         failed_detail.append("RO_ns_id={} delete error: {}".format(RO_nsir_id, e))
 575                         self.logger.error(logging_text + failed_detail[-1])
 576
 577                 if failed_detail:
 578                     self.logger.error(logging_text + " ;".join(failed_detail))
 579                     db_nsir_update["operational-status"] = "failed"
 580                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(failed_detail)
 581                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 582                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 583                     db_nsilcmop_update["statusEnteredTime"] = time()
 584                 else:
 585                     db_nsir_update["operational-status"] = "terminating"
 586                     db_nsir_update["config-status"] = "terminating"
 587                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 588                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "COMPLETED"
 589                     db_nsilcmop_update["statusEnteredTime"] = time()
 590                     if db_nsilcmop["operationParams"].get("autoremove"):
 591                         autoremove = True
 592
 593             db_nsir_update["detailed-status"] = "done"
 594             db_nsir_update["operational-status"] = "terminated"
 595             db_nsir_update["config-status"] = "terminated"
 596             db_nsilcmop_update["statusEnteredTime"] = time()
 597             db_nsilcmop_update["detailed-status"] = "done"
 598             return
 599
 600         except (LcmException, DbException) as e:
 601             self.logger.error(logging_text + "Exit Exception while '{}': {}".format(step, e))
 602             exc = e
 603         except asyncio.CancelledError:
 604             self.logger.error(logging_text + "Cancelled Exception while '{}'".format(step))
 605             exc = "Operation was cancelled"
 606         except Exception as e:
 607             exc = traceback.format_exc()
 608             self.logger.critical(logging_text + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 609                                  exc_info=True)
 610         finally:
 611             if exc:
 612                 if db_nsir:
 613                     db_nsir_update["_admin.deployed"] = nsir_deployed
 614                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 615                     db_nsir_update["operational-status"] = "failed"
 616                 if db_nsilcmop:
 617                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(step, exc)
 618                     db_nsilcmop_update["operationState"] = nsilcmop_operation_state = "FAILED"
 619                     db_nsilcmop_update["statusEnteredTime"] = time()
 620             try:
 621                 if db_nsir:
 622                     db_nsir_update["_admin.deployed"] = nsir_deployed
 623                     db_nsir_update["_admin.nsilcmop"] = None
 624                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 625                 if db_nsilcmop:
 626                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 627             except DbException as e:
 628                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 629
 630             if nsilcmop_operation_state:
 631                 try:
 632                     await self.msg.aiowrite("nsi", "terminated", {"nsir_id": nsir_id, "nsilcmop_id": nsilcmop_id,
 633                                                                   "operationState": nsilcmop_operation_state,
 634                                                                   "autoremove": autoremove},
 635                                             loop=self.loop)
 636                 except Exception as e:
 637                     self.logger.error(logging_text + "kafka_write notification Exception {}".format(e))
 638             self.logger.debug(logging_text + "Exit")
 639             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")