osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import asyncio
  17 import logging
  18 import logging.handlers
  19 import traceback
  20 from osm_lcm import ROclient
  21 from osm_lcm.lcm_utils import (
  22     LcmException,
  23     LcmBase,
  24     populate_dict,
  25     get_iterable,
  26     deep_get,
  27 )
  28 from osm_common.dbbase import DbException
  29 from time import time
  30 from copy import deepcopy
  31
  32
  33 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  34
  35
  36 class NetsliceLcm(LcmBase):
  37     def __init__(self, msg, lcm_tasks, config, ns):
  38         """
  39         Init, Connect to database, filesystem storage, and messaging
  40         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  41         :return: None
  42         """
  43         # logging
  44         self.logger = logging.getLogger("lcm.netslice")
  45         self.lcm_tasks = lcm_tasks
  46         self.ns = ns
  47         self.ro_config = config["RO"]
  48         self.timeout = config["timeout"]
  49
  50         super().__init__(msg, self.logger)
  51
  52     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  53         """
  54         Updates database nsir with the RO info for the created vld
  55         :param nsi_update_nsir: dictionary to be filled with the updated info
  56         :param db_nsir: content of db_nsir. This is also modified
  57         :param nsir_desc_RO: nsir descriptor from RO
  58         :return: Nothing, LcmException is raised on errors
  59         """
  60
  61         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  62             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  63                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  64                     continue
  65                 vld["vim-id"] = net_RO.get("vim_net_id")
  66                 vld["name"] = net_RO.get("vim_name")
  67                 vld["status"] = net_RO.get("status")
  68                 vld["status-detailed"] = net_RO.get("error_msg")
  69                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  70                 break
  71             else:
  72                 raise LcmException(
  73                     "ns_update_nsir: Not found vld={} at RO info".format(vld["id"])
  74                 )
  75
  76     async def instantiate(self, nsir_id, nsilcmop_id):
  77         # Try to lock HA task here
  78         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
  79         if not task_is_locked_by_me:
  80             return
  81
  82         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  83         self.logger.debug(logging_text + "Enter")
  84         # get all needed from database
  85         exc = None
  86         db_nsir = None
  87         db_nsilcmop = None
  88         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  89         db_nsilcmop_update = {}
  90         nsilcmop_operation_state = None
  91         vim_2_RO = {}
  92         RO = ROclient.ROClient(**self.ro_config)
  93         nsi_vld_instantiationi_params = {}
  94
  95         def ip_profile_2_RO(ip_profile):
  96             RO_ip_profile = deepcopy((ip_profile))
  97             if "dns-server" in RO_ip_profile:
  98                 if isinstance(RO_ip_profile["dns-server"], list):
  99                     RO_ip_profile["dns-address"] = []
 100                     for ds in RO_ip_profile.pop("dns-server"):
 101                         RO_ip_profile["dns-address"].append(ds["address"])
 102                 else:
 103                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 104             if RO_ip_profile.get("ip-version") == "ipv4":
 105                 RO_ip_profile["ip-version"] = "IPv4"
 106             if RO_ip_profile.get("ip-version") == "ipv6":
 107                 RO_ip_profile["ip-version"] = "IPv6"
 108             if "dhcp-params" in RO_ip_profile:
 109                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 110             return RO_ip_profile
 111
 112         def vim_account_2_RO(vim_account):
 113             """
 114             Translate a RO vim_account from OSM vim_account params
 115             :param ns_params: OSM instantiate params
 116             :return: The RO ns descriptor
 117             """
 118             if vim_account in vim_2_RO:
 119                 return vim_2_RO[vim_account]
 120
 121             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 122             if db_vim["_admin"]["operationalState"] != "ENABLED":
 123                 raise LcmException(
 124                     "VIM={} is not available. operationalState={}".format(
 125                         vim_account, db_vim["_admin"]["operationalState"]
 126                     )
 127                 )
 128             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 129             vim_2_RO[vim_account] = RO_vim_id
 130             return RO_vim_id
 131
 132         async def netslice_scenario_create(
 133             self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 134         ):
 135             """
 136             Create a network slice VLD through RO Scenario
 137             :param vld_id The VLD id inside nsir to be created
 138             :param nsir_id The nsir id
 139             """
 140             nonlocal nsi_vld_instantiationi_params
 141             ip_vld = None
 142             mgmt_network = False
 143             RO_vld_sites = []
 144             vld_id = vld_item["id"]
 145             netslice_vld = vld_item
 146             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 147             # self.logger.debug(logging_text + "Enter")
 148
 149             vld_shared = None
 150             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 151                 _filter = {
 152                     "_id.ne": nsir_id,
 153                     "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item,
 154                 }
 155                 shared_nsi = self.db.get_one(
 156                     "nsis", _filter, fail_on_empty=False, fail_on_more=False
 157                 )
 158                 if shared_nsi:
 159                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 160                         if vld_id == vlds["vld_id"]:
 161                             vld_shared = {
 162                                 "instance_scenario_id": vlds["netslice_scenario_id"],
 163                                 "osm_id": vld_id,
 164                             }
 165                             break
 166                     break
 167
 168             # Creating netslice-vld at RO
 169             RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
 170
 171             if vld_id in RO_nsir:
 172                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 173
 174             # If netslice-vld doesn't exists then create it
 175             else:
 176                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 177                 # Updating network names and datacenters from instantiation parameters for each VLD
 178                 for instantiation_params_vld in get_iterable(
 179                     db_nsir["instantiation_parameters"], "netslice-vld"
 180                 ):
 181                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 182                         ip_vld = deepcopy(instantiation_params_vld)
 183                         ip_vld.pop("name")
 184                         nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
 185
 186                 db_nsir_update_RO = {}
 187                 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
 188                 if self.ro_config["ng"]:
 189                     db_nsir_update_RO["netslice_scenario_id"] = (
 190                         vld_shared.get("instance_scenario_id")
 191                         if vld_shared
 192                         else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
 193                     )
 194                 else:  # if not self.ro_config["ng"]:
 195                     if netslice_vld.get("mgmt-network"):
 196                         mgmt_network = True
 197                     RO_ns_params = {}
 198                     RO_ns_params["name"] = netslice_vld["name"]
 199                     RO_ns_params["datacenter"] = vim_account_2_RO(
 200                         db_nsir["instantiation_parameters"]["vimAccountId"]
 201                     )
 202
 203                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 204                     # Use vim-network-id instantiation parameter
 205                     vim_network_option = None
 206                     if ip_vld:
 207                         if ip_vld.get("vim-network-id"):
 208                             vim_network_option = "vim-network-id"
 209                         elif ip_vld.get("vim-network-name"):
 210                             vim_network_option = "vim-network-name"
 211                         if ip_vld.get("ip-profile"):
 212                             populate_dict(
 213                                 RO_ns_params,
 214                                 ("networks", netslice_vld["name"], "ip-profile"),
 215                                 ip_profile_2_RO(ip_vld["ip-profile"]),
 216                             )
 217
 218                     if vim_network_option:
 219                         if ip_vld.get(vim_network_option):
 220                             if isinstance(ip_vld.get(vim_network_option), list):
 221                                 for vim_net_id in ip_vld.get(vim_network_option):
 222                                     for vim_account, vim_net in vim_net_id.items():
 223                                         RO_vld_sites.append(
 224                                             {
 225                                                 "netmap-use": vim_net,
 226                                                 "datacenter": vim_account_2_RO(
 227                                                     vim_account
 228                                                 ),
 229                                             }
 230                                         )
 231                             elif isinstance(ip_vld.get(vim_network_option), dict):
 232                                 for vim_account, vim_net in ip_vld.get(
 233                                     vim_network_option
 234                                 ).items():
 235                                     RO_vld_sites.append(
 236                                         {
 237                                             "netmap-use": vim_net,
 238                                             "datacenter": vim_account_2_RO(vim_account),
 239                                         }
 240                                     )
 241                             else:
 242                                 RO_vld_sites.append(
 243                                     {
 244                                         "netmap-use": ip_vld[vim_network_option],
 245                                         "datacenter": vim_account_2_RO(
 246                                             netslice_vld["vimAccountId"]
 247                                         ),
 248                                     }
 249                                 )
 250
 251                     # Use default netslice vim-network-name from template
 252                     else:
 253                         for nss_conn_point_ref in get_iterable(
 254                             netslice_vld, "nss-connection-point-ref"
 255                         ):
 256                             if nss_conn_point_ref.get("vimAccountId"):
 257                                 if (
 258                                     nss_conn_point_ref["vimAccountId"]
 259                                     != netslice_vld["vimAccountId"]
 260                                 ):
 261                                     RO_vld_sites.append(
 262                                         {
 263                                             "netmap-create": None,
 264                                             "datacenter": vim_account_2_RO(
 265                                                 nss_conn_point_ref["vimAccountId"]
 266                                             ),
 267                                         }
 268                                     )
 269
 270                     if vld_shared:
 271                         populate_dict(
 272                             RO_ns_params,
 273                             ("networks", netslice_vld["name"], "use-network"),
 274                             vld_shared,
 275                         )
 276
 277                     if RO_vld_sites:
 278                         populate_dict(
 279                             RO_ns_params,
 280                             ("networks", netslice_vld["name"], "sites"),
 281                             RO_vld_sites,
 282                         )
 283
 284                     RO_ns_params["scenario"] = {
 285                         "nets": [
 286                             {
 287                                 "name": netslice_vld["name"],
 288                                 "external": mgmt_network,
 289                                 "type": "bridge",
 290                             }
 291                         ]
 292                     }
 293
 294                     # self.logger.debug(logging_text + step)
 295                     desc = await RO.create("ns", descriptor=RO_ns_params)
 296                     db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 297                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 298
 299         def overwrite_nsd_params(self, db_nsir, nslcmop):
 300             nonlocal nsi_vld_instantiationi_params
 301             nonlocal db_nsir_update
 302             vld_op_list = []
 303             vld = None
 304             nsr_id = nslcmop.get("nsInstanceId")
 305             # Overwrite instantiation parameters in netslice runtime
 306             RO_list = db_nsir_admin["deployed"]["RO"]
 307
 308             for ro_item_index, RO_item in enumerate(RO_list):
 309                 netslice_vld = next(
 310                     (
 311                         n
 312                         for n in get_iterable(db_nsir["_admin"], "netslice-vld")
 313                         if RO_item.get("vld_id") == n.get("id")
 314                     ),
 315                     None,
 316                 )
 317                 if not netslice_vld:
 318                     continue
 319                 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 320                 # Search the cp of netslice-vld that match with nst:netslice-subnet
 321                 for nss_cp_item in get_iterable(
 322                     netslice_vld, "nss-connection-point-ref"
 323                 ):
 324                     # Search the netslice-subnet of nst that match
 325                     nss = next(
 326                         (
 327                             nss
 328                             for nss in get_iterable(
 329                                 db_nsir["_admin"], "netslice-subnet"
 330                             )
 331                             if nss_cp_item["nss-ref"] == nss["nss-id"]
 332                         ),
 333                         None,
 334                     )
 335                     # Compare nss-ref equal nss from nst
 336                     if not nss:
 337                         continue
 338                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 339                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 340                     for cp_nsd in db_nsds.get("sapd", ()):
 341                         if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
 342                             if nslcmop.get("operationParams"):
 343                                 if (
 344                                     nslcmop["operationParams"].get("nsName")
 345                                     == nss["nsName"]
 346                                 ):
 347                                     vld_id = RO_item["vld_id"]
 348                                     netslice_scenario_id = RO_item[
 349                                         "netslice_scenario_id"
 350                                     ]
 351                                     nslcmop_vld = {}
 352                                     nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
 353                                     for vld in get_iterable(
 354                                         nslcmop["operationParams"], "vld"
 355                                     ):
 356                                         if vld["name"] == cp_nsd["virtual-link-desc"]:
 357                                             nslcmop_vld.update(vld)
 358                                     if self.ro_config["ng"]:
 359                                         nslcmop_vld["common_id"] = netslice_scenario_id
 360                                         nslcmop_vld.update(
 361                                             nsi_vld_instantiationi_params.get(
 362                                                 RO_item["vld_id"], {}
 363                                             )
 364                                         )
 365                                     else:
 366                                         nslcmop_vld["ns-net"] = {
 367                                             vld_id: netslice_scenario_id
 368                                         }
 369                                     vld_op_list.append(nslcmop_vld)
 370             nslcmop["operationParams"]["vld"] = vld_op_list
 371             self.update_db_2(
 372                 "nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list}
 373             )
 374             return nsr_id, nslcmop
 375
 376         try:
 377             # wait for any previous tasks in process
 378             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 379
 380             step = "Getting nsir={} from db".format(nsir_id)
 381             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 382             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 383             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 384
 385             start_deploy = time()
 386             nsi_params = db_nsilcmop.get("operationParams")
 387             if nsi_params and nsi_params.get("timeout_nsi_deploy"):
 388                 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
 389             else:
 390                 timeout_nsi_deploy = self.timeout.get("nsi_deploy")
 391
 392             # Empty list to keep track of network service records status in the netslice
 393             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 394
 395             step = "Creating slice operational-status init"
 396             # Slice status Creating
 397             db_nsir_update["detailed-status"] = "creating"
 398             db_nsir_update["operational-status"] = "init"
 399             db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 400
 401             step = "Instantiating netslice VLDs before NS instantiation"
 402             # Creating netslice VLDs networking before NS instantiation
 403             db_nsir_update["detailed-status"] = step
 404             self.update_db_2("nsis", nsir_id, db_nsir_update)
 405             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 406             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 407                 await netslice_scenario_create(
 408                     self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 409                 )
 410
 411             step = "Instantiating netslice subnets"
 412             db_nsir_update["detailed-status"] = step
 413             self.update_db_2("nsis", nsir_id, db_nsir_update)
 414
 415             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 416
 417             # Check status of the VLDs and wait for creation
 418             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 419             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 420             # for netslice_scenario in netslice_scenarios:
 421             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 422             #                                  nsir_id, db_nsir_update_RO)
 423
 424             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 425             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 426
 427             # Iterate over the network services operation ids to instantiate NSs
 428             step = "Instantiating Netslice Subnets"
 429             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 430             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 431             for nslcmop_id in nslcmop_ids:
 432                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 433                 # Overwriting netslice-vld vim-net-id to ns
 434                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 435                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 436                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 437                 self.lcm_tasks.register(
 438                     "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 439                 )
 440
 441             # Wait until Network Slice is ready
 442             step = " Waiting nsi ready."
 443             nsrs_detailed_list_old = None
 444             self.logger.debug(logging_text + step)
 445
 446             # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
 447             while time() <= start_deploy + timeout_nsi_deploy:
 448                 # Check ns instantiation status
 449                 nsi_ready = True
 450                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 451                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 452                 nsrs_detailed_list_new = []
 453                 for nslcmop_item in nslcmop_ids:
 454                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 455                     status = nslcmop.get("operationState")
 456                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 457                     for nss in nsrs_detailed_list:
 458                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 459                             nss.update(
 460                                 {
 461                                     "nsrId": nslcmop["nsInstanceId"],
 462                                     "status": nslcmop["operationState"],
 463                                     "detailed-status": nslcmop.get("detailed-status"),
 464                                     "instantiated": True,
 465                                 }
 466                             )
 467                             nsrs_detailed_list_new.append(nss)
 468                     if status not in [
 469                         "COMPLETED",
 470                         "PARTIALLY_COMPLETED",
 471                         "FAILED",
 472                         "FAILED_TEMP",
 473                     ]:
 474                         nsi_ready = False
 475
 476                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 477                     nsrs_detailed_list_old = nsrs_detailed_list_new
 478                     self.update_db_2(
 479                         "nsis",
 480                         nsir_id,
 481                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 482                     )
 483
 484                 if nsi_ready:
 485                     error_list = []
 486                     step = "Network Slice Instance instantiated"
 487                     for nss in nsrs_detailed_list:
 488                         if nss["status"] in ("FAILED", "FAILED_TEMP"):
 489                             error_list.append(
 490                                 "NS {} {}: {}".format(
 491                                     nss["nsrId"], nss["status"], nss["detailed-status"]
 492                                 )
 493                             )
 494                     if error_list:
 495                         step = "instantiating"
 496                         raise LcmException("; ".join(error_list))
 497                     break
 498
 499                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 500                 await asyncio.sleep(5)
 501
 502             else:  # timeout_nsi_deploy reached:
 503                 raise LcmException("Timeout waiting nsi to be ready.")
 504
 505             db_nsir_update["operational-status"] = "running"
 506             db_nsir_update["detailed-status"] = "done"
 507             db_nsir_update["config-status"] = "configured"
 508             db_nsilcmop_update[
 509                 "operationState"
 510             ] = nsilcmop_operation_state = "COMPLETED"
 511             db_nsilcmop_update["statusEnteredTime"] = time()
 512             db_nsilcmop_update["detailed-status"] = "done"
 513             return
 514
 515         except (LcmException, DbException) as e:
 516             self.logger.error(
 517                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 518             )
 519             exc = e
 520         except asyncio.CancelledError:
 521             self.logger.error(
 522                 logging_text + "Cancelled Exception while '{}'".format(step)
 523             )
 524             exc = "Operation was cancelled"
 525         except Exception as e:
 526             exc = traceback.format_exc()
 527             self.logger.critical(
 528                 logging_text
 529                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 530                 exc_info=True,
 531             )
 532         finally:
 533             if exc:
 534                 if db_nsir:
 535                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 536                     db_nsir_update["operational-status"] = "failed"
 537                     db_nsir_update["config-status"] = "configured"
 538                 if db_nsilcmop:
 539                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 540                         step, exc
 541                     )
 542                     db_nsilcmop_update[
 543                         "operationState"
 544                     ] = nsilcmop_operation_state = "FAILED"
 545                     db_nsilcmop_update["statusEnteredTime"] = time()
 546             try:
 547                 if db_nsir:
 548                     db_nsir_update["_admin.nsilcmop"] = None
 549                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 550                 if db_nsilcmop:
 551                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 552             except DbException as e:
 553                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 554             if nsilcmop_operation_state:
 555                 try:
 556                     await self.msg.aiowrite(
 557                         "nsi",
 558                         "instantiated",
 559                         {
 560                             "nsir_id": nsir_id,
 561                             "nsilcmop_id": nsilcmop_id,
 562                             "operationState": nsilcmop_operation_state,
 563                         },
 564                     )
 565                 except Exception as e:
 566                     self.logger.error(
 567                         logging_text + "kafka_write notification Exception {}".format(e)
 568                     )
 569             self.logger.debug(logging_text + "Exit")
 570             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 571
 572     async def terminate(self, nsir_id, nsilcmop_id):
 573         # Try to lock HA task here
 574         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
 575         if not task_is_locked_by_me:
 576             return
 577
 578         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 579         self.logger.debug(logging_text + "Enter")
 580         exc = None
 581         db_nsir = None
 582         db_nsilcmop = None
 583         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 584         db_nsilcmop_update = {}
 585         RO = ROclient.ROClient(**self.ro_config)
 586         nsir_deployed = None
 587         failed_detail = []  # annotates all failed error messages
 588         nsilcmop_operation_state = None
 589         autoremove = False  # autoremove after terminated
 590         try:
 591             # wait for any previous tasks in process
 592             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 593
 594             step = "Getting nsir={} from db".format(nsir_id)
 595             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 596             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 597             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 598             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 599
 600             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 601             # CASE: Instance was terminated but there is a second request to terminate the instance
 602             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 603                 return
 604
 605             # Slice status Terminating
 606             db_nsir_update["operational-status"] = "terminating"
 607             db_nsir_update["config-status"] = "terminating"
 608             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 609             self.update_db_2("nsis", nsir_id, db_nsir_update)
 610
 611             # Gets the list to keep track of network service records status in the netslice
 612             nsrs_detailed_list = []
 613
 614             # Iterate over the network services operation ids to terminate NSs
 615             # TODO: (future improvement) look another way check the tasks instead of keep asking
 616             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 617             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 618             step = "Terminating Netslice Subnets"
 619             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 620             nslcmop_new = []
 621             for nslcmop_id in nslcmop_ids:
 622                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 623                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 624                 nss_in_use = self.db.get_list(
 625                     "nsis",
 626                     {
 627                         "_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 628                         "operational-status": {"$nin": ["terminated", "failed"]},
 629                     },
 630                 )
 631                 if len(nss_in_use) < 2:
 632                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 633                     self.lcm_tasks.register(
 634                         "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 635                     )
 636                     nslcmop_new.append(nslcmop_id)
 637                 else:
 638                     # Update shared nslcmop shared with active nsi
 639                     netsliceInstanceId = db_nsir["_id"]
 640                     for nsis_item in nss_in_use:
 641                         if db_nsir["_id"] != nsis_item["_id"]:
 642                             netsliceInstanceId = nsis_item["_id"]
 643                             break
 644                     self.db.set_one(
 645                         "nslcmops",
 646                         {"_id": nslcmop_id},
 647                         {"operationParams.netsliceInstanceId": netsliceInstanceId},
 648                     )
 649             self.db.set_one(
 650                 "nsilcmops",
 651                 {"_id": nsilcmop_id},
 652                 {"operationParams.nslcmops_ids": nslcmop_new},
 653             )
 654
 655             # Wait until Network Slice is terminated
 656             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(
 657                 nsir_id
 658             )
 659             nsrs_detailed_list_old = None
 660             self.logger.debug(logging_text + step)
 661
 662             termination_timeout = 2 * 3600  # Two hours
 663             while termination_timeout > 0:
 664                 # Check ns termination status
 665                 nsi_ready = True
 666                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 667                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 668                 nsrs_detailed_list_new = []
 669                 for nslcmop_item in nslcmop_ids:
 670                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 671                     status = nslcmop["operationState"]
 672                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 673                     for nss in nsrs_detailed_list:
 674                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 675                             nss.update(
 676                                 {
 677                                     "nsrId": nslcmop["nsInstanceId"],
 678                                     "status": nslcmop["operationState"],
 679                                     "detailed-status": nsir_status_detailed
 680                                     + "; {}".format(nslcmop.get("detailed-status")),
 681                                 }
 682                             )
 683                             nsrs_detailed_list_new.append(nss)
 684                     if status not in [
 685                         "COMPLETED",
 686                         "PARTIALLY_COMPLETED",
 687                         "FAILED",
 688                         "FAILED_TEMP",
 689                     ]:
 690                         nsi_ready = False
 691
 692                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 693                     nsrs_detailed_list_old = nsrs_detailed_list_new
 694                     self.update_db_2(
 695                         "nsis",
 696                         nsir_id,
 697                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 698                     )
 699
 700                 if nsi_ready:
 701                     # Check if it is the last used nss and mark isinstantiate: False
 702                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 703                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 704                     for nss in nsrs_detailed_list:
 705                         _filter = {
 706                             "_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 707                             "operational-status.ne": "terminated",
 708                             "_id.ne": nsir_id,
 709                         }
 710                         nsis_list = self.db.get_one(
 711                             "nsis", _filter, fail_on_empty=False, fail_on_more=False
 712                         )
 713                         if not nsis_list:
 714                             nss.update({"instantiated": False})
 715
 716                     step = "Network Slice Instance is terminated. nsi_id={}".format(
 717                         nsir_id
 718                     )
 719                     for items in nsrs_detailed_list:
 720                         if "FAILED" in items.values():
 721                             raise LcmException(
 722                                 "Error terminating NSI: {}".format(nsir_id)
 723                             )
 724                     break
 725
 726                 await asyncio.sleep(5)
 727                 termination_timeout -= 5
 728
 729             if termination_timeout <= 0:
 730                 raise LcmException(
 731                     "Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id)
 732                 )
 733
 734             # Delete netslice-vlds
 735             RO_nsir_id = RO_delete_action = None
 736             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 737                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 738                 try:
 739                     if not self.ro_config["ng"]:
 740                         step = db_nsir_update[
 741                             "detailed-status"
 742                         ] = "Deleting netslice-vld at RO"
 743                         db_nsilcmop_update[
 744                             "detailed-status"
 745                         ] = "Deleting netslice-vld at RO"
 746                         self.logger.debug(logging_text + step)
 747                         desc = await RO.delete("ns", RO_nsir_id)
 748                         RO_delete_action = desc["action_id"]
 749                         nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 750                         nsir_deployed_RO["vld_status"] = "DELETING"
 751                         db_nsir_update["_admin.deployed"] = nsir_deployed
 752                         self.update_db_2("nsis", nsir_id, db_nsir_update)
 753                         if RO_delete_action:
 754                             # wait until NS is deleted from VIM
 755                             step = "Waiting ns deleted from VIM. RO_id={}".format(
 756                                 RO_nsir_id
 757                             )
 758                             self.logger.debug(logging_text + step)
 759                 except ROclient.ROClientException as e:
 760                     if e.http_code == 404:  # not found
 761                         nsir_deployed_RO["vld_id"] = None
 762                         nsir_deployed_RO["vld_status"] = "DELETED"
 763                         self.logger.debug(
 764                             logging_text
 765                             + "RO_ns_id={} already deleted".format(RO_nsir_id)
 766                         )
 767                     elif e.http_code == 409:  # conflict
 768                         failed_detail.append(
 769                             "RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e)
 770                         )
 771                         self.logger.debug(logging_text + failed_detail[-1])
 772                     else:
 773                         failed_detail.append(
 774                             "RO_ns_id={} delete error: {}".format(RO_nsir_id, e)
 775                         )
 776                         self.logger.error(logging_text + failed_detail[-1])
 777
 778                 if failed_detail:
 779                     self.logger.error(logging_text + " ;".join(failed_detail))
 780                     db_nsir_update["operational-status"] = "failed"
 781                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(
 782                         failed_detail
 783                     )
 784                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 785                     db_nsilcmop_update[
 786                         "operationState"
 787                     ] = nsilcmop_operation_state = "FAILED"
 788                     db_nsilcmop_update["statusEnteredTime"] = time()
 789                 else:
 790                     db_nsir_update["operational-status"] = "terminating"
 791                     db_nsir_update["config-status"] = "terminating"
 792                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 793                     db_nsilcmop_update[
 794                         "operationState"
 795                     ] = nsilcmop_operation_state = "COMPLETED"
 796                     db_nsilcmop_update["statusEnteredTime"] = time()
 797                     if db_nsilcmop["operationParams"].get("autoremove"):
 798                         autoremove = True
 799
 800             db_nsir_update["detailed-status"] = "done"
 801             db_nsir_update["operational-status"] = "terminated"
 802             db_nsir_update["config-status"] = "terminated"
 803             db_nsilcmop_update["statusEnteredTime"] = time()
 804             db_nsilcmop_update["detailed-status"] = "done"
 805             return
 806
 807         except (LcmException, DbException) as e:
 808             self.logger.error(
 809                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 810             )
 811             exc = e
 812         except asyncio.CancelledError:
 813             self.logger.error(
 814                 logging_text + "Cancelled Exception while '{}'".format(step)
 815             )
 816             exc = "Operation was cancelled"
 817         except Exception as e:
 818             exc = traceback.format_exc()
 819             self.logger.critical(
 820                 logging_text
 821                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 822                 exc_info=True,
 823             )
 824         finally:
 825             if exc:
 826                 if db_nsir:
 827                     db_nsir_update["_admin.deployed"] = nsir_deployed
 828                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 829                     db_nsir_update["operational-status"] = "failed"
 830                 if db_nsilcmop:
 831                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 832                         step, exc
 833                     )
 834                     db_nsilcmop_update[
 835                         "operationState"
 836                     ] = nsilcmop_operation_state = "FAILED"
 837                     db_nsilcmop_update["statusEnteredTime"] = time()
 838             try:
 839                 if db_nsir:
 840                     db_nsir_update["_admin.deployed"] = nsir_deployed
 841                     db_nsir_update["_admin.nsilcmop"] = None
 842                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 843                 if db_nsilcmop:
 844                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 845             except DbException as e:
 846                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 847
 848             if nsilcmop_operation_state:
 849                 try:
 850                     await self.msg.aiowrite(
 851                         "nsi",
 852                         "terminated",
 853                         {
 854                             "nsir_id": nsir_id,
 855                             "nsilcmop_id": nsilcmop_id,
 856                             "operationState": nsilcmop_operation_state,
 857                             "autoremove": autoremove,
 858                         },
 859                     )
 860                 except Exception as e:
 861                     self.logger.error(
 862                         logging_text + "kafka_write notification Exception {}".format(e)
 863                     )
 864             self.logger.debug(logging_text + "Exit")
 865             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")