osm_lcm/netslice.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import asyncio
  17 import logging
  18 import logging.handlers
  19 import traceback
  20 from osm_lcm import ROclient
  21 from osm_lcm.lcm_utils import (
  22     LcmException,
  23     LcmBase,
  24     populate_dict,
  25     get_iterable,
  26     deep_get,
  27 )
  28 from osm_common.dbbase import DbException
  29 from time import time
  30 from copy import deepcopy
  31
  32
  33 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
  34
  35
  36 class NetsliceLcm(LcmBase):
  37
  38     timeout_nsi_deploy = 2 * 3600  # default global timeout for deployment a nsi
  39
  40     def __init__(self, msg, lcm_tasks, config, loop, ns):
  41         """
  42         Init, Connect to database, filesystem storage, and messaging
  43         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
  44         :return: None
  45         """
  46         # logging
  47         self.logger = logging.getLogger("lcm.netslice")
  48         self.loop = loop
  49         self.lcm_tasks = lcm_tasks
  50         self.ns = ns
  51         self.ro_config = config["ro_config"]
  52         self.timeout = config["timeout"]
  53
  54         super().__init__(msg, self.logger)
  55
  56     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
  57         """
  58         Updates database nsir with the RO info for the created vld
  59         :param nsi_update_nsir: dictionary to be filled with the updated info
  60         :param db_nsir: content of db_nsir. This is also modified
  61         :param nsir_desc_RO: nsir descriptor from RO
  62         :return: Nothing, LcmException is raised on errors
  63         """
  64
  65         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
  66             for net_RO in get_iterable(nsir_desc_RO, "nets"):
  67                 if vld["id"] != net_RO.get("ns_net_osm_id"):
  68                     continue
  69                 vld["vim-id"] = net_RO.get("vim_net_id")
  70                 vld["name"] = net_RO.get("vim_name")
  71                 vld["status"] = net_RO.get("status")
  72                 vld["status-detailed"] = net_RO.get("error_msg")
  73                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
  74                 break
  75             else:
  76                 raise LcmException(
  77                     "ns_update_nsir: Not found vld={} at RO info".format(vld["id"])
  78                 )
  79
  80     async def instantiate(self, nsir_id, nsilcmop_id):
  81
  82         # Try to lock HA task here
  83         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
  84         if not task_is_locked_by_me:
  85             return
  86
  87         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
  88         self.logger.debug(logging_text + "Enter")
  89         # get all needed from database
  90         exc = None
  91         db_nsir = None
  92         db_nsilcmop = None
  93         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
  94         db_nsilcmop_update = {}
  95         nsilcmop_operation_state = None
  96         vim_2_RO = {}
  97         RO = ROclient.ROClient(self.loop, **self.ro_config)
  98         nsi_vld_instantiationi_params = {}
  99
 100         def ip_profile_2_RO(ip_profile):
 101             RO_ip_profile = deepcopy((ip_profile))
 102             if "dns-server" in RO_ip_profile:
 103                 if isinstance(RO_ip_profile["dns-server"], list):
 104                     RO_ip_profile["dns-address"] = []
 105                     for ds in RO_ip_profile.pop("dns-server"):
 106                         RO_ip_profile["dns-address"].append(ds["address"])
 107                 else:
 108                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
 109             if RO_ip_profile.get("ip-version") == "ipv4":
 110                 RO_ip_profile["ip-version"] = "IPv4"
 111             if RO_ip_profile.get("ip-version") == "ipv6":
 112                 RO_ip_profile["ip-version"] = "IPv6"
 113             if "dhcp-params" in RO_ip_profile:
 114                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
 115             return RO_ip_profile
 116
 117         def vim_account_2_RO(vim_account):
 118             """
 119             Translate a RO vim_account from OSM vim_account params
 120             :param ns_params: OSM instantiate params
 121             :return: The RO ns descriptor
 122             """
 123             if vim_account in vim_2_RO:
 124                 return vim_2_RO[vim_account]
 125
 126             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
 127             if db_vim["_admin"]["operationalState"] != "ENABLED":
 128                 raise LcmException(
 129                     "VIM={} is not available. operationalState={}".format(
 130                         vim_account, db_vim["_admin"]["operationalState"]
 131                     )
 132                 )
 133             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
 134             vim_2_RO[vim_account] = RO_vim_id
 135             return RO_vim_id
 136
 137         async def netslice_scenario_create(
 138             self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 139         ):
 140             """
 141             Create a network slice VLD through RO Scenario
 142             :param vld_id The VLD id inside nsir to be created
 143             :param nsir_id The nsir id
 144             """
 145             nonlocal nsi_vld_instantiationi_params
 146             ip_vld = None
 147             mgmt_network = False
 148             RO_vld_sites = []
 149             vld_id = vld_item["id"]
 150             netslice_vld = vld_item
 151             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
 152             # self.logger.debug(logging_text + "Enter")
 153
 154             vld_shared = None
 155             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
 156                 _filter = {
 157                     "_id.ne": nsir_id,
 158                     "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item,
 159                 }
 160                 shared_nsi = self.db.get_one(
 161                     "nsis", _filter, fail_on_empty=False, fail_on_more=False
 162                 )
 163                 if shared_nsi:
 164                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
 165                         if vld_id == vlds["vld_id"]:
 166                             vld_shared = {
 167                                 "instance_scenario_id": vlds["netslice_scenario_id"],
 168                                 "osm_id": vld_id,
 169                             }
 170                             break
 171                     break
 172
 173             # Creating netslice-vld at RO
 174             RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
 175
 176             if vld_id in RO_nsir:
 177                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
 178
 179             # If netslice-vld doesn't exists then create it
 180             else:
 181                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
 182                 # Updating network names and datacenters from instantiation parameters for each VLD
 183                 for instantiation_params_vld in get_iterable(
 184                     db_nsir["instantiation_parameters"], "netslice-vld"
 185                 ):
 186                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
 187                         ip_vld = deepcopy(instantiation_params_vld)
 188                         ip_vld.pop("name")
 189                         nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
 190
 191                 db_nsir_update_RO = {}
 192                 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
 193                 if self.ro_config["ng"]:
 194                     db_nsir_update_RO["netslice_scenario_id"] = (
 195                         vld_shared.get("instance_scenario_id")
 196                         if vld_shared
 197                         else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
 198                     )
 199                 else:  # if not self.ro_config["ng"]:
 200                     if netslice_vld.get("mgmt-network"):
 201                         mgmt_network = True
 202                     RO_ns_params = {}
 203                     RO_ns_params["name"] = netslice_vld["name"]
 204                     RO_ns_params["datacenter"] = vim_account_2_RO(
 205                         db_nsir["instantiation_parameters"]["vimAccountId"]
 206                     )
 207
 208                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
 209                     # Use vim-network-id instantiation parameter
 210                     vim_network_option = None
 211                     if ip_vld:
 212                         if ip_vld.get("vim-network-id"):
 213                             vim_network_option = "vim-network-id"
 214                         elif ip_vld.get("vim-network-name"):
 215                             vim_network_option = "vim-network-name"
 216                         if ip_vld.get("ip-profile"):
 217                             populate_dict(
 218                                 RO_ns_params,
 219                                 ("networks", netslice_vld["name"], "ip-profile"),
 220                                 ip_profile_2_RO(ip_vld["ip-profile"]),
 221                             )
 222
 223                     if vim_network_option:
 224                         if ip_vld.get(vim_network_option):
 225                             if isinstance(ip_vld.get(vim_network_option), list):
 226                                 for vim_net_id in ip_vld.get(vim_network_option):
 227                                     for vim_account, vim_net in vim_net_id.items():
 228                                         RO_vld_sites.append(
 229                                             {
 230                                                 "netmap-use": vim_net,
 231                                                 "datacenter": vim_account_2_RO(
 232                                                     vim_account
 233                                                 ),
 234                                             }
 235                                         )
 236                             elif isinstance(ip_vld.get(vim_network_option), dict):
 237                                 for vim_account, vim_net in ip_vld.get(
 238                                     vim_network_option
 239                                 ).items():
 240                                     RO_vld_sites.append(
 241                                         {
 242                                             "netmap-use": vim_net,
 243                                             "datacenter": vim_account_2_RO(vim_account),
 244                                         }
 245                                     )
 246                             else:
 247                                 RO_vld_sites.append(
 248                                     {
 249                                         "netmap-use": ip_vld[vim_network_option],
 250                                         "datacenter": vim_account_2_RO(
 251                                             netslice_vld["vimAccountId"]
 252                                         ),
 253                                     }
 254                                 )
 255
 256                     # Use default netslice vim-network-name from template
 257                     else:
 258                         for nss_conn_point_ref in get_iterable(
 259                             netslice_vld, "nss-connection-point-ref"
 260                         ):
 261                             if nss_conn_point_ref.get("vimAccountId"):
 262                                 if (
 263                                     nss_conn_point_ref["vimAccountId"]
 264                                     != netslice_vld["vimAccountId"]
 265                                 ):
 266                                     RO_vld_sites.append(
 267                                         {
 268                                             "netmap-create": None,
 269                                             "datacenter": vim_account_2_RO(
 270                                                 nss_conn_point_ref["vimAccountId"]
 271                                             ),
 272                                         }
 273                                     )
 274
 275                     if vld_shared:
 276                         populate_dict(
 277                             RO_ns_params,
 278                             ("networks", netslice_vld["name"], "use-network"),
 279                             vld_shared,
 280                         )
 281
 282                     if RO_vld_sites:
 283                         populate_dict(
 284                             RO_ns_params,
 285                             ("networks", netslice_vld["name"], "sites"),
 286                             RO_vld_sites,
 287                         )
 288
 289                     RO_ns_params["scenario"] = {
 290                         "nets": [
 291                             {
 292                                 "name": netslice_vld["name"],
 293                                 "external": mgmt_network,
 294                                 "type": "bridge",
 295                             }
 296                         ]
 297                     }
 298
 299                     # self.logger.debug(logging_text + step)
 300                     desc = await RO.create("ns", descriptor=RO_ns_params)
 301                     db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
 302                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
 303
 304         def overwrite_nsd_params(self, db_nsir, nslcmop):
 305             nonlocal nsi_vld_instantiationi_params
 306             nonlocal db_nsir_update
 307             vld_op_list = []
 308             vld = None
 309             nsr_id = nslcmop.get("nsInstanceId")
 310             # Overwrite instantiation parameters in netslice runtime
 311             RO_list = db_nsir_admin["deployed"]["RO"]
 312
 313             for ro_item_index, RO_item in enumerate(RO_list):
 314                 netslice_vld = next(
 315                     (
 316                         n
 317                         for n in get_iterable(db_nsir["_admin"], "netslice-vld")
 318                         if RO_item.get("vld_id") == n.get("id")
 319                     ),
 320                     None,
 321                 )
 322                 if not netslice_vld:
 323                     continue
 324                 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
 325                 # Search the cp of netslice-vld that match with nst:netslice-subnet
 326                 for nss_cp_item in get_iterable(
 327                     netslice_vld, "nss-connection-point-ref"
 328                 ):
 329                     # Search the netslice-subnet of nst that match
 330                     nss = next(
 331                         (
 332                             nss
 333                             for nss in get_iterable(
 334                                 db_nsir["_admin"], "netslice-subnet"
 335                             )
 336                             if nss_cp_item["nss-ref"] == nss["nss-id"]
 337                         ),
 338                         None,
 339                     )
 340                     # Compare nss-ref equal nss from nst
 341                     if not nss:
 342                         continue
 343                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
 344                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
 345                     for cp_nsd in db_nsds.get("sapd", ()):
 346                         if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
 347                             if nslcmop.get("operationParams"):
 348                                 if (
 349                                     nslcmop["operationParams"].get("nsName")
 350                                     == nss["nsName"]
 351                                 ):
 352                                     vld_id = RO_item["vld_id"]
 353                                     netslice_scenario_id = RO_item[
 354                                         "netslice_scenario_id"
 355                                     ]
 356                                     nslcmop_vld = {}
 357                                     nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
 358                                     for vld in get_iterable(
 359                                         nslcmop["operationParams"], "vld"
 360                                     ):
 361                                         if vld["name"] == cp_nsd["virtual-link-desc"]:
 362                                             nslcmop_vld.update(vld)
 363                                     if self.ro_config["ng"]:
 364                                         nslcmop_vld["common_id"] = netslice_scenario_id
 365                                         nslcmop_vld.update(
 366                                             nsi_vld_instantiationi_params.get(
 367                                                 RO_item["vld_id"], {}
 368                                             )
 369                                         )
 370                                     else:
 371                                         nslcmop_vld["ns-net"] = {
 372                                             vld_id: netslice_scenario_id
 373                                         }
 374                                     vld_op_list.append(nslcmop_vld)
 375             nslcmop["operationParams"]["vld"] = vld_op_list
 376             self.update_db_2(
 377                 "nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list}
 378             )
 379             return nsr_id, nslcmop
 380
 381         try:
 382             # wait for any previous tasks in process
 383             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 384
 385             step = "Getting nsir={} from db".format(nsir_id)
 386             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 387             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 388             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 389
 390             start_deploy = time()
 391             nsi_params = db_nsilcmop.get("operationParams")
 392             if nsi_params and nsi_params.get("timeout_nsi_deploy"):
 393                 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
 394             else:
 395                 timeout_nsi_deploy = self.timeout.get(
 396                     "nsi_deploy", self.timeout_nsi_deploy
 397                 )
 398
 399             # Empty list to keep track of network service records status in the netslice
 400             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
 401
 402             step = "Creating slice operational-status init"
 403             # Slice status Creating
 404             db_nsir_update["detailed-status"] = "creating"
 405             db_nsir_update["operational-status"] = "init"
 406             db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
 407
 408             step = "Instantiating netslice VLDs before NS instantiation"
 409             # Creating netslice VLDs networking before NS instantiation
 410             db_nsir_update["detailed-status"] = step
 411             self.update_db_2("nsis", nsir_id, db_nsir_update)
 412             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
 413             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
 414                 await netslice_scenario_create(
 415                     self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
 416                 )
 417
 418             step = "Instantiating netslice subnets"
 419             db_nsir_update["detailed-status"] = step
 420             self.update_db_2("nsis", nsir_id, db_nsir_update)
 421
 422             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 423
 424             # Check status of the VLDs and wait for creation
 425             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
 426             # db_nsir_update_RO = deepcopy(netslice_scenarios)
 427             # for netslice_scenario in netslice_scenarios:
 428             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
 429             #                                  nsir_id, db_nsir_update_RO)
 430
 431             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
 432             # self.update_db_2("nsis", nsir_id, db_nsir_update)
 433
 434             # Iterate over the network services operation ids to instantiate NSs
 435             step = "Instantiating Netslice Subnets"
 436             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 437             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 438             for nslcmop_id in nslcmop_ids:
 439                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 440                 # Overwriting netslice-vld vim-net-id to ns
 441                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
 442                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
 443                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
 444                 self.lcm_tasks.register(
 445                     "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 446                 )
 447
 448             # Wait until Network Slice is ready
 449             step = " Waiting nsi ready."
 450             nsrs_detailed_list_old = None
 451             self.logger.debug(logging_text + step)
 452
 453             # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
 454             while time() <= start_deploy + timeout_nsi_deploy:
 455                 # Check ns instantiation status
 456                 nsi_ready = True
 457                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
 458                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
 459                 nsrs_detailed_list_new = []
 460                 for nslcmop_item in nslcmop_ids:
 461                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 462                     status = nslcmop.get("operationState")
 463                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 464                     for nss in nsrs_detailed_list:
 465                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 466                             nss.update(
 467                                 {
 468                                     "nsrId": nslcmop["nsInstanceId"],
 469                                     "status": nslcmop["operationState"],
 470                                     "detailed-status": nslcmop.get("detailed-status"),
 471                                     "instantiated": True,
 472                                 }
 473                             )
 474                             nsrs_detailed_list_new.append(nss)
 475                     if status not in [
 476                         "COMPLETED",
 477                         "PARTIALLY_COMPLETED",
 478                         "FAILED",
 479                         "FAILED_TEMP",
 480                     ]:
 481                         nsi_ready = False
 482
 483                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 484                     nsrs_detailed_list_old = nsrs_detailed_list_new
 485                     self.update_db_2(
 486                         "nsis",
 487                         nsir_id,
 488                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 489                     )
 490
 491                 if nsi_ready:
 492                     error_list = []
 493                     step = "Network Slice Instance instantiated"
 494                     for nss in nsrs_detailed_list:
 495                         if nss["status"] in ("FAILED", "FAILED_TEMP"):
 496                             error_list.append(
 497                                 "NS {} {}: {}".format(
 498                                     nss["nsrId"], nss["status"], nss["detailed-status"]
 499                                 )
 500                             )
 501                     if error_list:
 502                         step = "instantiating"
 503                         raise LcmException("; ".join(error_list))
 504                     break
 505
 506                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
 507                 await asyncio.sleep(5, loop=self.loop)
 508
 509             else:  # timeout_nsi_deploy reached:
 510                 raise LcmException("Timeout waiting nsi to be ready.")
 511
 512             db_nsir_update["operational-status"] = "running"
 513             db_nsir_update["detailed-status"] = "done"
 514             db_nsir_update["config-status"] = "configured"
 515             db_nsilcmop_update[
 516                 "operationState"
 517             ] = nsilcmop_operation_state = "COMPLETED"
 518             db_nsilcmop_update["statusEnteredTime"] = time()
 519             db_nsilcmop_update["detailed-status"] = "done"
 520             return
 521
 522         except (LcmException, DbException) as e:
 523             self.logger.error(
 524                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 525             )
 526             exc = e
 527         except asyncio.CancelledError:
 528             self.logger.error(
 529                 logging_text + "Cancelled Exception while '{}'".format(step)
 530             )
 531             exc = "Operation was cancelled"
 532         except Exception as e:
 533             exc = traceback.format_exc()
 534             self.logger.critical(
 535                 logging_text
 536                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 537                 exc_info=True,
 538             )
 539         finally:
 540             if exc:
 541                 if db_nsir:
 542                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 543                     db_nsir_update["operational-status"] = "failed"
 544                     db_nsir_update["config-status"] = "configured"
 545                 if db_nsilcmop:
 546                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 547                         step, exc
 548                     )
 549                     db_nsilcmop_update[
 550                         "operationState"
 551                     ] = nsilcmop_operation_state = "FAILED"
 552                     db_nsilcmop_update["statusEnteredTime"] = time()
 553             try:
 554                 if db_nsir:
 555                     db_nsir_update["_admin.nsilcmop"] = None
 556                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 557                 if db_nsilcmop:
 558                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 559             except DbException as e:
 560                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 561             if nsilcmop_operation_state:
 562                 try:
 563                     await self.msg.aiowrite(
 564                         "nsi",
 565                         "instantiated",
 566                         {
 567                             "nsir_id": nsir_id,
 568                             "nsilcmop_id": nsilcmop_id,
 569                             "operationState": nsilcmop_operation_state,
 570                         },
 571                     )
 572                 except Exception as e:
 573                     self.logger.error(
 574                         logging_text + "kafka_write notification Exception {}".format(e)
 575                     )
 576             self.logger.debug(logging_text + "Exit")
 577             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
 578
 579     async def terminate(self, nsir_id, nsilcmop_id):
 580
 581         # Try to lock HA task here
 582         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
 583         if not task_is_locked_by_me:
 584             return
 585
 586         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
 587         self.logger.debug(logging_text + "Enter")
 588         exc = None
 589         db_nsir = None
 590         db_nsilcmop = None
 591         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
 592         db_nsilcmop_update = {}
 593         RO = ROclient.ROClient(self.loop, **self.ro_config)
 594         nsir_deployed = None
 595         failed_detail = []  # annotates all failed error messages
 596         nsilcmop_operation_state = None
 597         autoremove = False  # autoremove after terminated
 598         try:
 599             # wait for any previous tasks in process
 600             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
 601
 602             step = "Getting nsir={} from db".format(nsir_id)
 603             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 604             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
 605             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
 606             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
 607
 608             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
 609             # CASE: Instance was terminated but there is a second request to terminate the instance
 610             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
 611                 return
 612
 613             # Slice status Terminating
 614             db_nsir_update["operational-status"] = "terminating"
 615             db_nsir_update["config-status"] = "terminating"
 616             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
 617             self.update_db_2("nsis", nsir_id, db_nsir_update)
 618
 619             # Gets the list to keep track of network service records status in the netslice
 620             nsrs_detailed_list = []
 621
 622             # Iterate over the network services operation ids to terminate NSs
 623             # TODO: (future improvement) look another way check the tasks instead of keep asking
 624             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
 625             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
 626             step = "Terminating Netslice Subnets"
 627             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
 628             nslcmop_new = []
 629             for nslcmop_id in nslcmop_ids:
 630                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
 631                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
 632                 nss_in_use = self.db.get_list(
 633                     "nsis",
 634                     {
 635                         "_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
 636                         "operational-status": {"$nin": ["terminated", "failed"]},
 637                     },
 638                 )
 639                 if len(nss_in_use) < 2:
 640                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
 641                     self.lcm_tasks.register(
 642                         "ns", nsr_id, nslcmop_id, "ns_instantiate", task
 643                     )
 644                     nslcmop_new.append(nslcmop_id)
 645                 else:
 646                     # Update shared nslcmop shared with active nsi
 647                     netsliceInstanceId = db_nsir["_id"]
 648                     for nsis_item in nss_in_use:
 649                         if db_nsir["_id"] != nsis_item["_id"]:
 650                             netsliceInstanceId = nsis_item["_id"]
 651                             break
 652                     self.db.set_one(
 653                         "nslcmops",
 654                         {"_id": nslcmop_id},
 655                         {"operationParams.netsliceInstanceId": netsliceInstanceId},
 656                     )
 657             self.db.set_one(
 658                 "nsilcmops",
 659                 {"_id": nsilcmop_id},
 660                 {"operationParams.nslcmops_ids": nslcmop_new},
 661             )
 662
 663             # Wait until Network Slice is terminated
 664             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(
 665                 nsir_id
 666             )
 667             nsrs_detailed_list_old = None
 668             self.logger.debug(logging_text + step)
 669
 670             termination_timeout = 2 * 3600  # Two hours
 671             while termination_timeout > 0:
 672                 # Check ns termination status
 673                 nsi_ready = True
 674                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 675                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 676                 nsrs_detailed_list_new = []
 677                 for nslcmop_item in nslcmop_ids:
 678                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
 679                     status = nslcmop["operationState"]
 680                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
 681                     for nss in nsrs_detailed_list:
 682                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
 683                             nss.update(
 684                                 {
 685                                     "nsrId": nslcmop["nsInstanceId"],
 686                                     "status": nslcmop["operationState"],
 687                                     "detailed-status": nsir_status_detailed
 688                                     + "; {}".format(nslcmop.get("detailed-status")),
 689                                 }
 690                             )
 691                             nsrs_detailed_list_new.append(nss)
 692                     if status not in [
 693                         "COMPLETED",
 694                         "PARTIALLY_COMPLETED",
 695                         "FAILED",
 696                         "FAILED_TEMP",
 697                     ]:
 698                         nsi_ready = False
 699
 700                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
 701                     nsrs_detailed_list_old = nsrs_detailed_list_new
 702                     self.update_db_2(
 703                         "nsis",
 704                         nsir_id,
 705                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
 706                     )
 707
 708                 if nsi_ready:
 709                     # Check if it is the last used nss and mark isinstantiate: False
 710                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
 711                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
 712                     for nss in nsrs_detailed_list:
 713                         _filter = {
 714                             "_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
 715                             "operational-status.ne": "terminated",
 716                             "_id.ne": nsir_id,
 717                         }
 718                         nsis_list = self.db.get_one(
 719                             "nsis", _filter, fail_on_empty=False, fail_on_more=False
 720                         )
 721                         if not nsis_list:
 722                             nss.update({"instantiated": False})
 723
 724                     step = "Network Slice Instance is terminated. nsi_id={}".format(
 725                         nsir_id
 726                     )
 727                     for items in nsrs_detailed_list:
 728                         if "FAILED" in items.values():
 729                             raise LcmException(
 730                                 "Error terminating NSI: {}".format(nsir_id)
 731                             )
 732                     break
 733
 734                 await asyncio.sleep(5, loop=self.loop)
 735                 termination_timeout -= 5
 736
 737             if termination_timeout <= 0:
 738                 raise LcmException(
 739                     "Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id)
 740                 )
 741
 742             # Delete netslice-vlds
 743             RO_nsir_id = RO_delete_action = None
 744             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
 745                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
 746                 try:
 747                     if not self.ro_config["ng"]:
 748                         step = db_nsir_update[
 749                             "detailed-status"
 750                         ] = "Deleting netslice-vld at RO"
 751                         db_nsilcmop_update[
 752                             "detailed-status"
 753                         ] = "Deleting netslice-vld at RO"
 754                         self.logger.debug(logging_text + step)
 755                         desc = await RO.delete("ns", RO_nsir_id)
 756                         RO_delete_action = desc["action_id"]
 757                         nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
 758                         nsir_deployed_RO["vld_status"] = "DELETING"
 759                         db_nsir_update["_admin.deployed"] = nsir_deployed
 760                         self.update_db_2("nsis", nsir_id, db_nsir_update)
 761                         if RO_delete_action:
 762                             # wait until NS is deleted from VIM
 763                             step = "Waiting ns deleted from VIM. RO_id={}".format(
 764                                 RO_nsir_id
 765                             )
 766                             self.logger.debug(logging_text + step)
 767                 except ROclient.ROClientException as e:
 768                     if e.http_code == 404:  # not found
 769                         nsir_deployed_RO["vld_id"] = None
 770                         nsir_deployed_RO["vld_status"] = "DELETED"
 771                         self.logger.debug(
 772                             logging_text
 773                             + "RO_ns_id={} already deleted".format(RO_nsir_id)
 774                         )
 775                     elif e.http_code == 409:  # conflict
 776                         failed_detail.append(
 777                             "RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e)
 778                         )
 779                         self.logger.debug(logging_text + failed_detail[-1])
 780                     else:
 781                         failed_detail.append(
 782                             "RO_ns_id={} delete error: {}".format(RO_nsir_id, e)
 783                         )
 784                         self.logger.error(logging_text + failed_detail[-1])
 785
 786                 if failed_detail:
 787                     self.logger.error(logging_text + " ;".join(failed_detail))
 788                     db_nsir_update["operational-status"] = "failed"
 789                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(
 790                         failed_detail
 791                     )
 792                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
 793                     db_nsilcmop_update[
 794                         "operationState"
 795                     ] = nsilcmop_operation_state = "FAILED"
 796                     db_nsilcmop_update["statusEnteredTime"] = time()
 797                 else:
 798                     db_nsir_update["operational-status"] = "terminating"
 799                     db_nsir_update["config-status"] = "terminating"
 800                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
 801                     db_nsilcmop_update[
 802                         "operationState"
 803                     ] = nsilcmop_operation_state = "COMPLETED"
 804                     db_nsilcmop_update["statusEnteredTime"] = time()
 805                     if db_nsilcmop["operationParams"].get("autoremove"):
 806                         autoremove = True
 807
 808             db_nsir_update["detailed-status"] = "done"
 809             db_nsir_update["operational-status"] = "terminated"
 810             db_nsir_update["config-status"] = "terminated"
 811             db_nsilcmop_update["statusEnteredTime"] = time()
 812             db_nsilcmop_update["detailed-status"] = "done"
 813             return
 814
 815         except (LcmException, DbException) as e:
 816             self.logger.error(
 817                 logging_text + "Exit Exception while '{}': {}".format(step, e)
 818             )
 819             exc = e
 820         except asyncio.CancelledError:
 821             self.logger.error(
 822                 logging_text + "Cancelled Exception while '{}'".format(step)
 823             )
 824             exc = "Operation was cancelled"
 825         except Exception as e:
 826             exc = traceback.format_exc()
 827             self.logger.critical(
 828                 logging_text
 829                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
 830                 exc_info=True,
 831             )
 832         finally:
 833             if exc:
 834                 if db_nsir:
 835                     db_nsir_update["_admin.deployed"] = nsir_deployed
 836                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
 837                     db_nsir_update["operational-status"] = "failed"
 838                 if db_nsilcmop:
 839                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
 840                         step, exc
 841                     )
 842                     db_nsilcmop_update[
 843                         "operationState"
 844                     ] = nsilcmop_operation_state = "FAILED"
 845                     db_nsilcmop_update["statusEnteredTime"] = time()
 846             try:
 847                 if db_nsir:
 848                     db_nsir_update["_admin.deployed"] = nsir_deployed
 849                     db_nsir_update["_admin.nsilcmop"] = None
 850                     self.update_db_2("nsis", nsir_id, db_nsir_update)
 851                 if db_nsilcmop:
 852                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
 853             except DbException as e:
 854                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
 855
 856             if nsilcmop_operation_state:
 857                 try:
 858                     await self.msg.aiowrite(
 859                         "nsi",
 860                         "terminated",
 861                         {
 862                             "nsir_id": nsir_id,
 863                             "nsilcmop_id": nsilcmop_id,
 864                             "operationState": nsilcmop_operation_state,
 865                             "autoremove": autoremove,
 866                         },
 867                         loop=self.loop,
 868                     )
 869                 except Exception as e:
 870                     self.logger.error(
 871                         logging_text + "kafka_write notification Exception {}".format(e)
 872                     )
 873             self.logger.debug(logging_text + "Exit")
 874             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")