Code Coverage

Cobertura Coverage Report > osm_lcm >

netslice.py

Trend

File Coverage summary

NameClassesLinesConditionals
netslice.py
100%
1/1
3%
15/440
100%
0/0

Coverage Breakdown by Class

NameLinesConditionals
netslice.py
3%
15/440
N/A

Source

osm_lcm/netslice.py
1 # -*- coding: utf-8 -*-
2 ##
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 #         http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14 ##
15
16 1 import asyncio
17 1 import logging
18 1 import logging.handlers
19 1 import traceback
20 1 from osm_lcm import ROclient
21 1 from osm_lcm.lcm_utils import (
22     LcmException,
23     LcmBase,
24     populate_dict,
25     get_iterable,
26     deep_get,
27 )
28 1 from osm_common.dbbase import DbException
29 1 from time import time
30 1 from copy import deepcopy
31
32
33 1 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
34
35
36 1 class NetsliceLcm(LcmBase):
37 1     def __init__(self, msg, lcm_tasks, config, loop, ns):
38         """
39         Init, Connect to database, filesystem storage, and messaging
40         :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
41         :return: None
42         """
43         # logging
44 0         self.logger = logging.getLogger("lcm.netslice")
45 0         self.loop = loop
46 0         self.lcm_tasks = lcm_tasks
47 0         self.ns = ns
48 0         self.ro_config = config["RO"]
49 0         self.timeout = config["timeout"]
50
51 0         super().__init__(msg, self.logger)
52
53 1     def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
54         """
55         Updates database nsir with the RO info for the created vld
56         :param nsi_update_nsir: dictionary to be filled with the updated info
57         :param db_nsir: content of db_nsir. This is also modified
58         :param nsir_desc_RO: nsir descriptor from RO
59         :return: Nothing, LcmException is raised on errors
60         """
61
62 0         for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
63 0             for net_RO in get_iterable(nsir_desc_RO, "nets"):
64 0                 if vld["id"] != net_RO.get("ns_net_osm_id"):
65 0                     continue
66 0                 vld["vim-id"] = net_RO.get("vim_net_id")
67 0                 vld["name"] = net_RO.get("vim_name")
68 0                 vld["status"] = net_RO.get("status")
69 0                 vld["status-detailed"] = net_RO.get("error_msg")
70 0                 nsi_update_nsir["vld.{}".format(vld_index)] = vld
71 0                 break
72             else:
73 0                 raise LcmException(
74                     "ns_update_nsir: Not found vld={} at RO info".format(vld["id"])
75                 )
76
77 1     async def instantiate(self, nsir_id, nsilcmop_id):
78         # Try to lock HA task here
79 0         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
80 0         if not task_is_locked_by_me:
81 0             return
82
83 0         logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
84 0         self.logger.debug(logging_text + "Enter")
85         # get all needed from database
86 0         exc = None
87 0         db_nsir = None
88 0         db_nsilcmop = None
89 0         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
90 0         db_nsilcmop_update = {}
91 0         nsilcmop_operation_state = None
92 0         vim_2_RO = {}
93 0         RO = ROclient.ROClient(self.loop, **self.ro_config)
94 0         nsi_vld_instantiationi_params = {}
95
96 0         def ip_profile_2_RO(ip_profile):
97 0             RO_ip_profile = deepcopy((ip_profile))
98 0             if "dns-server" in RO_ip_profile:
99 0                 if isinstance(RO_ip_profile["dns-server"], list):
100 0                     RO_ip_profile["dns-address"] = []
101 0                     for ds in RO_ip_profile.pop("dns-server"):
102 0                         RO_ip_profile["dns-address"].append(ds["address"])
103                 else:
104 0                     RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
105 0             if RO_ip_profile.get("ip-version") == "ipv4":
106 0                 RO_ip_profile["ip-version"] = "IPv4"
107 0             if RO_ip_profile.get("ip-version") == "ipv6":
108 0                 RO_ip_profile["ip-version"] = "IPv6"
109 0             if "dhcp-params" in RO_ip_profile:
110 0                 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
111 0             return RO_ip_profile
112
113 0         def vim_account_2_RO(vim_account):
114             """
115             Translate a RO vim_account from OSM vim_account params
116             :param ns_params: OSM instantiate params
117             :return: The RO ns descriptor
118             """
119 0             if vim_account in vim_2_RO:
120 0                 return vim_2_RO[vim_account]
121
122 0             db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
123 0             if db_vim["_admin"]["operationalState"] != "ENABLED":
124 0                 raise LcmException(
125                     "VIM={} is not available. operationalState={}".format(
126                         vim_account, db_vim["_admin"]["operationalState"]
127                     )
128                 )
129 0             RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
130 0             vim_2_RO[vim_account] = RO_vim_id
131 0             return RO_vim_id
132
133 0         async def netslice_scenario_create(
134             self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
135         ):
136             """
137             Create a network slice VLD through RO Scenario
138             :param vld_id The VLD id inside nsir to be created
139             :param nsir_id The nsir id
140             """
141             nonlocal nsi_vld_instantiationi_params
142 0             ip_vld = None
143 0             mgmt_network = False
144 0             RO_vld_sites = []
145 0             vld_id = vld_item["id"]
146 0             netslice_vld = vld_item
147             # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
148             # self.logger.debug(logging_text + "Enter")
149
150 0             vld_shared = None
151 0             for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
152 0                 _filter = {
153                     "_id.ne": nsir_id,
154                     "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item,
155                 }
156 0                 shared_nsi = self.db.get_one(
157                     "nsis", _filter, fail_on_empty=False, fail_on_more=False
158                 )
159 0                 if shared_nsi:
160 0                     for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
161 0                         if vld_id == vlds["vld_id"]:
162 0                             vld_shared = {
163                                 "instance_scenario_id": vlds["netslice_scenario_id"],
164                                 "osm_id": vld_id,
165                             }
166 0                             break
167 0                     break
168
169             # Creating netslice-vld at RO
170 0             RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
171
172 0             if vld_id in RO_nsir:
173 0                 db_nsir_update["_admin.deployed.RO"] = RO_nsir
174
175             # If netslice-vld doesn't exists then create it
176             else:
177                 # TODO: Check VDU type in all descriptors finding SRIOV / PT
178                 # Updating network names and datacenters from instantiation parameters for each VLD
179 0                 for instantiation_params_vld in get_iterable(
180                     db_nsir["instantiation_parameters"], "netslice-vld"
181                 ):
182 0                     if instantiation_params_vld.get("name") == netslice_vld["name"]:
183 0                         ip_vld = deepcopy(instantiation_params_vld)
184 0                         ip_vld.pop("name")
185 0                         nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
186
187 0                 db_nsir_update_RO = {}
188 0                 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
189 0                 if self.ro_config["ng"]:
190 0                     db_nsir_update_RO["netslice_scenario_id"] = (
191                         vld_shared.get("instance_scenario_id")
192                         if vld_shared
193                         else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
194                     )
195                 else:  # if not self.ro_config["ng"]:
196 0                     if netslice_vld.get("mgmt-network"):
197 0                         mgmt_network = True
198 0                     RO_ns_params = {}
199 0                     RO_ns_params["name"] = netslice_vld["name"]
200 0                     RO_ns_params["datacenter"] = vim_account_2_RO(
201                         db_nsir["instantiation_parameters"]["vimAccountId"]
202                     )
203
204                     # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
205                     # Use vim-network-id instantiation parameter
206 0                     vim_network_option = None
207 0                     if ip_vld:
208 0                         if ip_vld.get("vim-network-id"):
209 0                             vim_network_option = "vim-network-id"
210 0                         elif ip_vld.get("vim-network-name"):
211 0                             vim_network_option = "vim-network-name"
212 0                         if ip_vld.get("ip-profile"):
213 0                             populate_dict(
214                                 RO_ns_params,
215                                 ("networks", netslice_vld["name"], "ip-profile"),
216                                 ip_profile_2_RO(ip_vld["ip-profile"]),
217                             )
218
219 0                     if vim_network_option:
220 0                         if ip_vld.get(vim_network_option):
221 0                             if isinstance(ip_vld.get(vim_network_option), list):
222 0                                 for vim_net_id in ip_vld.get(vim_network_option):
223 0                                     for vim_account, vim_net in vim_net_id.items():
224 0                                         RO_vld_sites.append(
225                                             {
226                                                 "netmap-use": vim_net,
227                                                 "datacenter": vim_account_2_RO(
228                                                     vim_account
229                                                 ),
230                                             }
231                                         )
232 0                             elif isinstance(ip_vld.get(vim_network_option), dict):
233 0                                 for vim_account, vim_net in ip_vld.get(
234                                     vim_network_option
235                                 ).items():
236 0                                     RO_vld_sites.append(
237                                         {
238                                             "netmap-use": vim_net,
239                                             "datacenter": vim_account_2_RO(vim_account),
240                                         }
241                                     )
242                             else:
243 0                                 RO_vld_sites.append(
244                                     {
245                                         "netmap-use": ip_vld[vim_network_option],
246                                         "datacenter": vim_account_2_RO(
247                                             netslice_vld["vimAccountId"]
248                                         ),
249                                     }
250                                 )
251
252                     # Use default netslice vim-network-name from template
253                     else:
254 0                         for nss_conn_point_ref in get_iterable(
255                             netslice_vld, "nss-connection-point-ref"
256                         ):
257 0                             if nss_conn_point_ref.get("vimAccountId"):
258 0                                 if (
259                                     nss_conn_point_ref["vimAccountId"]
260                                     != netslice_vld["vimAccountId"]
261                                 ):
262 0                                     RO_vld_sites.append(
263                                         {
264                                             "netmap-create": None,
265                                             "datacenter": vim_account_2_RO(
266                                                 nss_conn_point_ref["vimAccountId"]
267                                             ),
268                                         }
269                                     )
270
271 0                     if vld_shared:
272 0                         populate_dict(
273                             RO_ns_params,
274                             ("networks", netslice_vld["name"], "use-network"),
275                             vld_shared,
276                         )
277
278 0                     if RO_vld_sites:
279 0                         populate_dict(
280                             RO_ns_params,
281                             ("networks", netslice_vld["name"], "sites"),
282                             RO_vld_sites,
283                         )
284
285 0                     RO_ns_params["scenario"] = {
286                         "nets": [
287                             {
288                                 "name": netslice_vld["name"],
289                                 "external": mgmt_network,
290                                 "type": "bridge",
291                             }
292                         ]
293                     }
294
295                     # self.logger.debug(logging_text + step)
296 0                     desc = await RO.create("ns", descriptor=RO_ns_params)
297 0                     db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
298 0                 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
299
300 0         def overwrite_nsd_params(self, db_nsir, nslcmop):
301             nonlocal nsi_vld_instantiationi_params
302             nonlocal db_nsir_update
303 0             vld_op_list = []
304 0             vld = None
305 0             nsr_id = nslcmop.get("nsInstanceId")
306             # Overwrite instantiation parameters in netslice runtime
307 0             RO_list = db_nsir_admin["deployed"]["RO"]
308
309 0             for ro_item_index, RO_item in enumerate(RO_list):
310 0                 netslice_vld = next(
311                     (
312                         n
313                         for n in get_iterable(db_nsir["_admin"], "netslice-vld")
314                         if RO_item.get("vld_id") == n.get("id")
315                     ),
316                     None,
317                 )
318 0                 if not netslice_vld:
319 0                     continue
320                 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
321                 # Search the cp of netslice-vld that match with nst:netslice-subnet
322 0                 for nss_cp_item in get_iterable(
323                     netslice_vld, "nss-connection-point-ref"
324                 ):
325                     # Search the netslice-subnet of nst that match
326 0                     nss = next(
327                         (
328                             nss
329                             for nss in get_iterable(
330                                 db_nsir["_admin"], "netslice-subnet"
331                             )
332                             if nss_cp_item["nss-ref"] == nss["nss-id"]
333                         ),
334                         None,
335                     )
336                     # Compare nss-ref equal nss from nst
337 0                     if not nss:
338 0                         continue
339 0                     db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
340                     # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
341 0                     for cp_nsd in db_nsds.get("sapd", ()):
342 0                         if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
343 0                             if nslcmop.get("operationParams"):
344 0                                 if (
345                                     nslcmop["operationParams"].get("nsName")
346                                     == nss["nsName"]
347                                 ):
348 0                                     vld_id = RO_item["vld_id"]
349 0                                     netslice_scenario_id = RO_item[
350                                         "netslice_scenario_id"
351                                     ]
352 0                                     nslcmop_vld = {}
353 0                                     nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
354 0                                     for vld in get_iterable(
355                                         nslcmop["operationParams"], "vld"
356                                     ):
357 0                                         if vld["name"] == cp_nsd["virtual-link-desc"]:
358 0                                             nslcmop_vld.update(vld)
359 0                                     if self.ro_config["ng"]:
360 0                                         nslcmop_vld["common_id"] = netslice_scenario_id
361 0                                         nslcmop_vld.update(
362                                             nsi_vld_instantiationi_params.get(
363                                                 RO_item["vld_id"], {}
364                                             )
365                                         )
366                                     else:
367 0                                         nslcmop_vld["ns-net"] = {
368                                             vld_id: netslice_scenario_id
369                                         }
370 0                                     vld_op_list.append(nslcmop_vld)
371 0             nslcmop["operationParams"]["vld"] = vld_op_list
372 0             self.update_db_2(
373                 "nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list}
374             )
375 0             return nsr_id, nslcmop
376
377 0         try:
378             # wait for any previous tasks in process
379 0             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
380
381 0             step = "Getting nsir={} from db".format(nsir_id)
382 0             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
383 0             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
384 0             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
385
386 0             start_deploy = time()
387 0             nsi_params = db_nsilcmop.get("operationParams")
388 0             if nsi_params and nsi_params.get("timeout_nsi_deploy"):
389 0                 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
390             else:
391 0                 timeout_nsi_deploy = self.timeout.get("nsi_deploy")
392
393             # Empty list to keep track of network service records status in the netslice
394 0             nsir_admin = db_nsir_admin = db_nsir.get("_admin")
395
396 0             step = "Creating slice operational-status init"
397             # Slice status Creating
398 0             db_nsir_update["detailed-status"] = "creating"
399 0             db_nsir_update["operational-status"] = "init"
400 0             db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
401
402 0             step = "Instantiating netslice VLDs before NS instantiation"
403             # Creating netslice VLDs networking before NS instantiation
404 0             db_nsir_update["detailed-status"] = step
405 0             self.update_db_2("nsis", nsir_id, db_nsir_update)
406 0             db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
407 0             for vld_item in get_iterable(nsir_admin, "netslice-vld"):
408 0                 await netslice_scenario_create(
409                     self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
410                 )
411
412 0             step = "Instantiating netslice subnets"
413 0             db_nsir_update["detailed-status"] = step
414 0             self.update_db_2("nsis", nsir_id, db_nsir_update)
415
416 0             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
417
418             # Check status of the VLDs and wait for creation
419             # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
420             # db_nsir_update_RO = deepcopy(netslice_scenarios)
421             # for netslice_scenario in netslice_scenarios:
422             #    await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
423             #                                  nsir_id, db_nsir_update_RO)
424
425             # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
426             # self.update_db_2("nsis", nsir_id, db_nsir_update)
427
428             # Iterate over the network services operation ids to instantiate NSs
429 0             step = "Instantiating Netslice Subnets"
430 0             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
431 0             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
432 0             for nslcmop_id in nslcmop_ids:
433 0                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
434                 # Overwriting netslice-vld vim-net-id to ns
435 0                 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
436 0                 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
437 0                 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
438 0                 self.lcm_tasks.register(
439                     "ns", nsr_id, nslcmop_id, "ns_instantiate", task
440                 )
441
442             # Wait until Network Slice is ready
443 0             step = " Waiting nsi ready."
444 0             nsrs_detailed_list_old = None
445 0             self.logger.debug(logging_text + step)
446
447             # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
448 0             while time() <= start_deploy + timeout_nsi_deploy:
449                 # Check ns instantiation status
450 0                 nsi_ready = True
451 0                 nsir = self.db.get_one("nsis", {"_id": nsir_id})
452 0                 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
453 0                 nsrs_detailed_list_new = []
454 0                 for nslcmop_item in nslcmop_ids:
455 0                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
456 0                     status = nslcmop.get("operationState")
457                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
458 0                     for nss in nsrs_detailed_list:
459 0                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
460 0                             nss.update(
461                                 {
462                                     "nsrId": nslcmop["nsInstanceId"],
463                                     "status": nslcmop["operationState"],
464                                     "detailed-status": nslcmop.get("detailed-status"),
465                                     "instantiated": True,
466                                 }
467                             )
468 0                             nsrs_detailed_list_new.append(nss)
469 0                     if status not in [
470                         "COMPLETED",
471                         "PARTIALLY_COMPLETED",
472                         "FAILED",
473                         "FAILED_TEMP",
474                     ]:
475 0                         nsi_ready = False
476
477 0                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
478 0                     nsrs_detailed_list_old = nsrs_detailed_list_new
479 0                     self.update_db_2(
480                         "nsis",
481                         nsir_id,
482                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
483                     )
484
485 0                 if nsi_ready:
486 0                     error_list = []
487 0                     step = "Network Slice Instance instantiated"
488 0                     for nss in nsrs_detailed_list:
489 0                         if nss["status"] in ("FAILED", "FAILED_TEMP"):
490 0                             error_list.append(
491                                 "NS {} {}: {}".format(
492                                     nss["nsrId"], nss["status"], nss["detailed-status"]
493                                 )
494                             )
495 0                     if error_list:
496 0                         step = "instantiating"
497 0                         raise LcmException("; ".join(error_list))
498 0                     break
499
500                 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
501 0                 await asyncio.sleep(5, loop=self.loop)
502
503             else:  # timeout_nsi_deploy reached:
504 0                 raise LcmException("Timeout waiting nsi to be ready.")
505
506 0             db_nsir_update["operational-status"] = "running"
507 0             db_nsir_update["detailed-status"] = "done"
508 0             db_nsir_update["config-status"] = "configured"
509 0             db_nsilcmop_update[
510                 "operationState"
511             ] = nsilcmop_operation_state = "COMPLETED"
512 0             db_nsilcmop_update["statusEnteredTime"] = time()
513 0             db_nsilcmop_update["detailed-status"] = "done"
514 0             return
515
516 0         except (LcmException, DbException) as e:
517 0             self.logger.error(
518                 logging_text + "Exit Exception while '{}': {}".format(step, e)
519             )
520 0             exc = e
521 0         except asyncio.CancelledError:
522 0             self.logger.error(
523                 logging_text + "Cancelled Exception while '{}'".format(step)
524             )
525 0             exc = "Operation was cancelled"
526 0         except Exception as e:
527 0             exc = traceback.format_exc()
528 0             self.logger.critical(
529                 logging_text
530                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
531                 exc_info=True,
532             )
533         finally:
534 0             if exc:
535 0                 if db_nsir:
536 0                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
537 0                     db_nsir_update["operational-status"] = "failed"
538 0                     db_nsir_update["config-status"] = "configured"
539 0                 if db_nsilcmop:
540 0                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
541                         step, exc
542                     )
543 0                     db_nsilcmop_update[
544                         "operationState"
545                     ] = nsilcmop_operation_state = "FAILED"
546 0                     db_nsilcmop_update["statusEnteredTime"] = time()
547 0             try:
548 0                 if db_nsir:
549 0                     db_nsir_update["_admin.nsilcmop"] = None
550 0                     self.update_db_2("nsis", nsir_id, db_nsir_update)
551 0                 if db_nsilcmop:
552 0                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
553 0             except DbException as e:
554 0                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
555 0             if nsilcmop_operation_state:
556 0                 try:
557 0                     await self.msg.aiowrite(
558                         "nsi",
559                         "instantiated",
560                         {
561                             "nsir_id": nsir_id,
562                             "nsilcmop_id": nsilcmop_id,
563                             "operationState": nsilcmop_operation_state,
564                         },
565                     )
566 0                 except Exception as e:
567 0                     self.logger.error(
568                         logging_text + "kafka_write notification Exception {}".format(e)
569                     )
570 0             self.logger.debug(logging_text + "Exit")
571 0             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
572
573 1     async def terminate(self, nsir_id, nsilcmop_id):
574         # Try to lock HA task here
575 0         task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
576 0         if not task_is_locked_by_me:
577 0             return
578
579 0         logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
580 0         self.logger.debug(logging_text + "Enter")
581 0         exc = None
582 0         db_nsir = None
583 0         db_nsilcmop = None
584 0         db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
585 0         db_nsilcmop_update = {}
586 0         RO = ROclient.ROClient(self.loop, **self.ro_config)
587 0         nsir_deployed = None
588 0         failed_detail = []  # annotates all failed error messages
589 0         nsilcmop_operation_state = None
590 0         autoremove = False  # autoremove after terminated
591 0         try:
592             # wait for any previous tasks in process
593 0             await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
594
595 0             step = "Getting nsir={} from db".format(nsir_id)
596 0             db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
597 0             nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
598 0             step = "Getting nsilcmop={} from db".format(nsilcmop_id)
599 0             db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
600
601             # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
602             # CASE: Instance was terminated but there is a second request to terminate the instance
603 0             if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
604 0                 return
605
606             # Slice status Terminating
607 0             db_nsir_update["operational-status"] = "terminating"
608 0             db_nsir_update["config-status"] = "terminating"
609 0             db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
610 0             self.update_db_2("nsis", nsir_id, db_nsir_update)
611
612             # Gets the list to keep track of network service records status in the netslice
613 0             nsrs_detailed_list = []
614
615             # Iterate over the network services operation ids to terminate NSs
616             # TODO: (future improvement) look another way check the tasks instead of keep asking
617             # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
618             # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
619 0             step = "Terminating Netslice Subnets"
620 0             nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
621 0             nslcmop_new = []
622 0             for nslcmop_id in nslcmop_ids:
623 0                 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
624 0                 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
625 0                 nss_in_use = self.db.get_list(
626                     "nsis",
627                     {
628                         "_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
629                         "operational-status": {"$nin": ["terminated", "failed"]},
630                     },
631                 )
632 0                 if len(nss_in_use) < 2:
633 0                     task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
634 0                     self.lcm_tasks.register(
635                         "ns", nsr_id, nslcmop_id, "ns_instantiate", task
636                     )
637 0                     nslcmop_new.append(nslcmop_id)
638                 else:
639                     # Update shared nslcmop shared with active nsi
640 0                     netsliceInstanceId = db_nsir["_id"]
641 0                     for nsis_item in nss_in_use:
642 0                         if db_nsir["_id"] != nsis_item["_id"]:
643 0                             netsliceInstanceId = nsis_item["_id"]
644 0                             break
645 0                     self.db.set_one(
646                         "nslcmops",
647                         {"_id": nslcmop_id},
648                         {"operationParams.netsliceInstanceId": netsliceInstanceId},
649                     )
650 0             self.db.set_one(
651                 "nsilcmops",
652                 {"_id": nsilcmop_id},
653                 {"operationParams.nslcmops_ids": nslcmop_new},
654             )
655
656             # Wait until Network Slice is terminated
657 0             step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(
658                 nsir_id
659             )
660 0             nsrs_detailed_list_old = None
661 0             self.logger.debug(logging_text + step)
662
663 0             termination_timeout = 2 * 3600  # Two hours
664 0             while termination_timeout > 0:
665                 # Check ns termination status
666 0                 nsi_ready = True
667 0                 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
668 0                 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
669 0                 nsrs_detailed_list_new = []
670 0                 for nslcmop_item in nslcmop_ids:
671 0                     nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
672 0                     status = nslcmop["operationState"]
673                     # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
674 0                     for nss in nsrs_detailed_list:
675 0                         if nss["nsrId"] == nslcmop["nsInstanceId"]:
676 0                             nss.update(
677                                 {
678                                     "nsrId": nslcmop["nsInstanceId"],
679                                     "status": nslcmop["operationState"],
680                                     "detailed-status": nsir_status_detailed
681                                     + "; {}".format(nslcmop.get("detailed-status")),
682                                 }
683                             )
684 0                             nsrs_detailed_list_new.append(nss)
685 0                     if status not in [
686                         "COMPLETED",
687                         "PARTIALLY_COMPLETED",
688                         "FAILED",
689                         "FAILED_TEMP",
690                     ]:
691 0                         nsi_ready = False
692
693 0                 if nsrs_detailed_list_new != nsrs_detailed_list_old:
694 0                     nsrs_detailed_list_old = nsrs_detailed_list_new
695 0                     self.update_db_2(
696                         "nsis",
697                         nsir_id,
698                         {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
699                     )
700
701 0                 if nsi_ready:
702                     # Check if it is the last used nss and mark isinstantiate: False
703 0                     db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
704 0                     nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
705 0                     for nss in nsrs_detailed_list:
706 0                         _filter = {
707                             "_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
708                             "operational-status.ne": "terminated",
709                             "_id.ne": nsir_id,
710                         }
711 0                         nsis_list = self.db.get_one(
712                             "nsis", _filter, fail_on_empty=False, fail_on_more=False
713                         )
714 0                         if not nsis_list:
715 0                             nss.update({"instantiated": False})
716
717 0                     step = "Network Slice Instance is terminated. nsi_id={}".format(
718                         nsir_id
719                     )
720 0                     for items in nsrs_detailed_list:
721 0                         if "FAILED" in items.values():
722 0                             raise LcmException(
723                                 "Error terminating NSI: {}".format(nsir_id)
724                             )
725 0                     break
726
727 0                 await asyncio.sleep(5, loop=self.loop)
728 0                 termination_timeout -= 5
729
730 0             if termination_timeout <= 0:
731 0                 raise LcmException(
732                     "Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id)
733                 )
734
735             # Delete netslice-vlds
736 0             RO_nsir_id = RO_delete_action = None
737 0             for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
738 0                 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
739 0                 try:
740 0                     if not self.ro_config["ng"]:
741 0                         step = db_nsir_update[
742                             "detailed-status"
743                         ] = "Deleting netslice-vld at RO"
744 0                         db_nsilcmop_update[
745                             "detailed-status"
746                         ] = "Deleting netslice-vld at RO"
747 0                         self.logger.debug(logging_text + step)
748 0                         desc = await RO.delete("ns", RO_nsir_id)
749 0                         RO_delete_action = desc["action_id"]
750 0                         nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
751 0                         nsir_deployed_RO["vld_status"] = "DELETING"
752 0                         db_nsir_update["_admin.deployed"] = nsir_deployed
753 0                         self.update_db_2("nsis", nsir_id, db_nsir_update)
754 0                         if RO_delete_action:
755                             # wait until NS is deleted from VIM
756 0                             step = "Waiting ns deleted from VIM. RO_id={}".format(
757                                 RO_nsir_id
758                             )
759 0                             self.logger.debug(logging_text + step)
760 0                 except ROclient.ROClientException as e:
761 0                     if e.http_code == 404:  # not found
762 0                         nsir_deployed_RO["vld_id"] = None
763 0                         nsir_deployed_RO["vld_status"] = "DELETED"
764 0                         self.logger.debug(
765                             logging_text
766                             + "RO_ns_id={} already deleted".format(RO_nsir_id)
767                         )
768 0                     elif e.http_code == 409:  # conflict
769 0                         failed_detail.append(
770                             "RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e)
771                         )
772 0                         self.logger.debug(logging_text + failed_detail[-1])
773                     else:
774 0                         failed_detail.append(
775                             "RO_ns_id={} delete error: {}".format(RO_nsir_id, e)
776                         )
777 0                         self.logger.error(logging_text + failed_detail[-1])
778
779 0                 if failed_detail:
780 0                     self.logger.error(logging_text + " ;".join(failed_detail))
781 0                     db_nsir_update["operational-status"] = "failed"
782 0                     db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(
783                         failed_detail
784                     )
785 0                     db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
786 0                     db_nsilcmop_update[
787                         "operationState"
788                     ] = nsilcmop_operation_state = "FAILED"
789 0                     db_nsilcmop_update["statusEnteredTime"] = time()
790                 else:
791 0                     db_nsir_update["operational-status"] = "terminating"
792 0                     db_nsir_update["config-status"] = "terminating"
793 0                     db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
794 0                     db_nsilcmop_update[
795                         "operationState"
796                     ] = nsilcmop_operation_state = "COMPLETED"
797 0                     db_nsilcmop_update["statusEnteredTime"] = time()
798 0                     if db_nsilcmop["operationParams"].get("autoremove"):
799 0                         autoremove = True
800
801 0             db_nsir_update["detailed-status"] = "done"
802 0             db_nsir_update["operational-status"] = "terminated"
803 0             db_nsir_update["config-status"] = "terminated"
804 0             db_nsilcmop_update["statusEnteredTime"] = time()
805 0             db_nsilcmop_update["detailed-status"] = "done"
806 0             return
807
808 0         except (LcmException, DbException) as e:
809 0             self.logger.error(
810                 logging_text + "Exit Exception while '{}': {}".format(step, e)
811             )
812 0             exc = e
813 0         except asyncio.CancelledError:
814 0             self.logger.error(
815                 logging_text + "Cancelled Exception while '{}'".format(step)
816             )
817 0             exc = "Operation was cancelled"
818 0         except Exception as e:
819 0             exc = traceback.format_exc()
820 0             self.logger.critical(
821                 logging_text
822                 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
823                 exc_info=True,
824             )
825         finally:
826 0             if exc:
827 0                 if db_nsir:
828 0                     db_nsir_update["_admin.deployed"] = nsir_deployed
829 0                     db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
830 0                     db_nsir_update["operational-status"] = "failed"
831 0                 if db_nsilcmop:
832 0                     db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
833                         step, exc
834                     )
835 0                     db_nsilcmop_update[
836                         "operationState"
837                     ] = nsilcmop_operation_state = "FAILED"
838 0                     db_nsilcmop_update["statusEnteredTime"] = time()
839 0             try:
840 0                 if db_nsir:
841 0                     db_nsir_update["_admin.deployed"] = nsir_deployed
842 0                     db_nsir_update["_admin.nsilcmop"] = None
843 0                     self.update_db_2("nsis", nsir_id, db_nsir_update)
844 0                 if db_nsilcmop:
845 0                     self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
846 0             except DbException as e:
847 0                 self.logger.error(logging_text + "Cannot update database: {}".format(e))
848
849 0             if nsilcmop_operation_state:
850 0                 try:
851 0                     await self.msg.aiowrite(
852                         "nsi",
853                         "terminated",
854                         {
855                             "nsir_id": nsir_id,
856                             "nsilcmop_id": nsilcmop_id,
857                             "operationState": nsilcmop_operation_state,
858                             "autoremove": autoremove,
859                         },
860                         loop=self.loop,
861                     )
862 0                 except Exception as e:
863 0                     self.logger.error(
864                         logging_text + "kafka_write notification Exception {}".format(e)
865                     )
866 0             self.logger.debug(logging_text + "Exit")
867 0             self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")