Reformat files according to new black validation
[osm/LCM.git] / osm_lcm / netslice.py
1 # -*- coding: utf-8 -*-
2 ##
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
13 # under the License.
14 ##
15
16 import asyncio
17 import logging
18 import logging.handlers
19 import traceback
20 from osm_lcm import ROclient
21 from osm_lcm.lcm_utils import (
22 LcmException,
23 LcmBase,
24 populate_dict,
25 get_iterable,
26 deep_get,
27 )
28 from osm_common.dbbase import DbException
29 from time import time
30 from copy import deepcopy
31
32
33 __author__ = "Felipe Vicens, Pol Alemany, Alfonso Tierno"
34
35
36 class NetsliceLcm(LcmBase):
37 def __init__(self, msg, lcm_tasks, config, loop, ns):
38 """
39 Init, Connect to database, filesystem storage, and messaging
40 :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
41 :return: None
42 """
43 # logging
44 self.logger = logging.getLogger("lcm.netslice")
45 self.loop = loop
46 self.lcm_tasks = lcm_tasks
47 self.ns = ns
48 self.ro_config = config["RO"]
49 self.timeout = config["timeout"]
50
51 super().__init__(msg, self.logger)
52
53 def nsi_update_nsir(self, nsi_update_nsir, db_nsir, nsir_desc_RO):
54 """
55 Updates database nsir with the RO info for the created vld
56 :param nsi_update_nsir: dictionary to be filled with the updated info
57 :param db_nsir: content of db_nsir. This is also modified
58 :param nsir_desc_RO: nsir descriptor from RO
59 :return: Nothing, LcmException is raised on errors
60 """
61
62 for vld_index, vld in enumerate(get_iterable(db_nsir, "vld")):
63 for net_RO in get_iterable(nsir_desc_RO, "nets"):
64 if vld["id"] != net_RO.get("ns_net_osm_id"):
65 continue
66 vld["vim-id"] = net_RO.get("vim_net_id")
67 vld["name"] = net_RO.get("vim_name")
68 vld["status"] = net_RO.get("status")
69 vld["status-detailed"] = net_RO.get("error_msg")
70 nsi_update_nsir["vld.{}".format(vld_index)] = vld
71 break
72 else:
73 raise LcmException(
74 "ns_update_nsir: Not found vld={} at RO info".format(vld["id"])
75 )
76
77 async def instantiate(self, nsir_id, nsilcmop_id):
78 # Try to lock HA task here
79 task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
80 if not task_is_locked_by_me:
81 return
82
83 logging_text = "Task netslice={} instantiate={} ".format(nsir_id, nsilcmop_id)
84 self.logger.debug(logging_text + "Enter")
85 # get all needed from database
86 exc = None
87 db_nsir = None
88 db_nsilcmop = None
89 db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
90 db_nsilcmop_update = {}
91 nsilcmop_operation_state = None
92 vim_2_RO = {}
93 RO = ROclient.ROClient(self.loop, **self.ro_config)
94 nsi_vld_instantiationi_params = {}
95
96 def ip_profile_2_RO(ip_profile):
97 RO_ip_profile = deepcopy((ip_profile))
98 if "dns-server" in RO_ip_profile:
99 if isinstance(RO_ip_profile["dns-server"], list):
100 RO_ip_profile["dns-address"] = []
101 for ds in RO_ip_profile.pop("dns-server"):
102 RO_ip_profile["dns-address"].append(ds["address"])
103 else:
104 RO_ip_profile["dns-address"] = RO_ip_profile.pop("dns-server")
105 if RO_ip_profile.get("ip-version") == "ipv4":
106 RO_ip_profile["ip-version"] = "IPv4"
107 if RO_ip_profile.get("ip-version") == "ipv6":
108 RO_ip_profile["ip-version"] = "IPv6"
109 if "dhcp-params" in RO_ip_profile:
110 RO_ip_profile["dhcp"] = RO_ip_profile.pop("dhcp-params")
111 return RO_ip_profile
112
113 def vim_account_2_RO(vim_account):
114 """
115 Translate a RO vim_account from OSM vim_account params
116 :param ns_params: OSM instantiate params
117 :return: The RO ns descriptor
118 """
119 if vim_account in vim_2_RO:
120 return vim_2_RO[vim_account]
121
122 db_vim = self.db.get_one("vim_accounts", {"_id": vim_account})
123 if db_vim["_admin"]["operationalState"] != "ENABLED":
124 raise LcmException(
125 "VIM={} is not available. operationalState={}".format(
126 vim_account, db_vim["_admin"]["operationalState"]
127 )
128 )
129 RO_vim_id = db_vim["_admin"]["deployed"]["RO"]
130 vim_2_RO[vim_account] = RO_vim_id
131 return RO_vim_id
132
133 async def netslice_scenario_create(
134 self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
135 ):
136 """
137 Create a network slice VLD through RO Scenario
138 :param vld_id The VLD id inside nsir to be created
139 :param nsir_id The nsir id
140 """
141 nonlocal nsi_vld_instantiationi_params
142 ip_vld = None
143 mgmt_network = False
144 RO_vld_sites = []
145 vld_id = vld_item["id"]
146 netslice_vld = vld_item
147 # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
148 # self.logger.debug(logging_text + "Enter")
149
150 vld_shared = None
151 for shared_nsrs_item in get_iterable(vld_item, "shared-nsrs-list"):
152 _filter = {
153 "_id.ne": nsir_id,
154 "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item,
155 }
156 shared_nsi = self.db.get_one(
157 "nsis", _filter, fail_on_empty=False, fail_on_more=False
158 )
159 if shared_nsi:
160 for vlds in get_iterable(shared_nsi["_admin"]["deployed"], "RO"):
161 if vld_id == vlds["vld_id"]:
162 vld_shared = {
163 "instance_scenario_id": vlds["netslice_scenario_id"],
164 "osm_id": vld_id,
165 }
166 break
167 break
168
169 # Creating netslice-vld at RO
170 RO_nsir = deep_get(db_nsir, ("_admin", "deployed", "RO"), [])
171
172 if vld_id in RO_nsir:
173 db_nsir_update["_admin.deployed.RO"] = RO_nsir
174
175 # If netslice-vld doesn't exists then create it
176 else:
177 # TODO: Check VDU type in all descriptors finding SRIOV / PT
178 # Updating network names and datacenters from instantiation parameters for each VLD
179 for instantiation_params_vld in get_iterable(
180 db_nsir["instantiation_parameters"], "netslice-vld"
181 ):
182 if instantiation_params_vld.get("name") == netslice_vld["name"]:
183 ip_vld = deepcopy(instantiation_params_vld)
184 ip_vld.pop("name")
185 nsi_vld_instantiationi_params[netslice_vld["name"]] = ip_vld
186
187 db_nsir_update_RO = {}
188 db_nsir_update_RO["vld_id"] = netslice_vld["name"]
189 if self.ro_config["ng"]:
190 db_nsir_update_RO["netslice_scenario_id"] = (
191 vld_shared.get("instance_scenario_id")
192 if vld_shared
193 else "nsir:{}:vld.{}".format(nsir_id, netslice_vld["name"])
194 )
195 else: # if not self.ro_config["ng"]:
196 if netslice_vld.get("mgmt-network"):
197 mgmt_network = True
198 RO_ns_params = {}
199 RO_ns_params["name"] = netslice_vld["name"]
200 RO_ns_params["datacenter"] = vim_account_2_RO(
201 db_nsir["instantiation_parameters"]["vimAccountId"]
202 )
203
204 # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
205 # Use vim-network-id instantiation parameter
206 vim_network_option = None
207 if ip_vld:
208 if ip_vld.get("vim-network-id"):
209 vim_network_option = "vim-network-id"
210 elif ip_vld.get("vim-network-name"):
211 vim_network_option = "vim-network-name"
212 if ip_vld.get("ip-profile"):
213 populate_dict(
214 RO_ns_params,
215 ("networks", netslice_vld["name"], "ip-profile"),
216 ip_profile_2_RO(ip_vld["ip-profile"]),
217 )
218
219 if vim_network_option:
220 if ip_vld.get(vim_network_option):
221 if isinstance(ip_vld.get(vim_network_option), list):
222 for vim_net_id in ip_vld.get(vim_network_option):
223 for vim_account, vim_net in vim_net_id.items():
224 RO_vld_sites.append(
225 {
226 "netmap-use": vim_net,
227 "datacenter": vim_account_2_RO(
228 vim_account
229 ),
230 }
231 )
232 elif isinstance(ip_vld.get(vim_network_option), dict):
233 for vim_account, vim_net in ip_vld.get(
234 vim_network_option
235 ).items():
236 RO_vld_sites.append(
237 {
238 "netmap-use": vim_net,
239 "datacenter": vim_account_2_RO(vim_account),
240 }
241 )
242 else:
243 RO_vld_sites.append(
244 {
245 "netmap-use": ip_vld[vim_network_option],
246 "datacenter": vim_account_2_RO(
247 netslice_vld["vimAccountId"]
248 ),
249 }
250 )
251
252 # Use default netslice vim-network-name from template
253 else:
254 for nss_conn_point_ref in get_iterable(
255 netslice_vld, "nss-connection-point-ref"
256 ):
257 if nss_conn_point_ref.get("vimAccountId"):
258 if (
259 nss_conn_point_ref["vimAccountId"]
260 != netslice_vld["vimAccountId"]
261 ):
262 RO_vld_sites.append(
263 {
264 "netmap-create": None,
265 "datacenter": vim_account_2_RO(
266 nss_conn_point_ref["vimAccountId"]
267 ),
268 }
269 )
270
271 if vld_shared:
272 populate_dict(
273 RO_ns_params,
274 ("networks", netslice_vld["name"], "use-network"),
275 vld_shared,
276 )
277
278 if RO_vld_sites:
279 populate_dict(
280 RO_ns_params,
281 ("networks", netslice_vld["name"], "sites"),
282 RO_vld_sites,
283 )
284
285 RO_ns_params["scenario"] = {
286 "nets": [
287 {
288 "name": netslice_vld["name"],
289 "external": mgmt_network,
290 "type": "bridge",
291 }
292 ]
293 }
294
295 # self.logger.debug(logging_text + step)
296 desc = await RO.create("ns", descriptor=RO_ns_params)
297 db_nsir_update_RO["netslice_scenario_id"] = desc["uuid"]
298 db_nsir_update["_admin.deployed.RO"].append(db_nsir_update_RO)
299
300 def overwrite_nsd_params(self, db_nsir, nslcmop):
301 nonlocal nsi_vld_instantiationi_params
302 nonlocal db_nsir_update
303 vld_op_list = []
304 vld = None
305 nsr_id = nslcmop.get("nsInstanceId")
306 # Overwrite instantiation parameters in netslice runtime
307 RO_list = db_nsir_admin["deployed"]["RO"]
308
309 for ro_item_index, RO_item in enumerate(RO_list):
310 netslice_vld = next(
311 (
312 n
313 for n in get_iterable(db_nsir["_admin"], "netslice-vld")
314 if RO_item.get("vld_id") == n.get("id")
315 ),
316 None,
317 )
318 if not netslice_vld:
319 continue
320 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
321 # Search the cp of netslice-vld that match with nst:netslice-subnet
322 for nss_cp_item in get_iterable(
323 netslice_vld, "nss-connection-point-ref"
324 ):
325 # Search the netslice-subnet of nst that match
326 nss = next(
327 (
328 nss
329 for nss in get_iterable(
330 db_nsir["_admin"], "netslice-subnet"
331 )
332 if nss_cp_item["nss-ref"] == nss["nss-id"]
333 ),
334 None,
335 )
336 # Compare nss-ref equal nss from nst
337 if not nss:
338 continue
339 db_nsds = self.db.get_one("nsds", {"_id": nss["nsdId"]})
340 # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
341 for cp_nsd in db_nsds.get("sapd", ()):
342 if cp_nsd["id"] == nss_cp_item["nsd-connection-point-ref"]:
343 if nslcmop.get("operationParams"):
344 if (
345 nslcmop["operationParams"].get("nsName")
346 == nss["nsName"]
347 ):
348 vld_id = RO_item["vld_id"]
349 netslice_scenario_id = RO_item[
350 "netslice_scenario_id"
351 ]
352 nslcmop_vld = {}
353 nslcmop_vld["name"] = cp_nsd["virtual-link-desc"]
354 for vld in get_iterable(
355 nslcmop["operationParams"], "vld"
356 ):
357 if vld["name"] == cp_nsd["virtual-link-desc"]:
358 nslcmop_vld.update(vld)
359 if self.ro_config["ng"]:
360 nslcmop_vld["common_id"] = netslice_scenario_id
361 nslcmop_vld.update(
362 nsi_vld_instantiationi_params.get(
363 RO_item["vld_id"], {}
364 )
365 )
366 else:
367 nslcmop_vld["ns-net"] = {
368 vld_id: netslice_scenario_id
369 }
370 vld_op_list.append(nslcmop_vld)
371 nslcmop["operationParams"]["vld"] = vld_op_list
372 self.update_db_2(
373 "nslcmops", nslcmop["_id"], {"operationParams.vld": vld_op_list}
374 )
375 return nsr_id, nslcmop
376
377 try:
378 # wait for any previous tasks in process
379 await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
380
381 step = "Getting nsir={} from db".format(nsir_id)
382 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
383 step = "Getting nsilcmop={} from db".format(nsilcmop_id)
384 db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
385
386 start_deploy = time()
387 nsi_params = db_nsilcmop.get("operationParams")
388 if nsi_params and nsi_params.get("timeout_nsi_deploy"):
389 timeout_nsi_deploy = nsi_params["timeout_nsi_deploy"]
390 else:
391 timeout_nsi_deploy = self.timeout.get("nsi_deploy")
392
393 # Empty list to keep track of network service records status in the netslice
394 nsir_admin = db_nsir_admin = db_nsir.get("_admin")
395
396 step = "Creating slice operational-status init"
397 # Slice status Creating
398 db_nsir_update["detailed-status"] = "creating"
399 db_nsir_update["operational-status"] = "init"
400 db_nsir_update["_admin.nsiState"] = "INSTANTIATED"
401
402 step = "Instantiating netslice VLDs before NS instantiation"
403 # Creating netslice VLDs networking before NS instantiation
404 db_nsir_update["detailed-status"] = step
405 self.update_db_2("nsis", nsir_id, db_nsir_update)
406 db_nsir_update["_admin.deployed.RO"] = db_nsir_admin["deployed"]["RO"]
407 for vld_item in get_iterable(nsir_admin, "netslice-vld"):
408 await netslice_scenario_create(
409 self, vld_item, nsir_id, db_nsir, db_nsir_admin, db_nsir_update
410 )
411
412 step = "Instantiating netslice subnets"
413 db_nsir_update["detailed-status"] = step
414 self.update_db_2("nsis", nsir_id, db_nsir_update)
415
416 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
417
418 # Check status of the VLDs and wait for creation
419 # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
420 # db_nsir_update_RO = deepcopy(netslice_scenarios)
421 # for netslice_scenario in netslice_scenarios:
422 # await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
423 # nsir_id, db_nsir_update_RO)
424
425 # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
426 # self.update_db_2("nsis", nsir_id, db_nsir_update)
427
428 # Iterate over the network services operation ids to instantiate NSs
429 step = "Instantiating Netslice Subnets"
430 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
431 nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
432 for nslcmop_id in nslcmop_ids:
433 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
434 # Overwriting netslice-vld vim-net-id to ns
435 nsr_id, nslcmop = overwrite_nsd_params(self, db_nsir, nslcmop)
436 step = "Launching ns={} instantiate={} task".format(nsr_id, nslcmop_id)
437 task = asyncio.ensure_future(self.ns.instantiate(nsr_id, nslcmop_id))
438 self.lcm_tasks.register(
439 "ns", nsr_id, nslcmop_id, "ns_instantiate", task
440 )
441
442 # Wait until Network Slice is ready
443 step = " Waiting nsi ready."
444 nsrs_detailed_list_old = None
445 self.logger.debug(logging_text + step)
446
447 # For HA, it is checked from database, as the ns operation may be managed by other LCM worker
448 while time() <= start_deploy + timeout_nsi_deploy:
449 # Check ns instantiation status
450 nsi_ready = True
451 nsir = self.db.get_one("nsis", {"_id": nsir_id})
452 nsrs_detailed_list = nsir["_admin"]["nsrs-detailed-list"]
453 nsrs_detailed_list_new = []
454 for nslcmop_item in nslcmop_ids:
455 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
456 status = nslcmop.get("operationState")
457 # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
458 for nss in nsrs_detailed_list:
459 if nss["nsrId"] == nslcmop["nsInstanceId"]:
460 nss.update(
461 {
462 "nsrId": nslcmop["nsInstanceId"],
463 "status": nslcmop["operationState"],
464 "detailed-status": nslcmop.get("detailed-status"),
465 "instantiated": True,
466 }
467 )
468 nsrs_detailed_list_new.append(nss)
469 if status not in [
470 "COMPLETED",
471 "PARTIALLY_COMPLETED",
472 "FAILED",
473 "FAILED_TEMP",
474 ]:
475 nsi_ready = False
476
477 if nsrs_detailed_list_new != nsrs_detailed_list_old:
478 nsrs_detailed_list_old = nsrs_detailed_list_new
479 self.update_db_2(
480 "nsis",
481 nsir_id,
482 {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
483 )
484
485 if nsi_ready:
486 error_list = []
487 step = "Network Slice Instance instantiated"
488 for nss in nsrs_detailed_list:
489 if nss["status"] in ("FAILED", "FAILED_TEMP"):
490 error_list.append(
491 "NS {} {}: {}".format(
492 nss["nsrId"], nss["status"], nss["detailed-status"]
493 )
494 )
495 if error_list:
496 step = "instantiating"
497 raise LcmException("; ".join(error_list))
498 break
499
500 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
501 await asyncio.sleep(5, loop=self.loop)
502
503 else: # timeout_nsi_deploy reached:
504 raise LcmException("Timeout waiting nsi to be ready.")
505
506 db_nsir_update["operational-status"] = "running"
507 db_nsir_update["detailed-status"] = "done"
508 db_nsir_update["config-status"] = "configured"
509 db_nsilcmop_update[
510 "operationState"
511 ] = nsilcmop_operation_state = "COMPLETED"
512 db_nsilcmop_update["statusEnteredTime"] = time()
513 db_nsilcmop_update["detailed-status"] = "done"
514 return
515
516 except (LcmException, DbException) as e:
517 self.logger.error(
518 logging_text + "Exit Exception while '{}': {}".format(step, e)
519 )
520 exc = e
521 except asyncio.CancelledError:
522 self.logger.error(
523 logging_text + "Cancelled Exception while '{}'".format(step)
524 )
525 exc = "Operation was cancelled"
526 except Exception as e:
527 exc = traceback.format_exc()
528 self.logger.critical(
529 logging_text
530 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
531 exc_info=True,
532 )
533 finally:
534 if exc:
535 if db_nsir:
536 db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
537 db_nsir_update["operational-status"] = "failed"
538 db_nsir_update["config-status"] = "configured"
539 if db_nsilcmop:
540 db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
541 step, exc
542 )
543 db_nsilcmop_update[
544 "operationState"
545 ] = nsilcmop_operation_state = "FAILED"
546 db_nsilcmop_update["statusEnteredTime"] = time()
547 try:
548 if db_nsir:
549 db_nsir_update["_admin.nsilcmop"] = None
550 self.update_db_2("nsis", nsir_id, db_nsir_update)
551 if db_nsilcmop:
552 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
553 except DbException as e:
554 self.logger.error(logging_text + "Cannot update database: {}".format(e))
555 if nsilcmop_operation_state:
556 try:
557 await self.msg.aiowrite(
558 "nsi",
559 "instantiated",
560 {
561 "nsir_id": nsir_id,
562 "nsilcmop_id": nsilcmop_id,
563 "operationState": nsilcmop_operation_state,
564 },
565 )
566 except Exception as e:
567 self.logger.error(
568 logging_text + "kafka_write notification Exception {}".format(e)
569 )
570 self.logger.debug(logging_text + "Exit")
571 self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_instantiate")
572
573 async def terminate(self, nsir_id, nsilcmop_id):
574 # Try to lock HA task here
575 task_is_locked_by_me = self.lcm_tasks.lock_HA("nsi", "nsilcmops", nsilcmop_id)
576 if not task_is_locked_by_me:
577 return
578
579 logging_text = "Task nsi={} terminate={} ".format(nsir_id, nsilcmop_id)
580 self.logger.debug(logging_text + "Enter")
581 exc = None
582 db_nsir = None
583 db_nsilcmop = None
584 db_nsir_update = {"_admin.nsilcmop": nsilcmop_id}
585 db_nsilcmop_update = {}
586 RO = ROclient.ROClient(self.loop, **self.ro_config)
587 nsir_deployed = None
588 failed_detail = [] # annotates all failed error messages
589 nsilcmop_operation_state = None
590 autoremove = False # autoremove after terminated
591 try:
592 # wait for any previous tasks in process
593 await self.lcm_tasks.waitfor_related_HA("nsi", "nsilcmops", nsilcmop_id)
594
595 step = "Getting nsir={} from db".format(nsir_id)
596 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
597 nsir_deployed = deepcopy(db_nsir["_admin"].get("deployed"))
598 step = "Getting nsilcmop={} from db".format(nsilcmop_id)
599 db_nsilcmop = self.db.get_one("nsilcmops", {"_id": nsilcmop_id})
600
601 # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
602 # CASE: Instance was terminated but there is a second request to terminate the instance
603 if db_nsir["_admin"]["nsiState"] == "NOT_INSTANTIATED":
604 return
605
606 # Slice status Terminating
607 db_nsir_update["operational-status"] = "terminating"
608 db_nsir_update["config-status"] = "terminating"
609 db_nsir_update["detailed-status"] = "Terminating Netslice subnets"
610 self.update_db_2("nsis", nsir_id, db_nsir_update)
611
612 # Gets the list to keep track of network service records status in the netslice
613 nsrs_detailed_list = []
614
615 # Iterate over the network services operation ids to terminate NSs
616 # TODO: (future improvement) look another way check the tasks instead of keep asking
617 # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
618 # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
619 step = "Terminating Netslice Subnets"
620 nslcmop_ids = db_nsilcmop["operationParams"].get("nslcmops_ids")
621 nslcmop_new = []
622 for nslcmop_id in nslcmop_ids:
623 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_id})
624 nsr_id = nslcmop["operationParams"].get("nsInstanceId")
625 nss_in_use = self.db.get_list(
626 "nsis",
627 {
628 "_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id,
629 "operational-status": {"$nin": ["terminated", "failed"]},
630 },
631 )
632 if len(nss_in_use) < 2:
633 task = asyncio.ensure_future(self.ns.terminate(nsr_id, nslcmop_id))
634 self.lcm_tasks.register(
635 "ns", nsr_id, nslcmop_id, "ns_instantiate", task
636 )
637 nslcmop_new.append(nslcmop_id)
638 else:
639 # Update shared nslcmop shared with active nsi
640 netsliceInstanceId = db_nsir["_id"]
641 for nsis_item in nss_in_use:
642 if db_nsir["_id"] != nsis_item["_id"]:
643 netsliceInstanceId = nsis_item["_id"]
644 break
645 self.db.set_one(
646 "nslcmops",
647 {"_id": nslcmop_id},
648 {"operationParams.netsliceInstanceId": netsliceInstanceId},
649 )
650 self.db.set_one(
651 "nsilcmops",
652 {"_id": nsilcmop_id},
653 {"operationParams.nslcmops_ids": nslcmop_new},
654 )
655
656 # Wait until Network Slice is terminated
657 step = nsir_status_detailed = " Waiting nsi terminated. nsi_id={}".format(
658 nsir_id
659 )
660 nsrs_detailed_list_old = None
661 self.logger.debug(logging_text + step)
662
663 termination_timeout = 2 * 3600 # Two hours
664 while termination_timeout > 0:
665 # Check ns termination status
666 nsi_ready = True
667 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
668 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
669 nsrs_detailed_list_new = []
670 for nslcmop_item in nslcmop_ids:
671 nslcmop = self.db.get_one("nslcmops", {"_id": nslcmop_item})
672 status = nslcmop["operationState"]
673 # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
674 for nss in nsrs_detailed_list:
675 if nss["nsrId"] == nslcmop["nsInstanceId"]:
676 nss.update(
677 {
678 "nsrId": nslcmop["nsInstanceId"],
679 "status": nslcmop["operationState"],
680 "detailed-status": nsir_status_detailed
681 + "; {}".format(nslcmop.get("detailed-status")),
682 }
683 )
684 nsrs_detailed_list_new.append(nss)
685 if status not in [
686 "COMPLETED",
687 "PARTIALLY_COMPLETED",
688 "FAILED",
689 "FAILED_TEMP",
690 ]:
691 nsi_ready = False
692
693 if nsrs_detailed_list_new != nsrs_detailed_list_old:
694 nsrs_detailed_list_old = nsrs_detailed_list_new
695 self.update_db_2(
696 "nsis",
697 nsir_id,
698 {"_admin.nsrs-detailed-list": nsrs_detailed_list_new},
699 )
700
701 if nsi_ready:
702 # Check if it is the last used nss and mark isinstantiate: False
703 db_nsir = self.db.get_one("nsis", {"_id": nsir_id})
704 nsrs_detailed_list = db_nsir["_admin"].get("nsrs-detailed-list")
705 for nss in nsrs_detailed_list:
706 _filter = {
707 "_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss["nsrId"],
708 "operational-status.ne": "terminated",
709 "_id.ne": nsir_id,
710 }
711 nsis_list = self.db.get_one(
712 "nsis", _filter, fail_on_empty=False, fail_on_more=False
713 )
714 if not nsis_list:
715 nss.update({"instantiated": False})
716
717 step = "Network Slice Instance is terminated. nsi_id={}".format(
718 nsir_id
719 )
720 for items in nsrs_detailed_list:
721 if "FAILED" in items.values():
722 raise LcmException(
723 "Error terminating NSI: {}".format(nsir_id)
724 )
725 break
726
727 await asyncio.sleep(5, loop=self.loop)
728 termination_timeout -= 5
729
730 if termination_timeout <= 0:
731 raise LcmException(
732 "Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id)
733 )
734
735 # Delete netslice-vlds
736 RO_nsir_id = RO_delete_action = None
737 for nsir_deployed_RO in get_iterable(nsir_deployed, "RO"):
738 RO_nsir_id = nsir_deployed_RO.get("netslice_scenario_id")
739 try:
740 if not self.ro_config["ng"]:
741 step = db_nsir_update[
742 "detailed-status"
743 ] = "Deleting netslice-vld at RO"
744 db_nsilcmop_update[
745 "detailed-status"
746 ] = "Deleting netslice-vld at RO"
747 self.logger.debug(logging_text + step)
748 desc = await RO.delete("ns", RO_nsir_id)
749 RO_delete_action = desc["action_id"]
750 nsir_deployed_RO["vld_delete_action_id"] = RO_delete_action
751 nsir_deployed_RO["vld_status"] = "DELETING"
752 db_nsir_update["_admin.deployed"] = nsir_deployed
753 self.update_db_2("nsis", nsir_id, db_nsir_update)
754 if RO_delete_action:
755 # wait until NS is deleted from VIM
756 step = "Waiting ns deleted from VIM. RO_id={}".format(
757 RO_nsir_id
758 )
759 self.logger.debug(logging_text + step)
760 except ROclient.ROClientException as e:
761 if e.http_code == 404: # not found
762 nsir_deployed_RO["vld_id"] = None
763 nsir_deployed_RO["vld_status"] = "DELETED"
764 self.logger.debug(
765 logging_text
766 + "RO_ns_id={} already deleted".format(RO_nsir_id)
767 )
768 elif e.http_code == 409: # conflict
769 failed_detail.append(
770 "RO_ns_id={} delete conflict: {}".format(RO_nsir_id, e)
771 )
772 self.logger.debug(logging_text + failed_detail[-1])
773 else:
774 failed_detail.append(
775 "RO_ns_id={} delete error: {}".format(RO_nsir_id, e)
776 )
777 self.logger.error(logging_text + failed_detail[-1])
778
779 if failed_detail:
780 self.logger.error(logging_text + " ;".join(failed_detail))
781 db_nsir_update["operational-status"] = "failed"
782 db_nsir_update["detailed-status"] = "Deletion errors " + "; ".join(
783 failed_detail
784 )
785 db_nsilcmop_update["detailed-status"] = "; ".join(failed_detail)
786 db_nsilcmop_update[
787 "operationState"
788 ] = nsilcmop_operation_state = "FAILED"
789 db_nsilcmop_update["statusEnteredTime"] = time()
790 else:
791 db_nsir_update["operational-status"] = "terminating"
792 db_nsir_update["config-status"] = "terminating"
793 db_nsir_update["_admin.nsiState"] = "NOT_INSTANTIATED"
794 db_nsilcmop_update[
795 "operationState"
796 ] = nsilcmop_operation_state = "COMPLETED"
797 db_nsilcmop_update["statusEnteredTime"] = time()
798 if db_nsilcmop["operationParams"].get("autoremove"):
799 autoremove = True
800
801 db_nsir_update["detailed-status"] = "done"
802 db_nsir_update["operational-status"] = "terminated"
803 db_nsir_update["config-status"] = "terminated"
804 db_nsilcmop_update["statusEnteredTime"] = time()
805 db_nsilcmop_update["detailed-status"] = "done"
806 return
807
808 except (LcmException, DbException) as e:
809 self.logger.error(
810 logging_text + "Exit Exception while '{}': {}".format(step, e)
811 )
812 exc = e
813 except asyncio.CancelledError:
814 self.logger.error(
815 logging_text + "Cancelled Exception while '{}'".format(step)
816 )
817 exc = "Operation was cancelled"
818 except Exception as e:
819 exc = traceback.format_exc()
820 self.logger.critical(
821 logging_text
822 + "Exit Exception {} while '{}': {}".format(type(e).__name__, step, e),
823 exc_info=True,
824 )
825 finally:
826 if exc:
827 if db_nsir:
828 db_nsir_update["_admin.deployed"] = nsir_deployed
829 db_nsir_update["detailed-status"] = "ERROR {}: {}".format(step, exc)
830 db_nsir_update["operational-status"] = "failed"
831 if db_nsilcmop:
832 db_nsilcmop_update["detailed-status"] = "FAILED {}: {}".format(
833 step, exc
834 )
835 db_nsilcmop_update[
836 "operationState"
837 ] = nsilcmop_operation_state = "FAILED"
838 db_nsilcmop_update["statusEnteredTime"] = time()
839 try:
840 if db_nsir:
841 db_nsir_update["_admin.deployed"] = nsir_deployed
842 db_nsir_update["_admin.nsilcmop"] = None
843 self.update_db_2("nsis", nsir_id, db_nsir_update)
844 if db_nsilcmop:
845 self.update_db_2("nsilcmops", nsilcmop_id, db_nsilcmop_update)
846 except DbException as e:
847 self.logger.error(logging_text + "Cannot update database: {}".format(e))
848
849 if nsilcmop_operation_state:
850 try:
851 await self.msg.aiowrite(
852 "nsi",
853 "terminated",
854 {
855 "nsir_id": nsir_id,
856 "nsilcmop_id": nsilcmop_id,
857 "operationState": nsilcmop_operation_state,
858 "autoremove": autoremove,
859 },
860 loop=self.loop,
861 )
862 except Exception as e:
863 self.logger.error(
864 logging_text + "kafka_write notification Exception {}".format(e)
865 )
866 self.logger.debug(logging_text + "Exit")
867 self.lcm_tasks.remove("nsi", nsir_id, nsilcmop_id, "nsi_terminate")