1 # -*- coding: utf-8 -*-
5 import logging
.handlers
8 from ns
import populate_dict
as populate_dict
10 from lcm_utils
import LcmException
, LcmBase
11 from osm_common
.dbbase
import DbException
13 from copy
import deepcopy
16 __author__
= "Felipe Vicens, Pol Alemany, Alfonso Tierno"
19 def get_iterable(in_dict
, in_key
):
21 Similar to <dict>.get(), but if value is None, False, ..., An empty tuple is returned instead
22 :param in_dict: a dictionary
23 :param in_key: the key to look for at in_dict
24 :return: in_dict[in_var] or () if it is None or not present
26 if not in_dict
.get(in_key
):
28 return in_dict
[in_key
]
31 class NetsliceLcm(LcmBase
):
33 total_deploy_timeout
= 2 * 3600 # global timeout for deployment
35 def __init__(self
, db
, msg
, fs
, lcm_tasks
, ro_config
, vca_config
, loop
):
37 Init, Connect to database, filesystem storage, and messaging
38 :param config: two level dictionary with configuration. Top level should contain 'database', 'storage',
42 self
.logger
= logging
.getLogger('lcm.netslice')
44 self
.lcm_tasks
= lcm_tasks
45 self
.ns
= ns
.NsLcm(db
, msg
, fs
, lcm_tasks
, ro_config
, vca_config
, loop
)
46 self
.ro_config
= ro_config
48 super().__init
__(db
, msg
, fs
, self
.logger
)
50 def nsi_update_nsir(self
, nsi_update_nsir
, db_nsir
, nsir_desc_RO
):
52 Updates database nsir with the RO info for the created vld
53 :param nsi_update_nsir: dictionary to be filled with the updated info
54 :param db_nsir: content of db_nsir. This is also modified
55 :param nsir_desc_RO: nsir descriptor from RO
56 :return: Nothing, LcmException is raised on errors
59 for vld_index
, vld
in enumerate(get_iterable(db_nsir
, "vld")):
60 for net_RO
in get_iterable(nsir_desc_RO
, "nets"):
61 if vld
["id"] != net_RO
.get("ns_net_osm_id"):
63 vld
["vim-id"] = net_RO
.get("vim_net_id")
64 vld
["name"] = net_RO
.get("vim_name")
65 vld
["status"] = net_RO
.get("status")
66 vld
["status-detailed"] = net_RO
.get("error_msg")
67 nsi_update_nsir
["vld.{}".format(vld_index
)] = vld
70 raise LcmException("ns_update_nsir: Not found vld={} at RO info".format(vld
["id"]))
72 async def instantiate(self
, nsir_id
, nsilcmop_id
):
73 logging_text
= "Task netslice={} instantiate={} ".format(nsir_id
, nsilcmop_id
)
74 self
.logger
.debug(logging_text
+ "Enter")
75 # get all needed from database
79 db_nsir_update
= {"_admin.nsilcmop": nsilcmop_id
}
80 db_nsilcmop_update
= {}
81 nsilcmop_operation_state
= None
83 RO
= ROclient
.ROClient(self
.loop
, **self
.ro_config
)
85 def ip_profile_2_RO(ip_profile
):
86 RO_ip_profile
= deepcopy((ip_profile
))
87 if "dns-server" in RO_ip_profile
:
88 if isinstance(RO_ip_profile
["dns-server"], list):
89 RO_ip_profile
["dns-address"] = []
90 for ds
in RO_ip_profile
.pop("dns-server"):
91 RO_ip_profile
["dns-address"].append(ds
['address'])
93 RO_ip_profile
["dns-address"] = RO_ip_profile
.pop("dns-server")
94 if RO_ip_profile
.get("ip-version") == "ipv4":
95 RO_ip_profile
["ip-version"] = "IPv4"
96 if RO_ip_profile
.get("ip-version") == "ipv6":
97 RO_ip_profile
["ip-version"] = "IPv6"
98 if "dhcp-params" in RO_ip_profile
:
99 RO_ip_profile
["dhcp"] = RO_ip_profile
.pop("dhcp-params")
102 def vim_account_2_RO(vim_account
):
104 Translate a RO vim_account from OSM vim_account params
105 :param ns_params: OSM instantiate params
106 :return: The RO ns descriptor
108 if vim_account
in vim_2_RO
:
109 return vim_2_RO
[vim_account
]
111 db_vim
= self
.db
.get_one("vim_accounts", {"_id": vim_account
})
112 if db_vim
["_admin"]["operationalState"] != "ENABLED":
113 raise LcmException("VIM={} is not available. operationalState={}".format(
114 vim_account
, db_vim
["_admin"]["operationalState"]))
115 RO_vim_id
= db_vim
["_admin"]["deployed"]["RO"]
116 vim_2_RO
[vim_account
] = RO_vim_id
119 async def netslice_scenario_create(self
, vld_item
, nsir_id
, db_nsir
, db_nsir_admin
, db_nsir_update
):
121 Create a network slice VLD through RO Scenario
122 :param vld_id The VLD id inside nsir to be created
123 :param nsir_id The nsir id
128 vld_id
= vld_item
["id"]
129 netslice_vld
= vld_item
130 # logging_text = "Task netslice={} instantiate_vld={} ".format(nsir_id, vld_id)
131 # self.logger.debug(logging_text + "Enter")
134 for shared_nsrs_item
in get_iterable(vld_item
, "shared-nsrs-list"):
135 _filter
= {"_id.ne": nsir_id
, "_admin.nsrs-detailed-list.ANYINDEX.nsrId": shared_nsrs_item
}
136 shared_nsi
= self
.db
.get_one("nsis", _filter
, fail_on_empty
=False, fail_on_more
=False)
138 for vlds
in get_iterable(shared_nsi
["_admin"]["deployed"], "RO"):
139 if vld_id
== vlds
["vld_id"]:
140 vld_shared
= {"instance_scenario_id": vlds
["netslice_scenario_id"], "osm_id": vld_id
}
144 # Creating netslice-vld at RO
145 RO_nsir
= db_nsir
["_admin"].get("deployed", {}).get("RO", [])
147 if vld_id
in RO_nsir
:
148 db_nsir_update
["_admin.deployed.RO"] = RO_nsir
150 # If netslice-vld doesn't exists then create it
152 # TODO: Check VDU type in all descriptors finding SRIOV / PT
153 # Updating network names and datacenters from instantiation parameters for each VLD
155 RO_ns_params
["name"] = netslice_vld
["name"]
156 RO_ns_params
["datacenter"] = vim_account_2_RO(db_nsir
["instantiation_parameters"]["vimAccountId"])
157 for instantiation_params_vld
in get_iterable(db_nsir
["instantiation_parameters"], "netslice-vld"):
158 if instantiation_params_vld
.get("name") == netslice_vld
["name"]:
159 ip_vld
= deepcopy(instantiation_params_vld
)
161 if netslice_vld
.get("mgmt-network"):
164 # Creating scenario if vim-network-name / vim-network-id are present as instantiation parameter
165 # Use vim-network-id instantiation parameter
166 vim_network_option
= None
168 if ip_vld
.get("vim-network-id"):
169 vim_network_option
= "vim-network-id"
170 elif ip_vld
.get("vim-network-name"):
171 vim_network_option
= "vim-network-name"
172 if ip_vld
.get("ip-profile"):
173 populate_dict(RO_ns_params
, ("networks", netslice_vld
["name"], "ip-profile"),
174 ip_profile_2_RO(ip_vld
["ip-profile"]))
176 if vim_network_option
:
177 if ip_vld
.get(vim_network_option
):
178 if isinstance(ip_vld
.get(vim_network_option
), list):
179 for vim_net_id
in ip_vld
.get(vim_network_option
):
180 for vim_account
, vim_net
in vim_net_id
.items():
181 RO_vld_sites
.append({
182 "netmap-use": vim_net
,
183 "datacenter": vim_account_2_RO(vim_account
)
185 elif isinstance(ip_vld
.get(vim_network_option
), dict):
186 for vim_account
, vim_net
in ip_vld
.get(vim_network_option
).items():
187 RO_vld_sites
.append({
188 "netmap-use": vim_net
,
189 "datacenter": vim_account_2_RO(vim_account
)
192 RO_vld_sites
.append({
193 "netmap-use": ip_vld
[vim_network_option
],
194 "datacenter": vim_account_2_RO(netslice_vld
["vimAccountId"])})
196 # Use default netslice vim-network-name from template
198 for nss_conn_point_ref
in get_iterable(netslice_vld
, "nss-connection-point-ref"):
199 if nss_conn_point_ref
.get("vimAccountId"):
200 if nss_conn_point_ref
["vimAccountId"] != netslice_vld
["vimAccountId"]:
201 RO_vld_sites
.append({
202 "netmap-create": None,
203 "datacenter": vim_account_2_RO(nss_conn_point_ref
["vimAccountId"])})
206 populate_dict(RO_ns_params
, ("networks", netslice_vld
["name"], "use-network"), vld_shared
)
209 populate_dict(RO_ns_params
, ("networks", netslice_vld
["name"], "sites"), RO_vld_sites
)
211 RO_ns_params
["scenario"] = {"nets": [{"name": netslice_vld
["name"],
212 "external": mgmt_network
, "type": "bridge"}]}
214 # self.logger.debug(logging_text + step)
215 db_nsir_update
["detailed-status"] = "Creating netslice-vld at RO"
216 desc
= await RO
.create("ns", descriptor
=RO_ns_params
)
217 db_nsir_update_RO
= {}
218 db_nsir_update_RO
["netslice_scenario_id"] = desc
["uuid"]
219 db_nsir_update_RO
["vld_id"] = RO_ns_params
["name"]
220 db_nsir_update
["_admin.deployed.RO"].append(db_nsir_update_RO
)
222 def overwrite_nsd_params(self
, db_nsir
, nslcmop
):
226 nsr_id
= nslcmop
.get("nsInstanceId")
227 # Overwrite instantiation parameters in netslice runtime
228 if db_nsir
.get("_admin"):
229 if db_nsir
["_admin"].get("deployed"):
230 db_admin_deployed_nsir
= db_nsir
["_admin"].get("deployed")
231 if db_admin_deployed_nsir
.get("RO"):
232 RO_list
= db_admin_deployed_nsir
["RO"]
234 for RO_item
in RO_list
:
235 for netslice_vld
in get_iterable(db_nsir
["_admin"], "netslice-vld"):
236 # if is equal vld of _admin with vld of netslice-vld then go for the CPs
237 if RO_item
.get("vld_id") == netslice_vld
.get("id"):
238 # Search the cp of netslice-vld that match with nst:netslice-subnet
239 for nss_cp_item
in get_iterable(netslice_vld
, "nss-connection-point-ref"):
240 # Search the netslice-subnet of nst that match
241 for nss
in get_iterable(db_nsir
["_admin"], "netslice-subnet"):
242 # Compare nss-ref equal nss from nst
243 if nss_cp_item
["nss-ref"] == nss
["nss-id"]:
244 db_nsds
= self
.db
.get_one("nsds", {"_id": nss
["nsdId"]})
245 # Go for nsd, and search the CP that match with nst:CP to get vld-id-ref
246 for cp_nsd
in db_nsds
["connection-point"]:
247 if cp_nsd
["name"] == nss_cp_item
["nsd-connection-point-ref"]:
248 if nslcmop
.get("operationParams"):
249 if nslcmop
["operationParams"].get("nsName") == nss
["nsName"]:
250 vld_id
= RO_item
["vld_id"]
251 netslice_scenario_id
= RO_item
["netslice_scenario_id"]
253 nslcmop_vld
["ns-net"] = {vld_id
: netslice_scenario_id
}
254 nslcmop_vld
["name"] = cp_nsd
["vld-id-ref"]
255 for vld
in get_iterable(nslcmop
["operationParams"], "vld"):
256 if vld
["name"] == cp_nsd
["vld-id-ref"]:
257 nslcmop_vld
.update(vld
)
258 vld_op_list
.append(nslcmop_vld
)
259 nslcmop
["operationParams"]["vld"] = vld_op_list
260 self
.update_db_2("nslcmops", nslcmop
["_id"], nslcmop
)
261 return nsr_id
, nslcmop
264 step
= "Getting nsir={} from db".format(nsir_id
)
265 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
266 step
= "Getting nsilcmop={} from db".format(nsilcmop_id
)
267 db_nsilcmop
= self
.db
.get_one("nsilcmops", {"_id": nsilcmop_id
})
269 # look if previous tasks is in process
270 task_name
, task_dependency
= self
.lcm_tasks
.lookfor_related("nsi", nsir_id
, nsilcmop_id
)
272 step
= db_nsilcmop_update
["detailed-status"] = \
273 "Waiting for related tasks to be completed: {}".format(task_name
)
274 self
.logger
.debug(logging_text
+ step
)
275 self
.update_db_2("nsilcmops", nsilcmop_id
, db_nsilcmop_update
)
276 _
, pending
= await asyncio
.wait(task_dependency
, timeout
=3600)
278 raise LcmException("Timeout waiting related tasks to be completed")
280 # Empty list to keep track of network service records status in the netslice
281 nsir_admin
= db_nsir_admin
= db_nsir
.get("_admin")
283 # Slice status Creating
284 db_nsir_update
["detailed-status"] = "creating"
285 db_nsir_update
["operational-status"] = "init"
286 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
288 # Creating netslice VLDs networking before NS instantiation
289 db_nsir_update
["_admin.deployed.RO"] = db_nsir_admin
["deployed"]["RO"]
290 for vld_item
in get_iterable(nsir_admin
, "netslice-vld"):
291 await netslice_scenario_create(self
, vld_item
, nsir_id
, db_nsir
, db_nsir_admin
, db_nsir_update
)
292 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
294 db_nsir_update
["detailed-status"] = "Creating netslice subnets at RO"
295 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
297 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
299 # Check status of the VLDs and wait for creation
300 # netslice_scenarios = db_nsir["_admin"]["deployed"]["RO"]
301 # db_nsir_update_RO = deepcopy(netslice_scenarios)
302 # for netslice_scenario in netslice_scenarios:
303 # await netslice_scenario_check(self, netslice_scenario["netslice_scenario_id"],
304 # nsir_id, db_nsir_update_RO)
306 # db_nsir_update["_admin.deployed.RO"] = db_nsir_update_RO
307 # self.update_db_2("nsis", nsir_id, db_nsir_update)
309 # Iterate over the network services operation ids to instantiate NSs
310 # TODO: (future improvement) look another way check the tasks instead of keep asking
311 # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
312 # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
313 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
314 nslcmop_ids
= db_nsilcmop
["operationParams"].get("nslcmops_ids")
315 for nslcmop_id
in nslcmop_ids
:
316 nslcmop
= self
.db
.get_one("nslcmops", {"_id": nslcmop_id
})
317 # Overwriting netslice-vld vim-net-id to ns
318 nsr_id
, nslcmop
= overwrite_nsd_params(self
, db_nsir
, nslcmop
)
319 step
= "Launching ns={} instantiate={} task".format(nsr_id
, nslcmop_id
)
320 task
= asyncio
.ensure_future(self
.ns
.instantiate(nsr_id
, nslcmop_id
))
321 self
.lcm_tasks
.register("ns", nsr_id
, nslcmop_id
, "ns_instantiate", task
)
323 # Wait until Network Slice is ready
324 step
= nsir_status_detailed
= " Waiting nsi ready. nsi_id={}".format(nsir_id
)
325 nsrs_detailed_list_old
= None
326 self
.logger
.debug(logging_text
+ step
)
328 # TODO: substitute while for await (all task to be done or not)
329 deployment_timeout
= 2 * 3600 # Two hours
330 while deployment_timeout
> 0:
331 # Check ns instantiation status
333 nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
334 nsir_admin
= nsir
["_admin"]
335 nsrs_detailed_list
= nsir
["_admin"]["nsrs-detailed-list"]
336 nsrs_detailed_list_new
= []
337 for nslcmop_item
in nslcmop_ids
:
338 nslcmop
= self
.db
.get_one("nslcmops", {"_id": nslcmop_item
})
339 status
= nslcmop
.get("operationState")
340 # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
341 for nss
in nsrs_detailed_list
:
342 if nss
["nsrId"] == nslcmop
["nsInstanceId"]:
343 nss
.update({"nsrId": nslcmop
["nsInstanceId"], "status": nslcmop
["operationState"],
345 nsir_status_detailed
+ "; {}".format(nslcmop
.get("detailed-status")),
346 "instantiated": True})
347 nsrs_detailed_list_new
.append(nss
)
348 if status
not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
351 if nsrs_detailed_list_new
!= nsrs_detailed_list_old
:
352 nsir_admin
["nsrs-detailed-list"] = nsrs_detailed_list_new
353 nsrs_detailed_list_old
= nsrs_detailed_list_new
354 db_nsir_update
["_admin"] = nsir_admin
355 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
358 step
= "Network Slice Instance is ready. nsi_id={}".format(nsir_id
)
359 for items
in nsrs_detailed_list
:
360 if "FAILED" in items
.values():
361 raise LcmException("Error deploying NSI: {}".format(nsir_id
))
364 # TODO: future improvement due to synchronism -> await asyncio.wait(vca_task_list, timeout=300)
365 await asyncio
.sleep(5, loop
=self
.loop
)
366 deployment_timeout
-= 5
368 if deployment_timeout
<= 0:
369 raise LcmException("Timeout waiting nsi to be ready. nsi_id={}".format(nsir_id
))
371 db_nsir_update
["operational-status"] = "running"
372 db_nsir_update
["detailed-status"] = "done"
373 db_nsir_update
["config-status"] = "configured"
374 db_nsilcmop_update
["operationState"] = nsilcmop_operation_state
= "COMPLETED"
375 db_nsilcmop_update
["statusEnteredTime"] = time()
376 db_nsilcmop_update
["detailed-status"] = "done"
379 except (LcmException
, DbException
) as e
:
380 self
.logger
.error(logging_text
+ "Exit Exception while '{}': {}".format(step
, e
))
382 except asyncio
.CancelledError
:
383 self
.logger
.error(logging_text
+ "Cancelled Exception while '{}'".format(step
))
384 exc
= "Operation was cancelled"
385 except Exception as e
:
386 exc
= traceback
.format_exc()
387 self
.logger
.critical(logging_text
+ "Exit Exception {} while '{}': {}".format(type(e
).__name
__, step
, e
),
392 db_nsir_update
["detailed-status"] = "ERROR {}: {}".format(step
, exc
)
393 db_nsir_update
["operational-status"] = "failed"
395 db_nsilcmop_update
["detailed-status"] = "FAILED {}: {}".format(step
, exc
)
396 db_nsilcmop_update
["operationState"] = nsilcmop_operation_state
= "FAILED"
397 db_nsilcmop_update
["statusEnteredTime"] = time()
400 db_nsir_update
["_admin.nsiState"] = "INSTANTIATED"
401 db_nsir_update
["_admin.nsilcmop"] = None
402 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
404 self
.update_db_2("nsilcmops", nsilcmop_id
, db_nsilcmop_update
)
405 except DbException
as e
:
406 self
.logger
.error(logging_text
+ "Cannot update database: {}".format(e
))
407 if nsilcmop_operation_state
:
409 await self
.msg
.aiowrite("nsi", "instantiated", {"nsir_id": nsir_id
, "nsilcmop_id": nsilcmop_id
,
410 "operationState": nsilcmop_operation_state
})
411 except Exception as e
:
412 self
.logger
.error(logging_text
+ "kafka_write notification Exception {}".format(e
))
413 self
.logger
.debug(logging_text
+ "Exit")
414 self
.lcm_tasks
.remove("nsi", nsir_id
, nsilcmop_id
, "nsi_instantiate")
416 async def terminate(self
, nsir_id
, nsilcmop_id
):
417 logging_text
= "Task nsi={} terminate={} ".format(nsir_id
, nsilcmop_id
)
418 self
.logger
.debug(logging_text
+ "Enter")
422 db_nsir_update
= {"_admin.nsilcmop": nsilcmop_id
}
423 db_nsilcmop_update
= {}
424 RO
= ROclient
.ROClient(self
.loop
, **self
.ro_config
)
426 failed_detail
= [] # annotates all failed error messages
427 nsilcmop_operation_state
= None
428 autoremove
= False # autoremove after terminated
430 step
= "Getting nsir={} from db".format(nsir_id
)
431 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
432 nsir_deployed
= deepcopy(db_nsir
["_admin"].get("deployed"))
433 step
= "Getting nsilcmop={} from db".format(nsilcmop_id
)
434 db_nsilcmop
= self
.db
.get_one("nsilcmops", {"_id": nsilcmop_id
})
436 # TODO: Check if makes sense check the nsiState=NOT_INSTANTIATED when terminate
437 # CASE: Instance was terminated but there is a second request to terminate the instance
438 if db_nsir
["_admin"]["nsiState"] == "NOT_INSTANTIATED":
441 # Slice status Terminating
442 db_nsir_update
["operational-status"] = "terminating"
443 db_nsir_update
["config-status"] = "terminating"
444 db_nsir_update
["detailed-status"] = "Terminating Netslice subnets"
445 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
447 # look if previous tasks is in process
448 task_name
, task_dependency
= self
.lcm_tasks
.lookfor_related("nsi", nsir_id
, nsilcmop_id
)
450 step
= db_nsilcmop_update
["detailed-status"] = \
451 "Waiting for related tasks to be completed: {}".format(task_name
)
452 self
.logger
.debug(logging_text
+ step
)
453 self
.update_db_2("nsilcmops", nsilcmop_id
, db_nsilcmop_update
)
454 _
, pending
= await asyncio
.wait(task_dependency
, timeout
=3600)
456 raise LcmException("Timeout waiting related tasks to be completed")
458 # Gets the list to keep track of network service records status in the netslice
459 nsir_admin
= db_nsir
["_admin"]
460 nsrs_detailed_list
= []
462 # Iterate over the network services operation ids to terminate NSs
463 # TODO: (future improvement) look another way check the tasks instead of keep asking
464 # -> https://docs.python.org/3/library/asyncio-task.html#waiting-primitives
465 # steps: declare ns_tasks, add task when terminate is called, await asyncio.wait(vca_task_list, timeout=300)
466 nslcmop_ids
= db_nsilcmop
["operationParams"].get("nslcmops_ids")
468 for nslcmop_id
in nslcmop_ids
:
469 nslcmop
= self
.db
.get_one("nslcmops", {"_id": nslcmop_id
})
470 nsr_id
= nslcmop
["operationParams"].get("nsInstanceId")
471 nss_in_use
= self
.db
.get_list("nsis", {"_admin.netslice-vld.ANYINDEX.shared-nsrs-list": nsr_id
,
472 "operational-status": {"$nin": ["terminated", "failed"]}})
473 if len(nss_in_use
) < 2:
474 task
= asyncio
.ensure_future(self
.ns
.terminate(nsr_id
, nslcmop_id
))
475 self
.lcm_tasks
.register("ns", nsr_id
, nslcmop_id
, "ns_instantiate", task
)
476 nslcmop_new
.append(nslcmop_id
)
478 # Update shared nslcmop shared with active nsi
479 netsliceInstanceId
= db_nsir
["_id"]
480 for nsis_item
in nss_in_use
:
481 if db_nsir
["_id"] != nsis_item
["_id"]:
482 netsliceInstanceId
= nsis_item
["_id"]
484 self
.db
.set_one("nslcmops", {"_id": nslcmop_id
},
485 {"operationParams.netsliceInstanceId": netsliceInstanceId
})
486 self
.db
.set_one("nsilcmops", {"_id": nsilcmop_id
}, {"operationParams.nslcmops_ids": nslcmop_new
})
488 # Wait until Network Slice is terminated
489 step
= nsir_status_detailed
= " Waiting nsi terminated. nsi_id={}".format(nsir_id
)
490 nsrs_detailed_list_old
= None
491 self
.logger
.debug(logging_text
+ step
)
493 termination_timeout
= 2 * 3600 # Two hours
494 while termination_timeout
> 0:
495 # Check ns termination status
497 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
498 nsir_admin
= db_nsir
["_admin"]
499 nsrs_detailed_list
= db_nsir
["_admin"].get("nsrs-detailed-list")
500 nsrs_detailed_list_new
= []
501 for nslcmop_item
in nslcmop_ids
:
502 nslcmop
= self
.db
.get_one("nslcmops", {"_id": nslcmop_item
})
503 status
= nslcmop
["operationState"]
504 # TODO: (future improvement) other possible status: ROLLING_BACK,ROLLED_BACK
505 for nss
in nsrs_detailed_list
:
506 if nss
["nsrId"] == nslcmop
["nsInstanceId"]:
507 nss
.update({"nsrId": nslcmop
["nsInstanceId"], "status": nslcmop
["operationState"],
509 nsir_status_detailed
+ "; {}".format(nslcmop
.get("detailed-status"))})
510 nsrs_detailed_list_new
.append(nss
)
511 if status
not in ["COMPLETED", "PARTIALLY_COMPLETED", "FAILED", "FAILED_TEMP"]:
514 if nsrs_detailed_list_new
!= nsrs_detailed_list_old
:
515 nsir_admin
["nsrs-detailed-list"] = nsrs_detailed_list_new
516 nsrs_detailed_list_old
= nsrs_detailed_list_new
517 db_nsir_update
["_admin"] = nsir_admin
518 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
521 # Check if it is the last used nss and mark isinstantiate: False
522 db_nsir
= self
.db
.get_one("nsis", {"_id": nsir_id
})
523 nsir_admin
= db_nsir
["_admin"]
524 nsrs_detailed_list
= db_nsir
["_admin"].get("nsrs-detailed-list")
525 for nss
in nsrs_detailed_list
:
526 _filter
= {"_admin.nsrs-detailed-list.ANYINDEX.nsrId": nss
["nsrId"],
527 "operational-status.ne": "terminated",
529 nsis_list
= self
.db
.get_one("nsis", _filter
, fail_on_empty
=False, fail_on_more
=False)
531 nss
.update({"instantiated": False})
532 db_nsir_update
["_admin"] = nsir_admin
533 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
535 step
= "Network Slice Instance is terminated. nsi_id={}".format(nsir_id
)
536 for items
in nsrs_detailed_list
:
537 if "FAILED" in items
.values():
538 raise LcmException("Error terminating NSI: {}".format(nsir_id
))
541 await asyncio
.sleep(5, loop
=self
.loop
)
542 termination_timeout
-= 5
544 if termination_timeout
<= 0:
545 raise LcmException("Timeout waiting nsi to be terminated. nsi_id={}".format(nsir_id
))
547 # Delete netslice-vlds
548 RO_nsir_id
= RO_delete_action
= None
549 for nsir_deployed_RO
in get_iterable(nsir_deployed
, "RO"):
550 RO_nsir_id
= nsir_deployed_RO
.get("netslice_scenario_id")
552 step
= db_nsir_update
["detailed-status"] = "Deleting netslice-vld at RO"
553 db_nsilcmop_update
["detailed-status"] = "Deleting netslice-vld at RO"
554 self
.logger
.debug(logging_text
+ step
)
555 desc
= await RO
.delete("ns", RO_nsir_id
)
556 RO_delete_action
= desc
["action_id"]
557 nsir_deployed_RO
["vld_delete_action_id"] = RO_delete_action
558 nsir_deployed_RO
["vld_status"] = "DELETING"
559 db_nsir_update
["_admin.deployed"] = nsir_deployed
560 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
562 # wait until NS is deleted from VIM
563 step
= "Waiting ns deleted from VIM. RO_id={}".format(RO_nsir_id
)
564 self
.logger
.debug(logging_text
+ step
)
565 except ROclient
.ROClientException
as e
:
566 if e
.http_code
== 404: # not found
567 nsir_deployed_RO
["vld_id"] = None
568 nsir_deployed_RO
["vld_status"] = "DELETED"
569 self
.logger
.debug(logging_text
+ "RO_ns_id={} already deleted".format(RO_nsir_id
))
570 elif e
.http_code
== 409: # conflict
571 failed_detail
.append("RO_ns_id={} delete conflict: {}".format(RO_nsir_id
, e
))
572 self
.logger
.debug(logging_text
+ failed_detail
[-1])
574 failed_detail
.append("RO_ns_id={} delete error: {}".format(RO_nsir_id
, e
))
575 self
.logger
.error(logging_text
+ failed_detail
[-1])
578 self
.logger
.error(logging_text
+ " ;".join(failed_detail
))
579 db_nsir_update
["operational-status"] = "failed"
580 db_nsir_update
["detailed-status"] = "Deletion errors " + "; ".join(failed_detail
)
581 db_nsilcmop_update
["detailed-status"] = "; ".join(failed_detail
)
582 db_nsilcmop_update
["operationState"] = nsilcmop_operation_state
= "FAILED"
583 db_nsilcmop_update
["statusEnteredTime"] = time()
585 db_nsir_update
["operational-status"] = "terminating"
586 db_nsir_update
["config-status"] = "terminating"
587 db_nsir_update
["_admin.nsiState"] = "NOT_INSTANTIATED"
588 db_nsilcmop_update
["operationState"] = nsilcmop_operation_state
= "COMPLETED"
589 db_nsilcmop_update
["statusEnteredTime"] = time()
590 if db_nsilcmop
["operationParams"].get("autoremove"):
593 db_nsir_update
["detailed-status"] = "done"
594 db_nsir_update
["operational-status"] = "terminated"
595 db_nsir_update
["config-status"] = "terminated"
596 db_nsilcmop_update
["statusEnteredTime"] = time()
597 db_nsilcmop_update
["detailed-status"] = "done"
600 except (LcmException
, DbException
) as e
:
601 self
.logger
.error(logging_text
+ "Exit Exception while '{}': {}".format(step
, e
))
603 except asyncio
.CancelledError
:
604 self
.logger
.error(logging_text
+ "Cancelled Exception while '{}'".format(step
))
605 exc
= "Operation was cancelled"
606 except Exception as e
:
607 exc
= traceback
.format_exc()
608 self
.logger
.critical(logging_text
+ "Exit Exception {} while '{}': {}".format(type(e
).__name
__, step
, e
),
613 db_nsir_update
["_admin.deployed"] = nsir_deployed
614 db_nsir_update
["detailed-status"] = "ERROR {}: {}".format(step
, exc
)
615 db_nsir_update
["operational-status"] = "failed"
617 db_nsilcmop_update
["detailed-status"] = "FAILED {}: {}".format(step
, exc
)
618 db_nsilcmop_update
["operationState"] = nsilcmop_operation_state
= "FAILED"
619 db_nsilcmop_update
["statusEnteredTime"] = time()
622 db_nsir_update
["_admin.deployed"] = nsir_deployed
623 db_nsir_update
["_admin.nsilcmop"] = None
624 self
.update_db_2("nsis", nsir_id
, db_nsir_update
)
626 self
.update_db_2("nsilcmops", nsilcmop_id
, db_nsilcmop_update
)
627 except DbException
as e
:
628 self
.logger
.error(logging_text
+ "Cannot update database: {}".format(e
))
630 if nsilcmop_operation_state
:
632 await self
.msg
.aiowrite("nsi", "terminated", {"nsir_id": nsir_id
, "nsilcmop_id": nsilcmop_id
,
633 "operationState": nsilcmop_operation_state
,
634 "autoremove": autoremove
},
636 except Exception as e
:
637 self
.logger
.error(logging_text
+ "kafka_write notification Exception {}".format(e
))
638 self
.logger
.debug(logging_text
+ "Exit")
639 self
.lcm_tasks
.remove("nsi", nsir_id
, nsilcmop_id
, "nsi_terminate")