adding more system tests.
[osm/NBI.git] / osm_nbi / descriptor_topics.py
1 # -*- coding: utf-8 -*-
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12 # implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import tarfile
17 import yaml
18 import json
19 # import logging
20 from hashlib import md5
21 from osm_common.dbbase import DbException, deep_update_rfc7396
22 from http import HTTPStatus
23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
24 from base_topic import BaseTopic, EngineException, get_iterable
25 from osm_im.vnfd import vnfd as vnfd_im
26 from osm_im.nsd import nsd as nsd_im
27 from osm_im.nst import nst as nst_im
28 from pyangbind.lib.serialise import pybindJSONDecoder
29 import pyangbind.lib.pybindJSON as pybindJSON
30
31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
32
33
34 class DescriptorTopic(BaseTopic):
35
36 def __init__(self, db, fs, msg):
37 BaseTopic.__init__(self, db, fs, msg)
38
39 def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
40 # 1. validate again with pyangbind
41 # 1.1. remove internal keys
42 internal_keys = {}
43 for k in ("_id", "_admin"):
44 if k in final_content:
45 internal_keys[k] = final_content.pop(k)
46 serialized = self._validate_input_new(final_content, force)
47 # 1.2. modify final_content with a serialized version
48 final_content.clear()
49 final_content.update(serialized)
50 # 1.3. restore internal keys
51 for k, v in internal_keys.items():
52 final_content[k] = v
53
54 if force:
55 return
56 # 2. check that this id is not present
57 if "id" in edit_content:
58 _filter = self._get_project_filter(session, write=False, show_all=False)
59 _filter["id"] = final_content["id"]
60 _filter["_id.neq"] = _id
61 if self.db.get_one(self.topic, _filter, fail_on_empty=False):
62 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
63 final_content["id"]),
64 HTTPStatus.CONFLICT)
65
66 @staticmethod
67 def format_on_new(content, project_id=None, make_public=False):
68 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
69 content["_admin"]["onboardingState"] = "CREATED"
70 content["_admin"]["operationalState"] = "DISABLED"
71 content["_admin"]["usageState"] = "NOT_IN_USE"
72
73 def delete(self, session, _id, force=False, dry_run=False):
74 """
75 Delete item by its internal _id
76 :param session: contains the used login username, working project, and admin rights
77 :param _id: server internal id
78 :param force: indicates if deletion must be forced in case of conflict
79 :param dry_run: make checking but do not delete
80 :return: dictionary with deleted item _id. It raises EngineException on error: not found, conflict, ...
81 """
82 # TODO add admin to filter, validate rights
83 v = BaseTopic.delete(self, session, _id, force, dry_run=True)
84 if dry_run:
85 return
86 v = self.db.del_one(self.topic, {"_id": _id})
87 self.fs.file_delete(_id, ignore_non_exist=True)
88 self._send_msg("delete", {"_id": _id})
89 return v
90
91 @staticmethod
92 def get_one_by_id(db, session, topic, id):
93 # find owned by this project
94 _filter = BaseTopic._get_project_filter(session, write=False, show_all=False)
95 _filter["id"] = id
96 desc_list = db.get_list(topic, _filter)
97 if len(desc_list) == 1:
98 return desc_list[0]
99 elif len(desc_list) > 1:
100 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
101 HTTPStatus.CONFLICT)
102
103 # not found any: try to find public
104 _filter = BaseTopic._get_project_filter(session, write=False, show_all=True)
105 _filter["id"] = id
106 desc_list = db.get_list(topic, _filter)
107 if not desc_list:
108 raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
109 elif len(desc_list) == 1:
110 return desc_list[0]
111 else:
112 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
113 topic[:-1], id), HTTPStatus.CONFLICT)
114
115 def new(self, rollback, session, indata=None, kwargs=None, headers=None, force=False, make_public=False):
116 """
117 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
118 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
119 (self.upload_content)
120 :param rollback: list to append created items at database in case a rollback may to be done
121 :param session: contains the used login username and working project
122 :param indata: data to be inserted
123 :param kwargs: used to override the indata descriptor
124 :param headers: http request headers
125 :param force: If True avoid some dependence checks
126 :param make_public: Make the created descriptor public to all projects
127 :return: _id: identity of the inserted data.
128 """
129
130 try:
131 # _remove_envelop
132 if indata:
133 if "userDefinedData" in indata:
134 indata = indata['userDefinedData']
135
136 # Override descriptor with query string kwargs
137 self._update_input_with_kwargs(indata, kwargs)
138 # uncomment when this method is implemented.
139 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
140 # indata = DescriptorTopic._validate_input_new(self, indata, force=force)
141
142 content = {"_admin": {"userDefinedData": indata}}
143 self.format_on_new(content, session["project_id"], make_public=make_public)
144 _id = self.db.create(self.topic, content)
145 rollback.append({"topic": self.topic, "_id": _id})
146 return _id
147 except ValidationError as e:
148 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
149
150 def upload_content(self, session, _id, indata, kwargs, headers, force=False):
151 """
152 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
153 :param session: session
154 :param _id : the nsd,vnfd is already created, this is the id
155 :param indata: http body request
156 :param kwargs: user query string to override parameters. NOT USED
157 :param headers: http request headers
158 :param force: to be more tolerant with validation
159 :return: True if package is completely uploaded or False if partial content has been uploded
160 Raise exception on error
161 """
162 # Check that _id exists and it is valid
163 current_desc = self.show(session, _id)
164
165 content_range_text = headers.get("Content-Range")
166 expected_md5 = headers.get("Content-File-MD5")
167 compressed = None
168 content_type = headers.get("Content-Type")
169 if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
170 "application/zip" in content_type:
171 compressed = "gzip"
172 filename = headers.get("Content-Filename")
173 if not filename:
174 filename = "package.tar.gz" if compressed else "package"
175 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
176 file_pkg = None
177 error_text = ""
178 try:
179 if content_range_text:
180 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
181 if content_range[0] != "bytes": # TODO check x<y not negative < total....
182 raise IndexError()
183 start = int(content_range[1])
184 end = int(content_range[2]) + 1
185 total = int(content_range[3])
186 else:
187 start = 0
188
189 if start:
190 if not self.fs.file_exists(_id, 'dir'):
191 raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
192 else:
193 self.fs.file_delete(_id, ignore_non_exist=True)
194 self.fs.mkdir(_id)
195
196 storage = self.fs.get_params()
197 storage["folder"] = _id
198
199 file_path = (_id, filename)
200 if self.fs.file_exists(file_path, 'file'):
201 file_size = self.fs.file_size(file_path)
202 else:
203 file_size = 0
204 if file_size != start:
205 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
206 file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
207 file_pkg = self.fs.file_open(file_path, 'a+b')
208 if isinstance(indata, dict):
209 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
210 file_pkg.write(indata_text.encode(encoding="utf-8"))
211 else:
212 indata_len = 0
213 while True:
214 indata_text = indata.read(4096)
215 indata_len += len(indata_text)
216 if not indata_text:
217 break
218 file_pkg.write(indata_text)
219 if content_range_text:
220 if indata_len != end-start:
221 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
222 start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
223 if end != total:
224 # TODO update to UPLOADING
225 return False
226
227 # PACKAGE UPLOADED
228 if expected_md5:
229 file_pkg.seek(0, 0)
230 file_md5 = md5()
231 chunk_data = file_pkg.read(1024)
232 while chunk_data:
233 file_md5.update(chunk_data)
234 chunk_data = file_pkg.read(1024)
235 if expected_md5 != file_md5.hexdigest():
236 raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
237 file_pkg.seek(0, 0)
238 if compressed == "gzip":
239 tar = tarfile.open(mode='r', fileobj=file_pkg)
240 descriptor_file_name = None
241 for tarinfo in tar:
242 tarname = tarinfo.name
243 tarname_path = tarname.split("/")
244 if not tarname_path[0] or ".." in tarname_path: # if start with "/" means absolute path
245 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
246 if len(tarname_path) == 1 and not tarinfo.isdir():
247 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
248 if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
249 storage["pkg-dir"] = tarname_path[0]
250 if len(tarname_path) == 2:
251 if descriptor_file_name:
252 raise EngineException(
253 "Found more than one descriptor file at package descriptor tar.gz")
254 descriptor_file_name = tarname
255 if not descriptor_file_name:
256 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
257 storage["descriptor"] = descriptor_file_name
258 storage["zipfile"] = filename
259 self.fs.file_extract(tar, _id)
260 with self.fs.file_open((_id, descriptor_file_name), "r") as descriptor_file:
261 content = descriptor_file.read()
262 else:
263 content = file_pkg.read()
264 storage["descriptor"] = descriptor_file_name = filename
265
266 if descriptor_file_name.endswith(".json"):
267 error_text = "Invalid json format "
268 indata = json.load(content)
269 else:
270 error_text = "Invalid yaml format "
271 indata = yaml.load(content)
272
273 current_desc["_admin"]["storage"] = storage
274 current_desc["_admin"]["onboardingState"] = "ONBOARDED"
275 current_desc["_admin"]["operationalState"] = "ENABLED"
276
277 indata = self._remove_envelop(indata)
278
279 # Override descriptor with query string kwargs
280 if kwargs:
281 self._update_input_with_kwargs(indata, kwargs)
282 # it will call overrides method at VnfdTopic or NsdTopic
283 # indata = self._validate_input_edit(indata, force=force)
284
285 deep_update_rfc7396(current_desc, indata)
286 self.check_conflict_on_edit(session, current_desc, indata, _id=_id, force=force)
287 self.db.replace(self.topic, _id, current_desc)
288
289 indata["_id"] = _id
290 self._send_msg("created", indata)
291
292 # TODO if descriptor has changed because kwargs update content and remove cached zip
293 # TODO if zip is not present creates one
294 return True
295
296 except EngineException:
297 raise
298 except IndexError:
299 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
300 HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
301 except IOError as e:
302 raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
303 except tarfile.ReadError as e:
304 raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
305 except (ValueError, yaml.YAMLError) as e:
306 raise EngineException(error_text + str(e))
307 except ValidationError as e:
308 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
309 finally:
310 if file_pkg:
311 file_pkg.close()
312
313 def get_file(self, session, _id, path=None, accept_header=None):
314 """
315 Return the file content of a vnfd or nsd
316 :param session: contains the used login username and working project
317 :param _id: Identity of the vnfd, nsd
318 :param path: artifact path or "$DESCRIPTOR" or None
319 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
320 :return: opened file plus Accept format or raises an exception
321 """
322 accept_text = accept_zip = False
323 if accept_header:
324 if 'text/plain' in accept_header or '*/*' in accept_header:
325 accept_text = True
326 if 'application/zip' in accept_header or '*/*' in accept_header:
327 accept_zip = 'application/zip'
328 elif 'application/gzip' in accept_header:
329 accept_zip = 'application/gzip'
330
331 if not accept_text and not accept_zip:
332 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
333 http_code=HTTPStatus.NOT_ACCEPTABLE)
334
335 content = self.show(session, _id)
336 if content["_admin"]["onboardingState"] != "ONBOARDED":
337 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
338 "onboardingState is {}".format(content["_admin"]["onboardingState"]),
339 http_code=HTTPStatus.CONFLICT)
340 storage = content["_admin"]["storage"]
341 if path is not None and path != "$DESCRIPTOR": # artifacts
342 if not storage.get('pkg-dir'):
343 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
344 if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
345 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
346 return folder_content, "text/plain"
347 # TODO manage folders in http
348 else:
349 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
350 "application/octet-stream"
351
352 # pkgtype accept ZIP TEXT -> result
353 # manyfiles yes X -> zip
354 # no yes -> error
355 # onefile yes no -> zip
356 # X yes -> text
357
358 if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
359 return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
360 elif storage.get('pkg-dir') and not accept_zip:
361 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
362 "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
363 else:
364 if not storage.get('zipfile'):
365 # TODO generate zipfile if not present
366 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
367 "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
368 return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
369
370 def pyangbind_validation(self, item, data, force=False):
371 try:
372 if item == "vnfds":
373 myvnfd = vnfd_im()
374 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
375 path_helper=True, skip_unknown=force)
376 out = pybindJSON.dumps(myvnfd, mode="ietf")
377 elif item == "nsds":
378 mynsd = nsd_im()
379 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
380 path_helper=True, skip_unknown=force)
381 out = pybindJSON.dumps(mynsd, mode="ietf")
382 elif item == "nsts":
383 mynst = nst_im()
384 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
385 path_helper=True, skip_unknown=force)
386 out = pybindJSON.dumps(mynst, mode="ietf")
387 else:
388 raise EngineException("Not possible to validate '{}' item".format(item),
389 http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
390
391 desc_out = self._remove_envelop(yaml.safe_load(out))
392 return desc_out
393
394 except Exception as e:
395 raise EngineException("Error in pyangbind validation: {}".format(str(e)),
396 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
397
398
399 class VnfdTopic(DescriptorTopic):
400 topic = "vnfds"
401 topic_msg = "vnfd"
402
403 def __init__(self, db, fs, msg):
404 DescriptorTopic.__init__(self, db, fs, msg)
405
406 @staticmethod
407 def _remove_envelop(indata=None):
408 if not indata:
409 return {}
410 clean_indata = indata
411 if clean_indata.get('vnfd:vnfd-catalog'):
412 clean_indata = clean_indata['vnfd:vnfd-catalog']
413 elif clean_indata.get('vnfd-catalog'):
414 clean_indata = clean_indata['vnfd-catalog']
415 if clean_indata.get('vnfd'):
416 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
417 raise EngineException("'vnfd' must be a list of only one element")
418 clean_indata = clean_indata['vnfd'][0]
419 elif clean_indata.get('vnfd:vnfd'):
420 if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
421 raise EngineException("'vnfd:vnfd' must be a list of only one element")
422 clean_indata = clean_indata['vnfd:vnfd'][0]
423 return clean_indata
424
425 def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
426 super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
427
428 # set type of vnfd
429 contains_pdu = False
430 contains_vdu = False
431 for vdu in get_iterable(final_content.get("vdu")):
432 if vdu.get("pdu-type"):
433 contains_pdu = True
434 else:
435 contains_vdu = True
436 if contains_pdu:
437 final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
438 elif contains_vdu:
439 final_content["_admin"]["type"] = "vnfd"
440 # if neither vud nor pdu do not fill type
441
442 def check_conflict_on_del(self, session, _id, force=False):
443 """
444 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
445 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
446 that uses this vnfd
447 :param session:
448 :param _id: vnfd inernal id
449 :param force: Avoid this checking
450 :return: None or raises EngineException with the conflict
451 """
452 if force:
453 return
454 descriptor = self.db.get_one("vnfds", {"_id": _id})
455 descriptor_id = descriptor.get("id")
456 if not descriptor_id: # empty vnfd not uploaded
457 return
458
459 _filter = self._get_project_filter(session, write=False, show_all=False)
460 # check vnfrs using this vnfd
461 _filter["vnfd-id"] = _id
462 if self.db.get_list("vnfrs", _filter):
463 raise EngineException("There is some VNFR that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
464 del _filter["vnfd-id"]
465 # check NSD using this VNFD
466 _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
467 if self.db.get_list("nsds", _filter):
468 raise EngineException("There is soame NSD that depends on this VNFD", http_code=HTTPStatus.CONFLICT)
469
470 def _validate_input_new(self, indata, force=False):
471 indata = self.pyangbind_validation("vnfds", indata, force)
472 # Cross references validation in the descriptor
473 if indata.get("vdu"):
474 if not indata.get("mgmt-interface"):
475 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
476 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
477 if indata["mgmt-interface"].get("cp"):
478 for cp in get_iterable(indata.get("connection-point")):
479 if cp["name"] == indata["mgmt-interface"]["cp"]:
480 break
481 else:
482 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
483 .format(indata["mgmt-interface"]["cp"]),
484 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
485
486 for vdu in get_iterable(indata.get("vdu")):
487 for interface in get_iterable(vdu.get("interface")):
488 if interface.get("external-connection-point-ref"):
489 for cp in get_iterable(indata.get("connection-point")):
490 if cp["name"] == interface["external-connection-point-ref"]:
491 break
492 else:
493 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
494 "must match an existing connection-point"
495 .format(vdu["id"], interface["name"],
496 interface["external-connection-point-ref"]),
497 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
498
499 elif interface.get("internal-connection-point-ref"):
500 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
501 if interface["internal-connection-point-ref"] == internal_cp.get("id"):
502 break
503 else:
504 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
505 "must match an existing vdu:internal-connection-point"
506 .format(vdu["id"], interface["name"],
507 interface["internal-connection-point-ref"]),
508 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
509 for ivld in get_iterable(indata.get("internal-vld")):
510 for icp in get_iterable(ivld.get("internal-connection-point")):
511 icp_mark = False
512 for vdu in get_iterable(indata.get("vdu")):
513 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
514 if icp["id-ref"] == internal_cp["id"]:
515 icp_mark = True
516 break
517 if icp_mark:
518 break
519 else:
520 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
521 "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
522 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
523 if ivld.get("ip-profile-ref"):
524 for ip_prof in get_iterable(indata.get("ip-profiles")):
525 if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
526 break
527 else:
528 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
529 ivld["id"], ivld["ip-profile-ref"]),
530 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
531 for mp in get_iterable(indata.get("monitoring-param")):
532 if mp.get("vdu-monitoring-param"):
533 mp_vmp_mark = False
534 for vdu in get_iterable(indata.get("vdu")):
535 for vmp in get_iterable(vdu.get("monitoring-param")):
536 if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
537 mp["vdu-monitoring-param"]["vdu-ref"]:
538 mp_vmp_mark = True
539 break
540 if mp_vmp_mark:
541 break
542 else:
543 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
544 "defined at vdu[id='{}'] or vdu does not exist"
545 .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
546 mp["vdu-monitoring-param"]["vdu-ref"]),
547 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
548 elif mp.get("vdu-metric"):
549 mp_vm_mark = False
550 for vdu in get_iterable(indata.get("vdu")):
551 if vdu.get("vdu-configuration"):
552 for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
553 if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
554 mp["vdu-metric"]["vdu-ref"]:
555 mp_vm_mark = True
556 break
557 if mp_vm_mark:
558 break
559 else:
560 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
561 "vdu[id='{}'] or vdu does not exist"
562 .format(mp["vdu-metric"]["vdu-metric-name-ref"],
563 mp["vdu-metric"]["vdu-ref"]),
564 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
565
566 for sgd in get_iterable(indata.get("scaling-group-descriptor")):
567 for sp in get_iterable(sgd.get("scaling-policy")):
568 for sc in get_iterable(sp.get("scaling-criteria")):
569 for mp in get_iterable(indata.get("monitoring-param")):
570 if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
571 break
572 else:
573 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
574 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
575 .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
576 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
577 for sgd_vdu in get_iterable(sgd.get("vdu")):
578 sgd_vdu_mark = False
579 for vdu in get_iterable(indata.get("vdu")):
580 if vdu["id"] == sgd_vdu["vdu-id-ref"]:
581 sgd_vdu_mark = True
582 break
583 if sgd_vdu_mark:
584 break
585 else:
586 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
587 .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
588 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
589 for sca in get_iterable(sgd.get("scaling-config-action")):
590 if not indata.get("vnf-configuration"):
591 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
592 "scaling-group-descriptor[name='{}']:scaling-config-action"
593 .format(sgd["name"]),
594 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
595 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
596 if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
597 break
598 else:
599 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
600 "primitive-name-ref='{}' does not match any "
601 "vnf-configuration:config-primitive:name"
602 .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
603 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
604 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
605 return indata
606
607 def _validate_input_edit(self, indata, force=False):
608 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
609 return indata
610
611
612 class NsdTopic(DescriptorTopic):
613 topic = "nsds"
614 topic_msg = "nsd"
615
616 def __init__(self, db, fs, msg):
617 DescriptorTopic.__init__(self, db, fs, msg)
618
619 @staticmethod
620 def _remove_envelop(indata=None):
621 if not indata:
622 return {}
623 clean_indata = indata
624
625 if clean_indata.get('nsd:nsd-catalog'):
626 clean_indata = clean_indata['nsd:nsd-catalog']
627 elif clean_indata.get('nsd-catalog'):
628 clean_indata = clean_indata['nsd-catalog']
629 if clean_indata.get('nsd'):
630 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
631 raise EngineException("'nsd' must be a list of only one element")
632 clean_indata = clean_indata['nsd'][0]
633 elif clean_indata.get('nsd:nsd'):
634 if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
635 raise EngineException("'nsd:nsd' must be a list of only one element")
636 clean_indata = clean_indata['nsd:nsd'][0]
637 return clean_indata
638
639 def _validate_input_new(self, indata, force=False):
640 indata = self.pyangbind_validation("nsds", indata, force)
641 # Cross references validation in the descriptor
642 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
643 for vld in get_iterable(indata.get("vld")):
644 for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
645 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
646 if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
647 if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
648 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
649 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
650 " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
651 constituent_vnfd["member-vnf-index"],
652 constituent_vnfd["vnfd-id-ref"]),
653 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
654 break
655 else:
656 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
657 "does not match any constituent-vnfd:member-vnf-index"
658 .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
659 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
660 return indata
661
662 def _validate_input_edit(self, indata, force=False):
663 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
664 return indata
665
666 def _check_descriptor_dependencies(self, session, descriptor, force=False):
667 """
668 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
669 connection points are ok
670 :param session: client session information
671 :param descriptor: descriptor to be inserted or edit
672 :param force: if true skip dependencies checking
673 :return: None or raises exception
674 """
675 if force:
676 return
677 member_vnfd_index = {}
678 if descriptor.get("constituent-vnfd") and not force:
679 for vnf in descriptor["constituent-vnfd"]:
680 vnfd_id = vnf["vnfd-id-ref"]
681 filter_q = self._get_project_filter(session, write=False, show_all=True)
682 filter_q["id"] = vnfd_id
683 vnf_list = self.db.get_list("vnfds", filter_q)
684 if not vnf_list:
685 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
686 "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
687 # elif len(vnf_list) > 1:
688 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
689 # http_code=HTTPStatus.CONFLICT)
690 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
691
692 # Cross references validation in the descriptor and vnfd connection point validation
693 for vld in get_iterable(descriptor.get("vld")):
694 for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
695 # look if this vnfd contains this connection point
696 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
697 if not vnfd:
698 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
699 "does not match any constituent-vnfd:member-vnf-index"
700 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
701 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
702 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
703 if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
704 break
705 else:
706 raise EngineException(
707 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
708 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
709 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
710 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
711 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
712
713 def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
714 super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
715
716 self._check_descriptor_dependencies(session, final_content, force)
717
718 def check_conflict_on_del(self, session, _id, force=False):
719 """
720 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
721 that NSD can be public and be used by other projects.
722 :param session:
723 :param _id: vnfd inernal id
724 :param force: Avoid this checking
725 :return: None or raises EngineException with the conflict
726 """
727 if force:
728 return
729 _filter = self._get_project_filter(session, write=False, show_all=False)
730 _filter["nsdId"] = _id
731 if self.db.get_list("nsrs", _filter):
732 raise EngineException("There is some NSR that depends on this NSD", http_code=HTTPStatus.CONFLICT)
733
734
735 class NstTopic(DescriptorTopic):
736 topic = "nsts"
737 topic_msg = "nst"
738
739 def __init__(self, db, fs, msg):
740 DescriptorTopic.__init__(self, db, fs, msg)
741
742 @staticmethod
743 def _remove_envelop(indata=None):
744 if not indata:
745 return {}
746 clean_indata = indata
747
748 if clean_indata.get('nst'):
749 if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
750 raise EngineException("'nst' must be a list only one element")
751 clean_indata = clean_indata['nst'][0]
752 elif clean_indata.get('nst:nst'):
753 if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
754 raise EngineException("'nst:nst' must be a list only one element")
755 clean_indata = clean_indata['nst:nst'][0]
756 return clean_indata
757
758 def _validate_input_edit(self, indata, force=False):
759 # TODO validate with pyangbind, serialize
760 return indata
761
762 def _validate_input_new(self, indata, force=False):
763 indata = self.pyangbind_validation("nsts", indata, force)
764 return indata.copy()
765
766 def _check_descriptor_dependencies(self, session, descriptor):
767 """
768 Check that the dependent descriptors exist on a new descriptor or edition
769 :param session: client session information
770 :param descriptor: descriptor to be inserted or edit
771 :return: None or raises exception
772 """
773 if not descriptor.get("netslice-subnet"):
774 return
775 for nsd in descriptor["netslice-subnet"]:
776 nsd_id = nsd["nsd-ref"]
777 filter_q = self._get_project_filter(session, write=False, show_all=True)
778 filter_q["id"] = nsd_id
779 if not self.db.get_list("nsds", filter_q):
780 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
781 "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
782
783 def check_conflict_on_edit(self, session, final_content, edit_content, _id, force=False):
784 super().check_conflict_on_edit(session, final_content, edit_content, _id, force=force)
785
786 self._check_descriptor_dependencies(session, final_content)
787
788 def check_conflict_on_del(self, session, _id, force=False):
789 """
790 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
791 that NST can be public and be used by other projects.
792 :param session:
793 :param _id: nst internal id
794 :param force: Avoid this checking
795 :return: None or raises EngineException with the conflict
796 """
797 # TODO: Check this method
798 if force:
799 return
800 # Get Network Slice Template from Database
801 _filter = self._get_project_filter(session, write=False, show_all=False)
802 _filter["_id"] = _id
803 nst = self.db.get_one("nsts", _filter)
804
805 # Search NSIs using NST via nst-ref
806 _filter = self._get_project_filter(session, write=False, show_all=False)
807 _filter["nst-ref"] = nst["id"]
808 nsis_list = self.db.get_list("nsis", _filter)
809 for nsi_item in nsis_list:
810 if nsi_item["_admin"].get("nsiState") != "TERMINATED":
811 raise EngineException("There is some NSIS that depends on this NST", http_code=HTTPStatus.CONFLICT)
812
813
814 class PduTopic(BaseTopic):
815 topic = "pdus"
816 topic_msg = "pdu"
817 schema_new = pdu_new_schema
818 schema_edit = pdu_edit_schema
819
820 def __init__(self, db, fs, msg):
821 BaseTopic.__init__(self, db, fs, msg)
822
823 @staticmethod
824 def format_on_new(content, project_id=None, make_public=False):
825 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
826 content["_admin"]["onboardingState"] = "CREATED"
827 content["_admin"]["operationalState"] = "ENABLED"
828 content["_admin"]["usageState"] = "NOT_IN_USE"
829
830 def check_conflict_on_del(self, session, _id, force=False):
831 if force:
832 return
833 # TODO Is it needed to check descriptors _admin.project_read/project_write??
834 _filter = {"vdur.pdu-id": _id}
835 if self.db.get_list("vnfrs", _filter):
836 raise EngineException("There is some NSR that uses this PDU", http_code=HTTPStatus.CONFLICT)