fix bug 791. Adding input validation for ns-create. It was only for ns-instantiate
[osm/NBI.git] / osm_nbi / descriptor_topics.py
1 # -*- coding: utf-8 -*-
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12 # implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import tarfile
17 import yaml
18 import json
19 # import logging
20 from hashlib import md5
21 from osm_common.dbbase import DbException, deep_update_rfc7396
22 from http import HTTPStatus
23 from validation import ValidationError, pdu_new_schema, pdu_edit_schema
24 from base_topic import BaseTopic, EngineException, get_iterable
25 from osm_im.vnfd import vnfd as vnfd_im
26 from osm_im.nsd import nsd as nsd_im
27 from osm_im.nst import nst as nst_im
28 from pyangbind.lib.serialise import pybindJSONDecoder
29 import pyangbind.lib.pybindJSON as pybindJSON
30
31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
32
33
34 class DescriptorTopic(BaseTopic):
35
36 def __init__(self, db, fs, msg):
37 BaseTopic.__init__(self, db, fs, msg)
38
39 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
40 super().check_conflict_on_edit(session, final_content, edit_content, _id)
41 # 1. validate again with pyangbind
42 # 1.1. remove internal keys
43 internal_keys = {}
44 for k in ("_id", "_admin"):
45 if k in final_content:
46 internal_keys[k] = final_content.pop(k)
47 storage_params = internal_keys["_admin"].get("storage")
48 serialized = self._validate_input_new(final_content, storage_params, session["force"])
49 # 1.2. modify final_content with a serialized version
50 final_content.clear()
51 final_content.update(serialized)
52 # 1.3. restore internal keys
53 for k, v in internal_keys.items():
54 final_content[k] = v
55
56 if session["force"]:
57 return
58 # 2. check that this id is not present
59 if "id" in edit_content:
60 _filter = self._get_project_filter(session)
61 _filter["id"] = final_content["id"]
62 _filter["_id.neq"] = _id
63 if self.db.get_one(self.topic, _filter, fail_on_empty=False):
64 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
65 final_content["id"]),
66 HTTPStatus.CONFLICT)
67
68 @staticmethod
69 def format_on_new(content, project_id=None, make_public=False):
70 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
71 content["_admin"]["onboardingState"] = "CREATED"
72 content["_admin"]["operationalState"] = "DISABLED"
73 content["_admin"]["usageState"] = "NOT_IN_USE"
74
75 def delete_extra(self, session, _id, db_content):
76 """
77 Deletes file system storage associated with the descriptor
78 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
79 :param _id: server internal id
80 :param db_content: The database content of the descriptor
81 :return: None if ok or raises EngineException with the problem
82 """
83 self.fs.file_delete(_id, ignore_non_exist=True)
84 self.fs.file_delete(_id + "_", ignore_non_exist=True) # remove temp folder
85
86 @staticmethod
87 def get_one_by_id(db, session, topic, id):
88 # find owned by this project
89 _filter = BaseTopic._get_project_filter(session)
90 _filter["id"] = id
91 desc_list = db.get_list(topic, _filter)
92 if len(desc_list) == 1:
93 return desc_list[0]
94 elif len(desc_list) > 1:
95 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
96 HTTPStatus.CONFLICT)
97
98 # not found any: try to find public
99 _filter = BaseTopic._get_project_filter(session)
100 _filter["id"] = id
101 desc_list = db.get_list(topic, _filter)
102 if not desc_list:
103 raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
104 elif len(desc_list) == 1:
105 return desc_list[0]
106 else:
107 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
108 topic[:-1], id), HTTPStatus.CONFLICT)
109
110 def new(self, rollback, session, indata=None, kwargs=None, headers=None):
111 """
112 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
113 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
114 (self.upload_content)
115 :param rollback: list to append created items at database in case a rollback may to be done
116 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
117 :param indata: data to be inserted
118 :param kwargs: used to override the indata descriptor
119 :param headers: http request headers
120 :return: _id: identity of the inserted data.
121 """
122
123 try:
124 # _remove_envelop
125 if indata:
126 if "userDefinedData" in indata:
127 indata = indata['userDefinedData']
128
129 # Override descriptor with query string kwargs
130 self._update_input_with_kwargs(indata, kwargs)
131 # uncomment when this method is implemented.
132 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
133 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
134
135 content = {"_admin": {"userDefinedData": indata}}
136 self.format_on_new(content, session["project_id"], make_public=session["public"])
137 _id = self.db.create(self.topic, content)
138 rollback.append({"topic": self.topic, "_id": _id})
139 return _id
140 except ValidationError as e:
141 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
142
143 def upload_content(self, session, _id, indata, kwargs, headers):
144 """
145 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
146 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
147 :param _id : the nsd,vnfd is already created, this is the id
148 :param indata: http body request
149 :param kwargs: user query string to override parameters. NOT USED
150 :param headers: http request headers
151 :return: True if package is completely uploaded or False if partial content has been uploded
152 Raise exception on error
153 """
154 # Check that _id exists and it is valid
155 current_desc = self.show(session, _id)
156
157 content_range_text = headers.get("Content-Range")
158 expected_md5 = headers.get("Content-File-MD5")
159 compressed = None
160 content_type = headers.get("Content-Type")
161 if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
162 "application/zip" in content_type:
163 compressed = "gzip"
164 filename = headers.get("Content-Filename")
165 if not filename:
166 filename = "package.tar.gz" if compressed else "package"
167 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
168 file_pkg = None
169 error_text = ""
170 try:
171 if content_range_text:
172 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
173 if content_range[0] != "bytes": # TODO check x<y not negative < total....
174 raise IndexError()
175 start = int(content_range[1])
176 end = int(content_range[2]) + 1
177 total = int(content_range[3])
178 else:
179 start = 0
180 temp_folder = _id + "_" # all the content is upload here and if ok, it is rename from id_ to is folder
181
182 if start:
183 if not self.fs.file_exists(temp_folder, 'dir'):
184 raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
185 else:
186 self.fs.file_delete(temp_folder, ignore_non_exist=True)
187 self.fs.mkdir(temp_folder)
188
189 storage = self.fs.get_params()
190 storage["folder"] = _id
191
192 file_path = (temp_folder, filename)
193 if self.fs.file_exists(file_path, 'file'):
194 file_size = self.fs.file_size(file_path)
195 else:
196 file_size = 0
197 if file_size != start:
198 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
199 file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
200 file_pkg = self.fs.file_open(file_path, 'a+b')
201 if isinstance(indata, dict):
202 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
203 file_pkg.write(indata_text.encode(encoding="utf-8"))
204 else:
205 indata_len = 0
206 while True:
207 indata_text = indata.read(4096)
208 indata_len += len(indata_text)
209 if not indata_text:
210 break
211 file_pkg.write(indata_text)
212 if content_range_text:
213 if indata_len != end-start:
214 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
215 start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
216 if end != total:
217 # TODO update to UPLOADING
218 return False
219
220 # PACKAGE UPLOADED
221 if expected_md5:
222 file_pkg.seek(0, 0)
223 file_md5 = md5()
224 chunk_data = file_pkg.read(1024)
225 while chunk_data:
226 file_md5.update(chunk_data)
227 chunk_data = file_pkg.read(1024)
228 if expected_md5 != file_md5.hexdigest():
229 raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
230 file_pkg.seek(0, 0)
231 if compressed == "gzip":
232 tar = tarfile.open(mode='r', fileobj=file_pkg)
233 descriptor_file_name = None
234 for tarinfo in tar:
235 tarname = tarinfo.name
236 tarname_path = tarname.split("/")
237 if not tarname_path[0] or ".." in tarname_path: # if start with "/" means absolute path
238 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
239 if len(tarname_path) == 1 and not tarinfo.isdir():
240 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
241 if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
242 storage["pkg-dir"] = tarname_path[0]
243 if len(tarname_path) == 2:
244 if descriptor_file_name:
245 raise EngineException(
246 "Found more than one descriptor file at package descriptor tar.gz")
247 descriptor_file_name = tarname
248 if not descriptor_file_name:
249 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
250 storage["descriptor"] = descriptor_file_name
251 storage["zipfile"] = filename
252 self.fs.file_extract(tar, temp_folder)
253 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
254 content = descriptor_file.read()
255 else:
256 content = file_pkg.read()
257 storage["descriptor"] = descriptor_file_name = filename
258
259 if descriptor_file_name.endswith(".json"):
260 error_text = "Invalid json format "
261 indata = json.load(content)
262 else:
263 error_text = "Invalid yaml format "
264 indata = yaml.load(content)
265
266 current_desc["_admin"]["storage"] = storage
267 current_desc["_admin"]["onboardingState"] = "ONBOARDED"
268 current_desc["_admin"]["operationalState"] = "ENABLED"
269
270 indata = self._remove_envelop(indata)
271
272 # Override descriptor with query string kwargs
273 if kwargs:
274 self._update_input_with_kwargs(indata, kwargs)
275 # it will call overrides method at VnfdTopic or NsdTopic
276 # indata = self._validate_input_edit(indata, force=session["force"])
277
278 deep_update_rfc7396(current_desc, indata)
279 self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
280 self.db.replace(self.topic, _id, current_desc)
281 self.fs.dir_rename(temp_folder, _id)
282
283 indata["_id"] = _id
284 self._send_msg("created", indata)
285
286 # TODO if descriptor has changed because kwargs update content and remove cached zip
287 # TODO if zip is not present creates one
288 return True
289
290 except EngineException:
291 raise
292 except IndexError:
293 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
294 HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
295 except IOError as e:
296 raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
297 except tarfile.ReadError as e:
298 raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
299 except (ValueError, yaml.YAMLError) as e:
300 raise EngineException(error_text + str(e))
301 except ValidationError as e:
302 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
303 finally:
304 if file_pkg:
305 file_pkg.close()
306
307 def get_file(self, session, _id, path=None, accept_header=None):
308 """
309 Return the file content of a vnfd or nsd
310 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
311 :param _id: Identity of the vnfd, nsd
312 :param path: artifact path or "$DESCRIPTOR" or None
313 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
314 :return: opened file plus Accept format or raises an exception
315 """
316 accept_text = accept_zip = False
317 if accept_header:
318 if 'text/plain' in accept_header or '*/*' in accept_header:
319 accept_text = True
320 if 'application/zip' in accept_header or '*/*' in accept_header:
321 accept_zip = 'application/zip'
322 elif 'application/gzip' in accept_header:
323 accept_zip = 'application/gzip'
324
325 if not accept_text and not accept_zip:
326 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
327 http_code=HTTPStatus.NOT_ACCEPTABLE)
328
329 content = self.show(session, _id)
330 if content["_admin"]["onboardingState"] != "ONBOARDED":
331 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
332 "onboardingState is {}".format(content["_admin"]["onboardingState"]),
333 http_code=HTTPStatus.CONFLICT)
334 storage = content["_admin"]["storage"]
335 if path is not None and path != "$DESCRIPTOR": # artifacts
336 if not storage.get('pkg-dir'):
337 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
338 if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
339 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
340 return folder_content, "text/plain"
341 # TODO manage folders in http
342 else:
343 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
344 "application/octet-stream"
345
346 # pkgtype accept ZIP TEXT -> result
347 # manyfiles yes X -> zip
348 # no yes -> error
349 # onefile yes no -> zip
350 # X yes -> text
351
352 if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
353 return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
354 elif storage.get('pkg-dir') and not accept_zip:
355 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
356 "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
357 else:
358 if not storage.get('zipfile'):
359 # TODO generate zipfile if not present
360 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
361 "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
362 return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
363
364 def pyangbind_validation(self, item, data, force=False):
365 try:
366 if item == "vnfds":
367 myvnfd = vnfd_im()
368 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
369 path_helper=True, skip_unknown=force)
370 out = pybindJSON.dumps(myvnfd, mode="ietf")
371 elif item == "nsds":
372 mynsd = nsd_im()
373 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
374 path_helper=True, skip_unknown=force)
375 out = pybindJSON.dumps(mynsd, mode="ietf")
376 elif item == "nsts":
377 mynst = nst_im()
378 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
379 path_helper=True, skip_unknown=force)
380 out = pybindJSON.dumps(mynst, mode="ietf")
381 else:
382 raise EngineException("Not possible to validate '{}' item".format(item),
383 http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
384
385 desc_out = self._remove_envelop(yaml.safe_load(out))
386 return desc_out
387
388 except Exception as e:
389 raise EngineException("Error in pyangbind validation: {}".format(str(e)),
390 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
391
392
393 class VnfdTopic(DescriptorTopic):
394 topic = "vnfds"
395 topic_msg = "vnfd"
396
397 def __init__(self, db, fs, msg):
398 DescriptorTopic.__init__(self, db, fs, msg)
399
400 @staticmethod
401 def _remove_envelop(indata=None):
402 if not indata:
403 return {}
404 clean_indata = indata
405 if clean_indata.get('vnfd:vnfd-catalog'):
406 clean_indata = clean_indata['vnfd:vnfd-catalog']
407 elif clean_indata.get('vnfd-catalog'):
408 clean_indata = clean_indata['vnfd-catalog']
409 if clean_indata.get('vnfd'):
410 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
411 raise EngineException("'vnfd' must be a list of only one element")
412 clean_indata = clean_indata['vnfd'][0]
413 elif clean_indata.get('vnfd:vnfd'):
414 if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
415 raise EngineException("'vnfd:vnfd' must be a list of only one element")
416 clean_indata = clean_indata['vnfd:vnfd'][0]
417 return clean_indata
418
419 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
420 super().check_conflict_on_edit(session, final_content, edit_content, _id)
421
422 # set type of vnfd
423 contains_pdu = False
424 contains_vdu = False
425 for vdu in get_iterable(final_content.get("vdu")):
426 if vdu.get("pdu-type"):
427 contains_pdu = True
428 else:
429 contains_vdu = True
430 if contains_pdu:
431 final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
432 elif contains_vdu:
433 final_content["_admin"]["type"] = "vnfd"
434 # if neither vud nor pdu do not fill type
435
436 def check_conflict_on_del(self, session, _id, db_content):
437 """
438 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
439 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
440 that uses this vnfd
441 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
442 :param _id: vnfd internal id
443 :param db_content: The database content of the _id.
444 :return: None or raises EngineException with the conflict
445 """
446 if session["force"]:
447 return
448 descriptor = db_content
449 descriptor_id = descriptor.get("id")
450 if not descriptor_id: # empty vnfd not uploaded
451 return
452
453 _filter = self._get_project_filter(session)
454
455 # check vnfrs using this vnfd
456 _filter["vnfd-id"] = _id
457 if self.db.get_list("vnfrs", _filter):
458 raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
459
460 # check NSD referencing this VNFD
461 del _filter["vnfd-id"]
462 _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
463 if self.db.get_list("nsds", _filter):
464 raise EngineException("There is at least one NSD referencing this descriptor",
465 http_code=HTTPStatus.CONFLICT)
466
467 def _validate_input_new(self, indata, storage_params, force=False):
468 indata = self.pyangbind_validation("vnfds", indata, force)
469 # Cross references validation in the descriptor
470 if indata.get("vdu"):
471 if not indata.get("mgmt-interface"):
472 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
473 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
474 if indata["mgmt-interface"].get("cp"):
475 for cp in get_iterable(indata.get("connection-point")):
476 if cp["name"] == indata["mgmt-interface"]["cp"]:
477 break
478 else:
479 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
480 .format(indata["mgmt-interface"]["cp"]),
481 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
482
483 for vdu in get_iterable(indata.get("vdu")):
484 for interface in get_iterable(vdu.get("interface")):
485 if interface.get("external-connection-point-ref"):
486 for cp in get_iterable(indata.get("connection-point")):
487 if cp["name"] == interface["external-connection-point-ref"]:
488 break
489 else:
490 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
491 "must match an existing connection-point"
492 .format(vdu["id"], interface["name"],
493 interface["external-connection-point-ref"]),
494 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
495
496 elif interface.get("internal-connection-point-ref"):
497 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
498 if interface["internal-connection-point-ref"] == internal_cp.get("id"):
499 break
500 else:
501 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
502 "must match an existing vdu:internal-connection-point"
503 .format(vdu["id"], interface["name"],
504 interface["internal-connection-point-ref"]),
505 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
506 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
507 if vdu.get("vdu-configuration"):
508 if vdu["vdu-configuration"].get("juju"):
509 if not self._validate_package_folders(storage_params, 'charms'):
510 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
511 "package".format(indata["id"], vdu["id"]))
512 # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
513 if vdu.get("cloud-init-file"):
514 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
515 raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
516 "package".format(indata["id"], vdu["id"]))
517 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
518 if indata.get("vnf-configuration"):
519 if indata["vnf-configuration"].get("juju"):
520 if not self._validate_package_folders(storage_params, 'charms'):
521 raise EngineException("Charm defined in vnf[id={}] but not present in "
522 "package".format(indata["id"]))
523 vld_names = [] # For detection of duplicated VLD names
524 for ivld in get_iterable(indata.get("internal-vld")):
525 # BEGIN Detection of duplicated VLD names
526 ivld_name = ivld["name"]
527 if ivld_name in vld_names:
528 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
529 .format(ivld["name"], indata["id"], ivld["id"]),
530 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
531 else:
532 vld_names.append(ivld_name)
533 # END Detection of duplicated VLD names
534 for icp in get_iterable(ivld.get("internal-connection-point")):
535 icp_mark = False
536 for vdu in get_iterable(indata.get("vdu")):
537 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
538 if icp["id-ref"] == internal_cp["id"]:
539 icp_mark = True
540 break
541 if icp_mark:
542 break
543 else:
544 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
545 "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
546 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
547 if ivld.get("ip-profile-ref"):
548 for ip_prof in get_iterable(indata.get("ip-profiles")):
549 if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
550 break
551 else:
552 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
553 ivld["id"], ivld["ip-profile-ref"]),
554 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
555 for mp in get_iterable(indata.get("monitoring-param")):
556 if mp.get("vdu-monitoring-param"):
557 mp_vmp_mark = False
558 for vdu in get_iterable(indata.get("vdu")):
559 for vmp in get_iterable(vdu.get("monitoring-param")):
560 if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
561 mp["vdu-monitoring-param"]["vdu-ref"]:
562 mp_vmp_mark = True
563 break
564 if mp_vmp_mark:
565 break
566 else:
567 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
568 "defined at vdu[id='{}'] or vdu does not exist"
569 .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
570 mp["vdu-monitoring-param"]["vdu-ref"]),
571 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
572 elif mp.get("vdu-metric"):
573 mp_vm_mark = False
574 for vdu in get_iterable(indata.get("vdu")):
575 if vdu.get("vdu-configuration"):
576 for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
577 if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
578 mp["vdu-metric"]["vdu-ref"]:
579 mp_vm_mark = True
580 break
581 if mp_vm_mark:
582 break
583 else:
584 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
585 "vdu[id='{}'] or vdu does not exist"
586 .format(mp["vdu-metric"]["vdu-metric-name-ref"],
587 mp["vdu-metric"]["vdu-ref"]),
588 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
589
590 for sgd in get_iterable(indata.get("scaling-group-descriptor")):
591 for sp in get_iterable(sgd.get("scaling-policy")):
592 for sc in get_iterable(sp.get("scaling-criteria")):
593 for mp in get_iterable(indata.get("monitoring-param")):
594 if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
595 break
596 else:
597 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
598 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
599 .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
600 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
601 for sgd_vdu in get_iterable(sgd.get("vdu")):
602 sgd_vdu_mark = False
603 for vdu in get_iterable(indata.get("vdu")):
604 if vdu["id"] == sgd_vdu["vdu-id-ref"]:
605 sgd_vdu_mark = True
606 break
607 if sgd_vdu_mark:
608 break
609 else:
610 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
611 .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
612 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
613 for sca in get_iterable(sgd.get("scaling-config-action")):
614 if not indata.get("vnf-configuration"):
615 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
616 "scaling-group-descriptor[name='{}']:scaling-config-action"
617 .format(sgd["name"]),
618 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
619 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
620 if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
621 break
622 else:
623 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
624 "primitive-name-ref='{}' does not match any "
625 "vnf-configuration:config-primitive:name"
626 .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
627 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
628 return indata
629
630 def _validate_input_edit(self, indata, force=False):
631 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
632 return indata
633
634 def _validate_package_folders(self, storage_params, folder, file=None):
635 if not storage_params or not storage_params.get("pkg-dir"):
636 return False
637 else:
638 if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
639 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
640 else:
641 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
642 if file:
643 return self.fs.file_exists("{}/{}".format(f, file), 'file')
644 else:
645 if self.fs.file_exists(f, 'dir'):
646 if self.fs.dir_ls(f):
647 return True
648 return False
649
650
651 class NsdTopic(DescriptorTopic):
652 topic = "nsds"
653 topic_msg = "nsd"
654
655 def __init__(self, db, fs, msg):
656 DescriptorTopic.__init__(self, db, fs, msg)
657
658 @staticmethod
659 def _remove_envelop(indata=None):
660 if not indata:
661 return {}
662 clean_indata = indata
663
664 if clean_indata.get('nsd:nsd-catalog'):
665 clean_indata = clean_indata['nsd:nsd-catalog']
666 elif clean_indata.get('nsd-catalog'):
667 clean_indata = clean_indata['nsd-catalog']
668 if clean_indata.get('nsd'):
669 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
670 raise EngineException("'nsd' must be a list of only one element")
671 clean_indata = clean_indata['nsd'][0]
672 elif clean_indata.get('nsd:nsd'):
673 if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
674 raise EngineException("'nsd:nsd' must be a list of only one element")
675 clean_indata = clean_indata['nsd:nsd'][0]
676 return clean_indata
677
678 def _validate_input_new(self, indata, storage_params, force=False):
679 indata = self.pyangbind_validation("nsds", indata, force)
680 # Cross references validation in the descriptor
681 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
682 for vld in get_iterable(indata.get("vld")):
683 if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
684 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
685 " You cannot set an ip-profile when mgmt-network is True"
686 .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
687 for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
688 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
689 if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
690 if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
691 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
692 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
693 " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
694 constituent_vnfd["member-vnf-index"],
695 constituent_vnfd["vnfd-id-ref"]),
696 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
697 break
698 else:
699 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
700 "does not match any constituent-vnfd:member-vnf-index"
701 .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
702 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
703 return indata
704
705 def _validate_input_edit(self, indata, force=False):
706 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
707 return indata
708
709 def _check_descriptor_dependencies(self, session, descriptor):
710 """
711 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
712 connection points are ok
713 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
714 :param descriptor: descriptor to be inserted or edit
715 :return: None or raises exception
716 """
717 if session["force"]:
718 return
719 member_vnfd_index = {}
720 if descriptor.get("constituent-vnfd") and not session["force"]:
721 for vnf in descriptor["constituent-vnfd"]:
722 vnfd_id = vnf["vnfd-id-ref"]
723 filter_q = self._get_project_filter(session)
724 filter_q["id"] = vnfd_id
725 vnf_list = self.db.get_list("vnfds", filter_q)
726 if not vnf_list:
727 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
728 "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
729 # elif len(vnf_list) > 1:
730 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
731 # http_code=HTTPStatus.CONFLICT)
732 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
733
734 # Cross references validation in the descriptor and vnfd connection point validation
735 for vld in get_iterable(descriptor.get("vld")):
736 for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
737 # look if this vnfd contains this connection point
738 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
739 if not vnfd:
740 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
741 "does not match any constituent-vnfd:member-vnf-index"
742 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
743 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
744 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
745 if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
746 break
747 else:
748 raise EngineException(
749 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
750 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
751 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
752 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
753 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
754
755 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
756 super().check_conflict_on_edit(session, final_content, edit_content, _id)
757
758 self._check_descriptor_dependencies(session, final_content)
759
760 def check_conflict_on_del(self, session, _id, db_content):
761 """
762 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
763 that NSD can be public and be used by other projects.
764 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
765 :param _id: nsd internal id
766 :param db_content: The database content of the _id
767 :return: None or raises EngineException with the conflict
768 """
769 if session["force"]:
770 return
771 descriptor = db_content
772 descriptor_id = descriptor.get("id")
773 if not descriptor_id: # empty nsd not uploaded
774 return
775
776 # check NSD used by NS
777 _filter = self._get_project_filter(session)
778 _filter["nsd-id"] = _id
779 if self.db.get_list("nsrs", _filter):
780 raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
781
782 # check NSD referenced by NST
783 del _filter["nsd-id"]
784 _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
785 if self.db.get_list("nsts", _filter):
786 raise EngineException("There is at least one NetSlice Template referencing this descriptor",
787 http_code=HTTPStatus.CONFLICT)
788
789
790 class NstTopic(DescriptorTopic):
791 topic = "nsts"
792 topic_msg = "nst"
793
794 def __init__(self, db, fs, msg):
795 DescriptorTopic.__init__(self, db, fs, msg)
796
797 @staticmethod
798 def _remove_envelop(indata=None):
799 if not indata:
800 return {}
801 clean_indata = indata
802
803 if clean_indata.get('nst'):
804 if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
805 raise EngineException("'nst' must be a list only one element")
806 clean_indata = clean_indata['nst'][0]
807 elif clean_indata.get('nst:nst'):
808 if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
809 raise EngineException("'nst:nst' must be a list only one element")
810 clean_indata = clean_indata['nst:nst'][0]
811 return clean_indata
812
813 def _validate_input_edit(self, indata, force=False):
814 # TODO validate with pyangbind, serialize
815 return indata
816
817 def _validate_input_new(self, indata, storage_params, force=False):
818 indata = self.pyangbind_validation("nsts", indata, force)
819 return indata.copy()
820
821 def _check_descriptor_dependencies(self, session, descriptor):
822 """
823 Check that the dependent descriptors exist on a new descriptor or edition
824 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
825 :param descriptor: descriptor to be inserted or edit
826 :return: None or raises exception
827 """
828 if not descriptor.get("netslice-subnet"):
829 return
830 for nsd in descriptor["netslice-subnet"]:
831 nsd_id = nsd["nsd-ref"]
832 filter_q = self._get_project_filter(session)
833 filter_q["id"] = nsd_id
834 if not self.db.get_list("nsds", filter_q):
835 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
836 "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
837
838 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
839 super().check_conflict_on_edit(session, final_content, edit_content, _id)
840
841 self._check_descriptor_dependencies(session, final_content)
842
843 def check_conflict_on_del(self, session, _id, db_content):
844 """
845 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
846 that NST can be public and be used by other projects.
847 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
848 :param _id: nst internal id
849 :param db_content: The database content of the _id.
850 :return: None or raises EngineException with the conflict
851 """
852 # TODO: Check this method
853 if session["force"]:
854 return
855 # Get Network Slice Template from Database
856 _filter = self._get_project_filter(session)
857 _filter["nst-id"] = _id
858 if self.db.get_list("nsis", _filter):
859 raise EngineException("there is at least one Netslice Instance using this descriptor",
860 http_code=HTTPStatus.CONFLICT)
861
862
863 class PduTopic(BaseTopic):
864 topic = "pdus"
865 topic_msg = "pdu"
866 schema_new = pdu_new_schema
867 schema_edit = pdu_edit_schema
868
869 def __init__(self, db, fs, msg):
870 BaseTopic.__init__(self, db, fs, msg)
871
872 @staticmethod
873 def format_on_new(content, project_id=None, make_public=False):
874 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
875 content["_admin"]["onboardingState"] = "CREATED"
876 content["_admin"]["operationalState"] = "ENABLED"
877 content["_admin"]["usageState"] = "NOT_IN_USE"
878
879 def check_conflict_on_del(self, session, _id, db_content):
880 """
881 Check that there is not any vnfr that uses this PDU
882 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
883 :param _id: pdu internal id
884 :param db_content: The database content of the _id.
885 :return: None or raises EngineException with the conflict
886 """
887 if session["force"]:
888 return
889
890 _filter = self._get_project_filter(session)
891 _filter["vdur.pdu-id"] = _id
892 if self.db.get_list("vnfrs", _filter):
893 raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)