feature8029 do not validate vim_account type
[osm/NBI.git] / osm_nbi / descriptor_topics.py
1 # -*- coding: utf-8 -*-
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12 # implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import tarfile
17 import yaml
18 import json
19 # import logging
20 from hashlib import md5
21 from osm_common.dbbase import DbException, deep_update_rfc7396
22 from http import HTTPStatus
23 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
24 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
25 from osm_im.vnfd import vnfd as vnfd_im
26 from osm_im.nsd import nsd as nsd_im
27 from osm_im.nst import nst as nst_im
28 from pyangbind.lib.serialise import pybindJSONDecoder
29 import pyangbind.lib.pybindJSON as pybindJSON
30
31 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
32
33
34 class DescriptorTopic(BaseTopic):
35
36 def __init__(self, db, fs, msg, auth):
37 BaseTopic.__init__(self, db, fs, msg, auth)
38
39 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
40 super().check_conflict_on_edit(session, final_content, edit_content, _id)
41 # 1. validate again with pyangbind
42 # 1.1. remove internal keys
43 internal_keys = {}
44 for k in ("_id", "_admin"):
45 if k in final_content:
46 internal_keys[k] = final_content.pop(k)
47 storage_params = internal_keys["_admin"].get("storage")
48 serialized = self._validate_input_new(final_content, storage_params, session["force"])
49 # 1.2. modify final_content with a serialized version
50 final_content.clear()
51 final_content.update(serialized)
52 # 1.3. restore internal keys
53 for k, v in internal_keys.items():
54 final_content[k] = v
55
56 if session["force"]:
57 return
58 # 2. check that this id is not present
59 if "id" in edit_content:
60 _filter = self._get_project_filter(session)
61 _filter["id"] = final_content["id"]
62 _filter["_id.neq"] = _id
63 if self.db.get_one(self.topic, _filter, fail_on_empty=False):
64 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
65 final_content["id"]),
66 HTTPStatus.CONFLICT)
67
68 @staticmethod
69 def format_on_new(content, project_id=None, make_public=False):
70 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
71 content["_admin"]["onboardingState"] = "CREATED"
72 content["_admin"]["operationalState"] = "DISABLED"
73 content["_admin"]["usageState"] = "NOT_IN_USE"
74
75 def delete_extra(self, session, _id, db_content):
76 """
77 Deletes file system storage associated with the descriptor
78 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
79 :param _id: server internal id
80 :param db_content: The database content of the descriptor
81 :return: None if ok or raises EngineException with the problem
82 """
83 self.fs.file_delete(_id, ignore_non_exist=True)
84 self.fs.file_delete(_id + "_", ignore_non_exist=True) # remove temp folder
85
86 @staticmethod
87 def get_one_by_id(db, session, topic, id):
88 # find owned by this project
89 _filter = BaseTopic._get_project_filter(session)
90 _filter["id"] = id
91 desc_list = db.get_list(topic, _filter)
92 if len(desc_list) == 1:
93 return desc_list[0]
94 elif len(desc_list) > 1:
95 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
96 HTTPStatus.CONFLICT)
97
98 # not found any: try to find public
99 _filter = BaseTopic._get_project_filter(session)
100 _filter["id"] = id
101 desc_list = db.get_list(topic, _filter)
102 if not desc_list:
103 raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
104 elif len(desc_list) == 1:
105 return desc_list[0]
106 else:
107 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
108 topic[:-1], id), HTTPStatus.CONFLICT)
109
110 def new(self, rollback, session, indata=None, kwargs=None, headers=None):
111 """
112 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
113 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
114 (self.upload_content)
115 :param rollback: list to append created items at database in case a rollback may to be done
116 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
117 :param indata: data to be inserted
118 :param kwargs: used to override the indata descriptor
119 :param headers: http request headers
120 :return: _id, None: identity of the inserted data; and None as there is not any operation
121 """
122
123 try:
124 # Check Quota
125 self.check_quota(session)
126
127 # _remove_envelop
128 if indata:
129 if "userDefinedData" in indata:
130 indata = indata['userDefinedData']
131
132 # Override descriptor with query string kwargs
133 self._update_input_with_kwargs(indata, kwargs)
134 # uncomment when this method is implemented.
135 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
136 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
137
138 content = {"_admin": {"userDefinedData": indata}}
139 self.format_on_new(content, session["project_id"], make_public=session["public"])
140 _id = self.db.create(self.topic, content)
141 rollback.append({"topic": self.topic, "_id": _id})
142 self._send_msg("created", {"_id": _id})
143 return _id, None
144 except ValidationError as e:
145 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
146
147 def upload_content(self, session, _id, indata, kwargs, headers):
148 """
149 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
150 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
151 :param _id : the nsd,vnfd is already created, this is the id
152 :param indata: http body request
153 :param kwargs: user query string to override parameters. NOT USED
154 :param headers: http request headers
155 :return: True if package is completely uploaded or False if partial content has been uploded
156 Raise exception on error
157 """
158 # Check that _id exists and it is valid
159 current_desc = self.show(session, _id)
160
161 content_range_text = headers.get("Content-Range")
162 expected_md5 = headers.get("Content-File-MD5")
163 compressed = None
164 content_type = headers.get("Content-Type")
165 if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
166 "application/zip" in content_type:
167 compressed = "gzip"
168 filename = headers.get("Content-Filename")
169 if not filename:
170 filename = "package.tar.gz" if compressed else "package"
171 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
172 file_pkg = None
173 error_text = ""
174 try:
175 if content_range_text:
176 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
177 if content_range[0] != "bytes": # TODO check x<y not negative < total....
178 raise IndexError()
179 start = int(content_range[1])
180 end = int(content_range[2]) + 1
181 total = int(content_range[3])
182 else:
183 start = 0
184 temp_folder = _id + "_" # all the content is upload here and if ok, it is rename from id_ to is folder
185
186 if start:
187 if not self.fs.file_exists(temp_folder, 'dir'):
188 raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
189 else:
190 self.fs.file_delete(temp_folder, ignore_non_exist=True)
191 self.fs.mkdir(temp_folder)
192
193 storage = self.fs.get_params()
194 storage["folder"] = _id
195
196 file_path = (temp_folder, filename)
197 if self.fs.file_exists(file_path, 'file'):
198 file_size = self.fs.file_size(file_path)
199 else:
200 file_size = 0
201 if file_size != start:
202 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
203 file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
204 file_pkg = self.fs.file_open(file_path, 'a+b')
205 if isinstance(indata, dict):
206 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
207 file_pkg.write(indata_text.encode(encoding="utf-8"))
208 else:
209 indata_len = 0
210 while True:
211 indata_text = indata.read(4096)
212 indata_len += len(indata_text)
213 if not indata_text:
214 break
215 file_pkg.write(indata_text)
216 if content_range_text:
217 if indata_len != end-start:
218 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
219 start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
220 if end != total:
221 # TODO update to UPLOADING
222 return False
223
224 # PACKAGE UPLOADED
225 if expected_md5:
226 file_pkg.seek(0, 0)
227 file_md5 = md5()
228 chunk_data = file_pkg.read(1024)
229 while chunk_data:
230 file_md5.update(chunk_data)
231 chunk_data = file_pkg.read(1024)
232 if expected_md5 != file_md5.hexdigest():
233 raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
234 file_pkg.seek(0, 0)
235 if compressed == "gzip":
236 tar = tarfile.open(mode='r', fileobj=file_pkg)
237 descriptor_file_name = None
238 for tarinfo in tar:
239 tarname = tarinfo.name
240 tarname_path = tarname.split("/")
241 if not tarname_path[0] or ".." in tarname_path: # if start with "/" means absolute path
242 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
243 if len(tarname_path) == 1 and not tarinfo.isdir():
244 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
245 if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
246 storage["pkg-dir"] = tarname_path[0]
247 if len(tarname_path) == 2:
248 if descriptor_file_name:
249 raise EngineException(
250 "Found more than one descriptor file at package descriptor tar.gz")
251 descriptor_file_name = tarname
252 if not descriptor_file_name:
253 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
254 storage["descriptor"] = descriptor_file_name
255 storage["zipfile"] = filename
256 self.fs.file_extract(tar, temp_folder)
257 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
258 content = descriptor_file.read()
259 else:
260 content = file_pkg.read()
261 storage["descriptor"] = descriptor_file_name = filename
262
263 if descriptor_file_name.endswith(".json"):
264 error_text = "Invalid json format "
265 indata = json.load(content)
266 else:
267 error_text = "Invalid yaml format "
268 indata = yaml.load(content, Loader=yaml.SafeLoader)
269
270 current_desc["_admin"]["storage"] = storage
271 current_desc["_admin"]["onboardingState"] = "ONBOARDED"
272 current_desc["_admin"]["operationalState"] = "ENABLED"
273
274 indata = self._remove_envelop(indata)
275
276 # Override descriptor with query string kwargs
277 if kwargs:
278 self._update_input_with_kwargs(indata, kwargs)
279 # it will call overrides method at VnfdTopic or NsdTopic
280 # indata = self._validate_input_edit(indata, force=session["force"])
281
282 deep_update_rfc7396(current_desc, indata)
283 self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
284 self.db.replace(self.topic, _id, current_desc)
285 self.fs.dir_rename(temp_folder, _id)
286
287 indata["_id"] = _id
288 self._send_msg("edited", indata)
289
290 # TODO if descriptor has changed because kwargs update content and remove cached zip
291 # TODO if zip is not present creates one
292 return True
293
294 except EngineException:
295 raise
296 except IndexError:
297 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
298 HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
299 except IOError as e:
300 raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
301 except tarfile.ReadError as e:
302 raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
303 except (ValueError, yaml.YAMLError) as e:
304 raise EngineException(error_text + str(e))
305 except ValidationError as e:
306 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
307 finally:
308 if file_pkg:
309 file_pkg.close()
310
311 def get_file(self, session, _id, path=None, accept_header=None):
312 """
313 Return the file content of a vnfd or nsd
314 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
315 :param _id: Identity of the vnfd, nsd
316 :param path: artifact path or "$DESCRIPTOR" or None
317 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
318 :return: opened file plus Accept format or raises an exception
319 """
320 accept_text = accept_zip = False
321 if accept_header:
322 if 'text/plain' in accept_header or '*/*' in accept_header:
323 accept_text = True
324 if 'application/zip' in accept_header or '*/*' in accept_header:
325 accept_zip = 'application/zip'
326 elif 'application/gzip' in accept_header:
327 accept_zip = 'application/gzip'
328
329 if not accept_text and not accept_zip:
330 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
331 http_code=HTTPStatus.NOT_ACCEPTABLE)
332
333 content = self.show(session, _id)
334 if content["_admin"]["onboardingState"] != "ONBOARDED":
335 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
336 "onboardingState is {}".format(content["_admin"]["onboardingState"]),
337 http_code=HTTPStatus.CONFLICT)
338 storage = content["_admin"]["storage"]
339 if path is not None and path != "$DESCRIPTOR": # artifacts
340 if not storage.get('pkg-dir'):
341 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
342 if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
343 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
344 return folder_content, "text/plain"
345 # TODO manage folders in http
346 else:
347 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
348 "application/octet-stream"
349
350 # pkgtype accept ZIP TEXT -> result
351 # manyfiles yes X -> zip
352 # no yes -> error
353 # onefile yes no -> zip
354 # X yes -> text
355
356 if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
357 return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
358 elif storage.get('pkg-dir') and not accept_zip:
359 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
360 "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
361 else:
362 if not storage.get('zipfile'):
363 # TODO generate zipfile if not present
364 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
365 "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
366 return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
367
368 def pyangbind_validation(self, item, data, force=False):
369 try:
370 if item == "vnfds":
371 myvnfd = vnfd_im()
372 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
373 path_helper=True, skip_unknown=force)
374 out = pybindJSON.dumps(myvnfd, mode="ietf")
375 elif item == "nsds":
376 mynsd = nsd_im()
377 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
378 path_helper=True, skip_unknown=force)
379 out = pybindJSON.dumps(mynsd, mode="ietf")
380 elif item == "nsts":
381 mynst = nst_im()
382 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
383 path_helper=True, skip_unknown=force)
384 out = pybindJSON.dumps(mynst, mode="ietf")
385 else:
386 raise EngineException("Not possible to validate '{}' item".format(item),
387 http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
388
389 desc_out = self._remove_envelop(yaml.safe_load(out))
390 return desc_out
391
392 except Exception as e:
393 raise EngineException("Error in pyangbind validation: {}".format(str(e)),
394 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
395
396
397 class VnfdTopic(DescriptorTopic):
398 topic = "vnfds"
399 topic_msg = "vnfd"
400
401 def __init__(self, db, fs, msg, auth):
402 DescriptorTopic.__init__(self, db, fs, msg, auth)
403
404 @staticmethod
405 def _remove_envelop(indata=None):
406 if not indata:
407 return {}
408 clean_indata = indata
409 if clean_indata.get('vnfd:vnfd-catalog'):
410 clean_indata = clean_indata['vnfd:vnfd-catalog']
411 elif clean_indata.get('vnfd-catalog'):
412 clean_indata = clean_indata['vnfd-catalog']
413 if clean_indata.get('vnfd'):
414 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
415 raise EngineException("'vnfd' must be a list of only one element")
416 clean_indata = clean_indata['vnfd'][0]
417 elif clean_indata.get('vnfd:vnfd'):
418 if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
419 raise EngineException("'vnfd:vnfd' must be a list of only one element")
420 clean_indata = clean_indata['vnfd:vnfd'][0]
421 return clean_indata
422
423 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
424 super().check_conflict_on_edit(session, final_content, edit_content, _id)
425
426 # set type of vnfd
427 contains_pdu = False
428 contains_vdu = False
429 for vdu in get_iterable(final_content.get("vdu")):
430 if vdu.get("pdu-type"):
431 contains_pdu = True
432 else:
433 contains_vdu = True
434 if contains_pdu:
435 final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
436 elif contains_vdu:
437 final_content["_admin"]["type"] = "vnfd"
438 # if neither vud nor pdu do not fill type
439
440 def check_conflict_on_del(self, session, _id, db_content):
441 """
442 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
443 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
444 that uses this vnfd
445 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
446 :param _id: vnfd internal id
447 :param db_content: The database content of the _id.
448 :return: None or raises EngineException with the conflict
449 """
450 if session["force"]:
451 return
452 descriptor = db_content
453 descriptor_id = descriptor.get("id")
454 if not descriptor_id: # empty vnfd not uploaded
455 return
456
457 _filter = self._get_project_filter(session)
458
459 # check vnfrs using this vnfd
460 _filter["vnfd-id"] = _id
461 if self.db.get_list("vnfrs", _filter):
462 raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
463
464 # check NSD referencing this VNFD
465 del _filter["vnfd-id"]
466 _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
467 if self.db.get_list("nsds", _filter):
468 raise EngineException("There is at least one NSD referencing this descriptor",
469 http_code=HTTPStatus.CONFLICT)
470
471 def _validate_input_new(self, indata, storage_params, force=False):
472 indata = self.pyangbind_validation("vnfds", indata, force)
473 # Cross references validation in the descriptor
474 if indata.get("vdu"):
475 if not indata.get("mgmt-interface"):
476 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
477 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
478 if indata["mgmt-interface"].get("cp"):
479 for cp in get_iterable(indata.get("connection-point")):
480 if cp["name"] == indata["mgmt-interface"]["cp"]:
481 break
482 else:
483 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
484 .format(indata["mgmt-interface"]["cp"]),
485 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
486
487 for vdu in get_iterable(indata.get("vdu")):
488 for interface in get_iterable(vdu.get("interface")):
489 if interface.get("external-connection-point-ref"):
490 for cp in get_iterable(indata.get("connection-point")):
491 if cp["name"] == interface["external-connection-point-ref"]:
492 break
493 else:
494 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
495 "must match an existing connection-point"
496 .format(vdu["id"], interface["name"],
497 interface["external-connection-point-ref"]),
498 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
499
500 elif interface.get("internal-connection-point-ref"):
501 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
502 if interface["internal-connection-point-ref"] == internal_cp.get("id"):
503 break
504 else:
505 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
506 "must match an existing vdu:internal-connection-point"
507 .format(vdu["id"], interface["name"],
508 interface["internal-connection-point-ref"]),
509 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
510 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
511 if vdu.get("vdu-configuration"):
512 if vdu["vdu-configuration"].get("juju"):
513 if not self._validate_package_folders(storage_params, 'charms'):
514 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
515 "package".format(indata["id"], vdu["id"]))
516 # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
517 if vdu.get("cloud-init-file"):
518 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
519 raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
520 "package".format(indata["id"], vdu["id"]))
521 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
522 if indata.get("vnf-configuration"):
523 if indata["vnf-configuration"].get("juju"):
524 if not self._validate_package_folders(storage_params, 'charms'):
525 raise EngineException("Charm defined in vnf[id={}] but not present in "
526 "package".format(indata["id"]))
527 vld_names = [] # For detection of duplicated VLD names
528 for ivld in get_iterable(indata.get("internal-vld")):
529 # BEGIN Detection of duplicated VLD names
530 ivld_name = ivld["name"]
531 if ivld_name in vld_names:
532 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
533 .format(ivld["name"], indata["id"], ivld["id"]),
534 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
535 else:
536 vld_names.append(ivld_name)
537 # END Detection of duplicated VLD names
538 for icp in get_iterable(ivld.get("internal-connection-point")):
539 icp_mark = False
540 for vdu in get_iterable(indata.get("vdu")):
541 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
542 if icp["id-ref"] == internal_cp["id"]:
543 icp_mark = True
544 break
545 if icp_mark:
546 break
547 else:
548 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
549 "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
550 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
551 if ivld.get("ip-profile-ref"):
552 for ip_prof in get_iterable(indata.get("ip-profiles")):
553 if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
554 break
555 else:
556 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
557 ivld["id"], ivld["ip-profile-ref"]),
558 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
559 for mp in get_iterable(indata.get("monitoring-param")):
560 if mp.get("vdu-monitoring-param"):
561 mp_vmp_mark = False
562 for vdu in get_iterable(indata.get("vdu")):
563 for vmp in get_iterable(vdu.get("monitoring-param")):
564 if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
565 mp["vdu-monitoring-param"]["vdu-ref"]:
566 mp_vmp_mark = True
567 break
568 if mp_vmp_mark:
569 break
570 else:
571 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
572 "defined at vdu[id='{}'] or vdu does not exist"
573 .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
574 mp["vdu-monitoring-param"]["vdu-ref"]),
575 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
576 elif mp.get("vdu-metric"):
577 mp_vm_mark = False
578 for vdu in get_iterable(indata.get("vdu")):
579 if vdu.get("vdu-configuration"):
580 for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
581 if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
582 mp["vdu-metric"]["vdu-ref"]:
583 mp_vm_mark = True
584 break
585 if mp_vm_mark:
586 break
587 else:
588 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
589 "vdu[id='{}'] or vdu does not exist"
590 .format(mp["vdu-metric"]["vdu-metric-name-ref"],
591 mp["vdu-metric"]["vdu-ref"]),
592 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
593
594 for sgd in get_iterable(indata.get("scaling-group-descriptor")):
595 for sp in get_iterable(sgd.get("scaling-policy")):
596 for sc in get_iterable(sp.get("scaling-criteria")):
597 for mp in get_iterable(indata.get("monitoring-param")):
598 if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
599 break
600 else:
601 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
602 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
603 .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
604 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
605 for sgd_vdu in get_iterable(sgd.get("vdu")):
606 sgd_vdu_mark = False
607 for vdu in get_iterable(indata.get("vdu")):
608 if vdu["id"] == sgd_vdu["vdu-id-ref"]:
609 sgd_vdu_mark = True
610 break
611 if sgd_vdu_mark:
612 break
613 else:
614 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
615 .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
616 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
617 for sca in get_iterable(sgd.get("scaling-config-action")):
618 if not indata.get("vnf-configuration"):
619 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
620 "scaling-group-descriptor[name='{}']:scaling-config-action"
621 .format(sgd["name"]),
622 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
623 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
624 if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
625 break
626 else:
627 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
628 "primitive-name-ref='{}' does not match any "
629 "vnf-configuration:config-primitive:name"
630 .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
631 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
632 return indata
633
634 def _validate_input_edit(self, indata, force=False):
635 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
636 return indata
637
638 def _validate_package_folders(self, storage_params, folder, file=None):
639 if not storage_params or not storage_params.get("pkg-dir"):
640 return False
641 else:
642 if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
643 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
644 else:
645 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
646 if file:
647 return self.fs.file_exists("{}/{}".format(f, file), 'file')
648 else:
649 if self.fs.file_exists(f, 'dir'):
650 if self.fs.dir_ls(f):
651 return True
652 return False
653
654
655 class NsdTopic(DescriptorTopic):
656 topic = "nsds"
657 topic_msg = "nsd"
658
659 def __init__(self, db, fs, msg, auth):
660 DescriptorTopic.__init__(self, db, fs, msg, auth)
661
662 @staticmethod
663 def _remove_envelop(indata=None):
664 if not indata:
665 return {}
666 clean_indata = indata
667
668 if clean_indata.get('nsd:nsd-catalog'):
669 clean_indata = clean_indata['nsd:nsd-catalog']
670 elif clean_indata.get('nsd-catalog'):
671 clean_indata = clean_indata['nsd-catalog']
672 if clean_indata.get('nsd'):
673 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
674 raise EngineException("'nsd' must be a list of only one element")
675 clean_indata = clean_indata['nsd'][0]
676 elif clean_indata.get('nsd:nsd'):
677 if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
678 raise EngineException("'nsd:nsd' must be a list of only one element")
679 clean_indata = clean_indata['nsd:nsd'][0]
680 return clean_indata
681
682 def _validate_input_new(self, indata, storage_params, force=False):
683 indata = self.pyangbind_validation("nsds", indata, force)
684 # Cross references validation in the descriptor
685 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
686 for vld in get_iterable(indata.get("vld")):
687 if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
688 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
689 " You cannot set an ip-profile when mgmt-network is True"
690 .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
691 for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
692 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
693 if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
694 if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
695 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
696 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
697 " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
698 constituent_vnfd["member-vnf-index"],
699 constituent_vnfd["vnfd-id-ref"]),
700 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
701 break
702 else:
703 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
704 "does not match any constituent-vnfd:member-vnf-index"
705 .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
706 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
707 return indata
708
709 def _validate_input_edit(self, indata, force=False):
710 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
711 return indata
712
713 def _check_descriptor_dependencies(self, session, descriptor):
714 """
715 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
716 connection points are ok
717 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
718 :param descriptor: descriptor to be inserted or edit
719 :return: None or raises exception
720 """
721 if session["force"]:
722 return
723 member_vnfd_index = {}
724 if descriptor.get("constituent-vnfd") and not session["force"]:
725 for vnf in descriptor["constituent-vnfd"]:
726 vnfd_id = vnf["vnfd-id-ref"]
727 filter_q = self._get_project_filter(session)
728 filter_q["id"] = vnfd_id
729 vnf_list = self.db.get_list("vnfds", filter_q)
730 if not vnf_list:
731 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
732 "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
733 # elif len(vnf_list) > 1:
734 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
735 # http_code=HTTPStatus.CONFLICT)
736 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
737
738 # Cross references validation in the descriptor and vnfd connection point validation
739 for vld in get_iterable(descriptor.get("vld")):
740 for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
741 # look if this vnfd contains this connection point
742 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
743 if not vnfd:
744 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
745 "does not match any constituent-vnfd:member-vnf-index"
746 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"]),
747 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
748 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
749 if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
750 break
751 else:
752 raise EngineException(
753 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
754 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
755 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
756 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
757 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
758
759 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
760 super().check_conflict_on_edit(session, final_content, edit_content, _id)
761
762 self._check_descriptor_dependencies(session, final_content)
763
764 def check_conflict_on_del(self, session, _id, db_content):
765 """
766 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
767 that NSD can be public and be used by other projects.
768 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
769 :param _id: nsd internal id
770 :param db_content: The database content of the _id
771 :return: None or raises EngineException with the conflict
772 """
773 if session["force"]:
774 return
775 descriptor = db_content
776 descriptor_id = descriptor.get("id")
777 if not descriptor_id: # empty nsd not uploaded
778 return
779
780 # check NSD used by NS
781 _filter = self._get_project_filter(session)
782 _filter["nsd-id"] = _id
783 if self.db.get_list("nsrs", _filter):
784 raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
785
786 # check NSD referenced by NST
787 del _filter["nsd-id"]
788 _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
789 if self.db.get_list("nsts", _filter):
790 raise EngineException("There is at least one NetSlice Template referencing this descriptor",
791 http_code=HTTPStatus.CONFLICT)
792
793
794 class NstTopic(DescriptorTopic):
795 topic = "nsts"
796 topic_msg = "nst"
797
798 def __init__(self, db, fs, msg, auth):
799 DescriptorTopic.__init__(self, db, fs, msg, auth)
800
801 @staticmethod
802 def _remove_envelop(indata=None):
803 if not indata:
804 return {}
805 clean_indata = indata
806
807 if clean_indata.get('nst'):
808 if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
809 raise EngineException("'nst' must be a list only one element")
810 clean_indata = clean_indata['nst'][0]
811 elif clean_indata.get('nst:nst'):
812 if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
813 raise EngineException("'nst:nst' must be a list only one element")
814 clean_indata = clean_indata['nst:nst'][0]
815 return clean_indata
816
817 def _validate_input_edit(self, indata, force=False):
818 # TODO validate with pyangbind, serialize
819 return indata
820
821 def _validate_input_new(self, indata, storage_params, force=False):
822 indata = self.pyangbind_validation("nsts", indata, force)
823 return indata.copy()
824
825 def _check_descriptor_dependencies(self, session, descriptor):
826 """
827 Check that the dependent descriptors exist on a new descriptor or edition
828 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
829 :param descriptor: descriptor to be inserted or edit
830 :return: None or raises exception
831 """
832 if not descriptor.get("netslice-subnet"):
833 return
834 for nsd in descriptor["netslice-subnet"]:
835 nsd_id = nsd["nsd-ref"]
836 filter_q = self._get_project_filter(session)
837 filter_q["id"] = nsd_id
838 if not self.db.get_list("nsds", filter_q):
839 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
840 "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
841
842 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
843 super().check_conflict_on_edit(session, final_content, edit_content, _id)
844
845 self._check_descriptor_dependencies(session, final_content)
846
847 def check_conflict_on_del(self, session, _id, db_content):
848 """
849 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
850 that NST can be public and be used by other projects.
851 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
852 :param _id: nst internal id
853 :param db_content: The database content of the _id.
854 :return: None or raises EngineException with the conflict
855 """
856 # TODO: Check this method
857 if session["force"]:
858 return
859 # Get Network Slice Template from Database
860 _filter = self._get_project_filter(session)
861 _filter["_admin.nst-id"] = _id
862 if self.db.get_list("nsis", _filter):
863 raise EngineException("there is at least one Netslice Instance using this descriptor",
864 http_code=HTTPStatus.CONFLICT)
865
866
867 class PduTopic(BaseTopic):
868 topic = "pdus"
869 topic_msg = "pdu"
870 schema_new = pdu_new_schema
871 schema_edit = pdu_edit_schema
872
873 def __init__(self, db, fs, msg, auth):
874 BaseTopic.__init__(self, db, fs, msg, auth)
875
876 @staticmethod
877 def format_on_new(content, project_id=None, make_public=False):
878 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
879 content["_admin"]["onboardingState"] = "CREATED"
880 content["_admin"]["operationalState"] = "ENABLED"
881 content["_admin"]["usageState"] = "NOT_IN_USE"
882
883 def check_conflict_on_del(self, session, _id, db_content):
884 """
885 Check that there is not any vnfr that uses this PDU
886 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
887 :param _id: pdu internal id
888 :param db_content: The database content of the _id.
889 :return: None or raises EngineException with the conflict
890 """
891 if session["force"]:
892 return
893
894 _filter = self._get_project_filter(session)
895 _filter["vdur.pdu-id"] = _id
896 if self.db.get_list("vnfrs", _filter):
897 raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)