change repo type to helm-chart,juju-bundle
[osm/NBI.git] / osm_nbi / descriptor_topics.py
1 # -*- coding: utf-8 -*-
2
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12 # implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15
16 import tarfile
17 import yaml
18 import json
19 # import logging
20 from hashlib import md5
21 from osm_common.dbbase import DbException, deep_update_rfc7396
22 from http import HTTPStatus
23 from time import time
24 from osm_nbi.validation import ValidationError, pdu_new_schema, pdu_edit_schema
25 from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable
26 from osm_im.vnfd import vnfd as vnfd_im
27 from osm_im.nsd import nsd as nsd_im
28 from osm_im.nst import nst as nst_im
29 from pyangbind.lib.serialise import pybindJSONDecoder
30 import pyangbind.lib.pybindJSON as pybindJSON
31
32 __author__ = "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
33
34
35 class DescriptorTopic(BaseTopic):
36
37 def __init__(self, db, fs, msg, auth):
38 BaseTopic.__init__(self, db, fs, msg, auth)
39
40 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
41 super().check_conflict_on_edit(session, final_content, edit_content, _id)
42
43 def _check_unique_id_name(descriptor, position=""):
44 for desc_key, desc_item in descriptor.items():
45 if isinstance(desc_item, list) and desc_item:
46 used_ids = []
47 desc_item_id = None
48 for index, list_item in enumerate(desc_item):
49 if isinstance(list_item, dict):
50 _check_unique_id_name(list_item, "{}.{}[{}]"
51 .format(position, desc_key, index))
52 # Base case
53 if index == 0 and (list_item.get("id") or list_item.get("name")):
54 desc_item_id = "id" if list_item.get("id") else "name"
55 if desc_item_id and list_item.get(desc_item_id):
56 if list_item[desc_item_id] in used_ids:
57 position = "{}.{}[{}]".format(position, desc_key, index)
58 raise EngineException("Error: identifier {} '{}' is not unique and repeats at '{}'"
59 .format(desc_item_id, list_item[desc_item_id],
60 position), HTTPStatus.UNPROCESSABLE_ENTITY)
61 used_ids.append(list_item[desc_item_id])
62 _check_unique_id_name(final_content)
63 # 1. validate again with pyangbind
64 # 1.1. remove internal keys
65 internal_keys = {}
66 for k in ("_id", "_admin"):
67 if k in final_content:
68 internal_keys[k] = final_content.pop(k)
69 storage_params = internal_keys["_admin"].get("storage")
70 serialized = self._validate_input_new(final_content, storage_params, session["force"])
71 # 1.2. modify final_content with a serialized version
72 final_content.clear()
73 final_content.update(serialized)
74 # 1.3. restore internal keys
75 for k, v in internal_keys.items():
76 final_content[k] = v
77
78 if session["force"]:
79 return
80 # 2. check that this id is not present
81 if "id" in edit_content:
82 _filter = self._get_project_filter(session)
83 _filter["id"] = final_content["id"]
84 _filter["_id.neq"] = _id
85 if self.db.get_one(self.topic, _filter, fail_on_empty=False):
86 raise EngineException("{} with id '{}' already exists for this project".format(self.topic[:-1],
87 final_content["id"]),
88 HTTPStatus.CONFLICT)
89
90 @staticmethod
91 def format_on_new(content, project_id=None, make_public=False):
92 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
93 content["_admin"]["onboardingState"] = "CREATED"
94 content["_admin"]["operationalState"] = "DISABLED"
95 content["_admin"]["usageState"] = "NOT_IN_USE"
96
97 def delete_extra(self, session, _id, db_content):
98 """
99 Deletes file system storage associated with the descriptor
100 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
101 :param _id: server internal id
102 :param db_content: The database content of the descriptor
103 :return: None if ok or raises EngineException with the problem
104 """
105 self.fs.file_delete(_id, ignore_non_exist=True)
106 self.fs.file_delete(_id + "_", ignore_non_exist=True) # remove temp folder
107
108 @staticmethod
109 def get_one_by_id(db, session, topic, id):
110 # find owned by this project
111 _filter = BaseTopic._get_project_filter(session)
112 _filter["id"] = id
113 desc_list = db.get_list(topic, _filter)
114 if len(desc_list) == 1:
115 return desc_list[0]
116 elif len(desc_list) > 1:
117 raise DbException("Found more than one {} with id='{}' belonging to this project".format(topic[:-1], id),
118 HTTPStatus.CONFLICT)
119
120 # not found any: try to find public
121 _filter = BaseTopic._get_project_filter(session)
122 _filter["id"] = id
123 desc_list = db.get_list(topic, _filter)
124 if not desc_list:
125 raise DbException("Not found any {} with id='{}'".format(topic[:-1], id), HTTPStatus.NOT_FOUND)
126 elif len(desc_list) == 1:
127 return desc_list[0]
128 else:
129 raise DbException("Found more than one public {} with id='{}'; and no one belonging to this project".format(
130 topic[:-1], id), HTTPStatus.CONFLICT)
131
132 def new(self, rollback, session, indata=None, kwargs=None, headers=None):
133 """
134 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
135 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
136 (self.upload_content)
137 :param rollback: list to append created items at database in case a rollback may to be done
138 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
139 :param indata: data to be inserted
140 :param kwargs: used to override the indata descriptor
141 :param headers: http request headers
142 :return: _id, None: identity of the inserted data; and None as there is not any operation
143 """
144
145 try:
146 # Check Quota
147 self.check_quota(session)
148
149 # _remove_envelop
150 if indata:
151 if "userDefinedData" in indata:
152 indata = indata['userDefinedData']
153
154 # Override descriptor with query string kwargs
155 self._update_input_with_kwargs(indata, kwargs)
156 # uncomment when this method is implemented.
157 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
158 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
159
160 content = {"_admin": {"userDefinedData": indata}}
161 self.format_on_new(content, session["project_id"], make_public=session["public"])
162 _id = self.db.create(self.topic, content)
163 rollback.append({"topic": self.topic, "_id": _id})
164 self._send_msg("created", {"_id": _id})
165 return _id, None
166 except ValidationError as e:
167 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
168
169 def upload_content(self, session, _id, indata, kwargs, headers):
170 """
171 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
172 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
173 :param _id : the nsd,vnfd is already created, this is the id
174 :param indata: http body request
175 :param kwargs: user query string to override parameters. NOT USED
176 :param headers: http request headers
177 :return: True if package is completely uploaded or False if partial content has been uploded
178 Raise exception on error
179 """
180 # Check that _id exists and it is valid
181 current_desc = self.show(session, _id)
182
183 content_range_text = headers.get("Content-Range")
184 expected_md5 = headers.get("Content-File-MD5")
185 compressed = None
186 content_type = headers.get("Content-Type")
187 if content_type and "application/gzip" in content_type or "application/x-gzip" in content_type or \
188 "application/zip" in content_type:
189 compressed = "gzip"
190 filename = headers.get("Content-Filename")
191 if not filename:
192 filename = "package.tar.gz" if compressed else "package"
193 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
194 file_pkg = None
195 error_text = ""
196 try:
197 if content_range_text:
198 content_range = content_range_text.replace("-", " ").replace("/", " ").split()
199 if content_range[0] != "bytes": # TODO check x<y not negative < total....
200 raise IndexError()
201 start = int(content_range[1])
202 end = int(content_range[2]) + 1
203 total = int(content_range[3])
204 else:
205 start = 0
206 temp_folder = _id + "_" # all the content is upload here and if ok, it is rename from id_ to is folder
207
208 if start:
209 if not self.fs.file_exists(temp_folder, 'dir'):
210 raise EngineException("invalid Transaction-Id header", HTTPStatus.NOT_FOUND)
211 else:
212 self.fs.file_delete(temp_folder, ignore_non_exist=True)
213 self.fs.mkdir(temp_folder)
214
215 storage = self.fs.get_params()
216 storage["folder"] = _id
217
218 file_path = (temp_folder, filename)
219 if self.fs.file_exists(file_path, 'file'):
220 file_size = self.fs.file_size(file_path)
221 else:
222 file_size = 0
223 if file_size != start:
224 raise EngineException("invalid Content-Range start sequence, expected '{}' but received '{}'".format(
225 file_size, start), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
226 file_pkg = self.fs.file_open(file_path, 'a+b')
227 if isinstance(indata, dict):
228 indata_text = yaml.safe_dump(indata, indent=4, default_flow_style=False)
229 file_pkg.write(indata_text.encode(encoding="utf-8"))
230 else:
231 indata_len = 0
232 while True:
233 indata_text = indata.read(4096)
234 indata_len += len(indata_text)
235 if not indata_text:
236 break
237 file_pkg.write(indata_text)
238 if content_range_text:
239 if indata_len != end-start:
240 raise EngineException("Mismatch between Content-Range header {}-{} and body length of {}".format(
241 start, end-1, indata_len), HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
242 if end != total:
243 # TODO update to UPLOADING
244 return False
245
246 # PACKAGE UPLOADED
247 if expected_md5:
248 file_pkg.seek(0, 0)
249 file_md5 = md5()
250 chunk_data = file_pkg.read(1024)
251 while chunk_data:
252 file_md5.update(chunk_data)
253 chunk_data = file_pkg.read(1024)
254 if expected_md5 != file_md5.hexdigest():
255 raise EngineException("Error, MD5 mismatch", HTTPStatus.CONFLICT)
256 file_pkg.seek(0, 0)
257 if compressed == "gzip":
258 tar = tarfile.open(mode='r', fileobj=file_pkg)
259 descriptor_file_name = None
260 for tarinfo in tar:
261 tarname = tarinfo.name
262 tarname_path = tarname.split("/")
263 if not tarname_path[0] or ".." in tarname_path: # if start with "/" means absolute path
264 raise EngineException("Absolute path or '..' are not allowed for package descriptor tar.gz")
265 if len(tarname_path) == 1 and not tarinfo.isdir():
266 raise EngineException("All files must be inside a dir for package descriptor tar.gz")
267 if tarname.endswith(".yaml") or tarname.endswith(".json") or tarname.endswith(".yml"):
268 storage["pkg-dir"] = tarname_path[0]
269 if len(tarname_path) == 2:
270 if descriptor_file_name:
271 raise EngineException(
272 "Found more than one descriptor file at package descriptor tar.gz")
273 descriptor_file_name = tarname
274 if not descriptor_file_name:
275 raise EngineException("Not found any descriptor file at package descriptor tar.gz")
276 storage["descriptor"] = descriptor_file_name
277 storage["zipfile"] = filename
278 self.fs.file_extract(tar, temp_folder)
279 with self.fs.file_open((temp_folder, descriptor_file_name), "r") as descriptor_file:
280 content = descriptor_file.read()
281 else:
282 content = file_pkg.read()
283 storage["descriptor"] = descriptor_file_name = filename
284
285 if descriptor_file_name.endswith(".json"):
286 error_text = "Invalid json format "
287 indata = json.load(content)
288 else:
289 error_text = "Invalid yaml format "
290 indata = yaml.load(content, Loader=yaml.SafeLoader)
291
292 current_desc["_admin"]["storage"] = storage
293 current_desc["_admin"]["onboardingState"] = "ONBOARDED"
294 current_desc["_admin"]["operationalState"] = "ENABLED"
295
296 indata = self._remove_envelop(indata)
297
298 # Override descriptor with query string kwargs
299 if kwargs:
300 self._update_input_with_kwargs(indata, kwargs)
301 # it will call overrides method at VnfdTopic or NsdTopic
302 # indata = self._validate_input_edit(indata, force=session["force"])
303
304 deep_update_rfc7396(current_desc, indata)
305 self.check_conflict_on_edit(session, current_desc, indata, _id=_id)
306 current_desc["_admin"]["modified"] = time()
307 self.db.replace(self.topic, _id, current_desc)
308 self.fs.dir_rename(temp_folder, _id)
309
310 indata["_id"] = _id
311 self._send_msg("edited", indata)
312
313 # TODO if descriptor has changed because kwargs update content and remove cached zip
314 # TODO if zip is not present creates one
315 return True
316
317 except EngineException:
318 raise
319 except IndexError:
320 raise EngineException("invalid Content-Range header format. Expected 'bytes start-end/total'",
321 HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE)
322 except IOError as e:
323 raise EngineException("invalid upload transaction sequence: '{}'".format(e), HTTPStatus.BAD_REQUEST)
324 except tarfile.ReadError as e:
325 raise EngineException("invalid file content {}".format(e), HTTPStatus.BAD_REQUEST)
326 except (ValueError, yaml.YAMLError) as e:
327 raise EngineException(error_text + str(e))
328 except ValidationError as e:
329 raise EngineException(e, HTTPStatus.UNPROCESSABLE_ENTITY)
330 finally:
331 if file_pkg:
332 file_pkg.close()
333
334 def get_file(self, session, _id, path=None, accept_header=None):
335 """
336 Return the file content of a vnfd or nsd
337 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
338 :param _id: Identity of the vnfd, nsd
339 :param path: artifact path or "$DESCRIPTOR" or None
340 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
341 :return: opened file plus Accept format or raises an exception
342 """
343 accept_text = accept_zip = False
344 if accept_header:
345 if 'text/plain' in accept_header or '*/*' in accept_header:
346 accept_text = True
347 if 'application/zip' in accept_header or '*/*' in accept_header:
348 accept_zip = 'application/zip'
349 elif 'application/gzip' in accept_header:
350 accept_zip = 'application/gzip'
351
352 if not accept_text and not accept_zip:
353 raise EngineException("provide request header 'Accept' with 'application/zip' or 'text/plain'",
354 http_code=HTTPStatus.NOT_ACCEPTABLE)
355
356 content = self.show(session, _id)
357 if content["_admin"]["onboardingState"] != "ONBOARDED":
358 raise EngineException("Cannot get content because this resource is not at 'ONBOARDED' state. "
359 "onboardingState is {}".format(content["_admin"]["onboardingState"]),
360 http_code=HTTPStatus.CONFLICT)
361 storage = content["_admin"]["storage"]
362 if path is not None and path != "$DESCRIPTOR": # artifacts
363 if not storage.get('pkg-dir'):
364 raise EngineException("Packages does not contains artifacts", http_code=HTTPStatus.BAD_REQUEST)
365 if self.fs.file_exists((storage['folder'], storage['pkg-dir'], *path), 'dir'):
366 folder_content = self.fs.dir_ls((storage['folder'], storage['pkg-dir'], *path))
367 return folder_content, "text/plain"
368 # TODO manage folders in http
369 else:
370 return self.fs.file_open((storage['folder'], storage['pkg-dir'], *path), "rb"),\
371 "application/octet-stream"
372
373 # pkgtype accept ZIP TEXT -> result
374 # manyfiles yes X -> zip
375 # no yes -> error
376 # onefile yes no -> zip
377 # X yes -> text
378
379 if accept_text and (not storage.get('pkg-dir') or path == "$DESCRIPTOR"):
380 return self.fs.file_open((storage['folder'], storage['descriptor']), "r"), "text/plain"
381 elif storage.get('pkg-dir') and not accept_zip:
382 raise EngineException("Packages that contains several files need to be retrieved with 'application/zip'"
383 "Accept header", http_code=HTTPStatus.NOT_ACCEPTABLE)
384 else:
385 if not storage.get('zipfile'):
386 # TODO generate zipfile if not present
387 raise EngineException("Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
388 "future versions", http_code=HTTPStatus.NOT_ACCEPTABLE)
389 return self.fs.file_open((storage['folder'], storage['zipfile']), "rb"), accept_zip
390
391 def pyangbind_validation(self, item, data, force=False):
392 try:
393 if item == "vnfds":
394 myvnfd = vnfd_im()
395 pybindJSONDecoder.load_ietf_json({'vnfd:vnfd-catalog': {'vnfd': [data]}}, None, None, obj=myvnfd,
396 path_helper=True, skip_unknown=force)
397 out = pybindJSON.dumps(myvnfd, mode="ietf")
398 elif item == "nsds":
399 mynsd = nsd_im()
400 pybindJSONDecoder.load_ietf_json({'nsd:nsd-catalog': {'nsd': [data]}}, None, None, obj=mynsd,
401 path_helper=True, skip_unknown=force)
402 out = pybindJSON.dumps(mynsd, mode="ietf")
403 elif item == "nsts":
404 mynst = nst_im()
405 pybindJSONDecoder.load_ietf_json({'nst': [data]}, None, None, obj=mynst,
406 path_helper=True, skip_unknown=force)
407 out = pybindJSON.dumps(mynst, mode="ietf")
408 else:
409 raise EngineException("Not possible to validate '{}' item".format(item),
410 http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
411
412 desc_out = self._remove_envelop(yaml.safe_load(out))
413 return desc_out
414
415 except Exception as e:
416 raise EngineException("Error in pyangbind validation: {}".format(str(e)),
417 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
418
419
420 class VnfdTopic(DescriptorTopic):
421 topic = "vnfds"
422 topic_msg = "vnfd"
423
424 def __init__(self, db, fs, msg, auth):
425 DescriptorTopic.__init__(self, db, fs, msg, auth)
426
427 @staticmethod
428 def _remove_envelop(indata=None):
429 if not indata:
430 return {}
431 clean_indata = indata
432 if clean_indata.get('vnfd:vnfd-catalog'):
433 clean_indata = clean_indata['vnfd:vnfd-catalog']
434 elif clean_indata.get('vnfd-catalog'):
435 clean_indata = clean_indata['vnfd-catalog']
436 if clean_indata.get('vnfd'):
437 if not isinstance(clean_indata['vnfd'], list) or len(clean_indata['vnfd']) != 1:
438 raise EngineException("'vnfd' must be a list of only one element")
439 clean_indata = clean_indata['vnfd'][0]
440 elif clean_indata.get('vnfd:vnfd'):
441 if not isinstance(clean_indata['vnfd:vnfd'], list) or len(clean_indata['vnfd:vnfd']) != 1:
442 raise EngineException("'vnfd:vnfd' must be a list of only one element")
443 clean_indata = clean_indata['vnfd:vnfd'][0]
444 return clean_indata
445
446 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
447 super().check_conflict_on_edit(session, final_content, edit_content, _id)
448
449 # set type of vnfd
450 contains_pdu = False
451 contains_vdu = False
452 for vdu in get_iterable(final_content.get("vdu")):
453 if vdu.get("pdu-type"):
454 contains_pdu = True
455 else:
456 contains_vdu = True
457 if contains_pdu:
458 final_content["_admin"]["type"] = "hnfd" if contains_vdu else "pnfd"
459 elif contains_vdu:
460 final_content["_admin"]["type"] = "vnfd"
461 # if neither vud nor pdu do not fill type
462
463 def check_conflict_on_del(self, session, _id, db_content):
464 """
465 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
466 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
467 that uses this vnfd
468 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
469 :param _id: vnfd internal id
470 :param db_content: The database content of the _id.
471 :return: None or raises EngineException with the conflict
472 """
473 if session["force"]:
474 return
475 descriptor = db_content
476 descriptor_id = descriptor.get("id")
477 if not descriptor_id: # empty vnfd not uploaded
478 return
479
480 _filter = self._get_project_filter(session)
481
482 # check vnfrs using this vnfd
483 _filter["vnfd-id"] = _id
484 if self.db.get_list("vnfrs", _filter):
485 raise EngineException("There is at least one VNF using this descriptor", http_code=HTTPStatus.CONFLICT)
486
487 # check NSD referencing this VNFD
488 del _filter["vnfd-id"]
489 _filter["constituent-vnfd.ANYINDEX.vnfd-id-ref"] = descriptor_id
490 if self.db.get_list("nsds", _filter):
491 raise EngineException("There is at least one NSD referencing this descriptor",
492 http_code=HTTPStatus.CONFLICT)
493
494 def _validate_input_new(self, indata, storage_params, force=False):
495 indata = self.pyangbind_validation("vnfds", indata, force)
496 # Cross references validation in the descriptor
497 if indata.get("vdu"):
498 if not indata.get("mgmt-interface"):
499 raise EngineException("'mgmt-interface' is a mandatory field and it is not defined",
500 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
501 if indata["mgmt-interface"].get("cp"):
502 for cp in get_iterable(indata.get("connection-point")):
503 if cp["name"] == indata["mgmt-interface"]["cp"]:
504 break
505 else:
506 raise EngineException("mgmt-interface:cp='{}' must match an existing connection-point"
507 .format(indata["mgmt-interface"]["cp"]),
508 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
509
510 for vdu in get_iterable(indata.get("vdu")):
511 for interface in get_iterable(vdu.get("interface")):
512 if interface.get("external-connection-point-ref"):
513 for cp in get_iterable(indata.get("connection-point")):
514 if cp["name"] == interface["external-connection-point-ref"]:
515 break
516 else:
517 raise EngineException("vdu[id='{}']:interface[name='{}']:external-connection-point-ref='{}' "
518 "must match an existing connection-point"
519 .format(vdu["id"], interface["name"],
520 interface["external-connection-point-ref"]),
521 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
522
523 elif interface.get("internal-connection-point-ref"):
524 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
525 if interface["internal-connection-point-ref"] == internal_cp.get("id"):
526 break
527 else:
528 raise EngineException("vdu[id='{}']:interface[name='{}']:internal-connection-point-ref='{}' "
529 "must match an existing vdu:internal-connection-point"
530 .format(vdu["id"], interface["name"],
531 interface["internal-connection-point-ref"]),
532 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
533 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
534 if vdu.get("vdu-configuration"):
535 if vdu["vdu-configuration"].get("juju"):
536 if not self._validate_package_folders(storage_params, 'charms'):
537 raise EngineException("Charm defined in vnf[id={}]:vdu[id={}] but not present in "
538 "package".format(indata["id"], vdu["id"]))
539 # Validate that if descriptor contains cloud-init, artifacts _admin.storage."pkg-dir" is not none
540 if vdu.get("cloud-init-file"):
541 if not self._validate_package_folders(storage_params, 'cloud_init', vdu["cloud-init-file"]):
542 raise EngineException("Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
543 "package".format(indata["id"], vdu["id"]))
544 # Validate that if descriptor contains charms, artifacts _admin.storage."pkg-dir" is not none
545 if indata.get("vnf-configuration"):
546 if indata["vnf-configuration"].get("juju"):
547 if not self._validate_package_folders(storage_params, 'charms'):
548 raise EngineException("Charm defined in vnf[id={}] but not present in "
549 "package".format(indata["id"]))
550 vld_names = [] # For detection of duplicated VLD names
551 for ivld in get_iterable(indata.get("internal-vld")):
552 # BEGIN Detection of duplicated VLD names
553 ivld_name = ivld["name"]
554 if ivld_name in vld_names:
555 raise EngineException("Duplicated VLD name '{}' in vnfd[id={}]:internal-vld[id={}]"
556 .format(ivld["name"], indata["id"], ivld["id"]),
557 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
558 else:
559 vld_names.append(ivld_name)
560 # END Detection of duplicated VLD names
561 for icp in get_iterable(ivld.get("internal-connection-point")):
562 icp_mark = False
563 for vdu in get_iterable(indata.get("vdu")):
564 for internal_cp in get_iterable(vdu.get("internal-connection-point")):
565 if icp["id-ref"] == internal_cp["id"]:
566 icp_mark = True
567 break
568 if icp_mark:
569 break
570 else:
571 raise EngineException("internal-vld[id='{}']:internal-connection-point='{}' must match an existing "
572 "vdu:internal-connection-point".format(ivld["id"], icp["id-ref"]),
573 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
574 if ivld.get("ip-profile-ref"):
575 for ip_prof in get_iterable(indata.get("ip-profiles")):
576 if ip_prof["name"] == get_iterable(ivld.get("ip-profile-ref")):
577 break
578 else:
579 raise EngineException("internal-vld[id='{}']:ip-profile-ref='{}' does not exist".format(
580 ivld["id"], ivld["ip-profile-ref"]),
581 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
582 for mp in get_iterable(indata.get("monitoring-param")):
583 if mp.get("vdu-monitoring-param"):
584 mp_vmp_mark = False
585 for vdu in get_iterable(indata.get("vdu")):
586 for vmp in get_iterable(vdu.get("monitoring-param")):
587 if vmp["id"] == mp["vdu-monitoring-param"].get("vdu-monitoring-param-ref") and vdu["id"] ==\
588 mp["vdu-monitoring-param"]["vdu-ref"]:
589 mp_vmp_mark = True
590 break
591 if mp_vmp_mark:
592 break
593 else:
594 raise EngineException("monitoring-param:vdu-monitoring-param:vdu-monitoring-param-ref='{}' not "
595 "defined at vdu[id='{}'] or vdu does not exist"
596 .format(mp["vdu-monitoring-param"]["vdu-monitoring-param-ref"],
597 mp["vdu-monitoring-param"]["vdu-ref"]),
598 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
599 elif mp.get("vdu-metric"):
600 mp_vm_mark = False
601 for vdu in get_iterable(indata.get("vdu")):
602 if vdu.get("vdu-configuration"):
603 for metric in get_iterable(vdu["vdu-configuration"].get("metrics")):
604 if metric["name"] == mp["vdu-metric"]["vdu-metric-name-ref"] and vdu["id"] == \
605 mp["vdu-metric"]["vdu-ref"]:
606 mp_vm_mark = True
607 break
608 if mp_vm_mark:
609 break
610 else:
611 raise EngineException("monitoring-param:vdu-metric:vdu-metric-name-ref='{}' not defined at "
612 "vdu[id='{}'] or vdu does not exist"
613 .format(mp["vdu-metric"]["vdu-metric-name-ref"],
614 mp["vdu-metric"]["vdu-ref"]),
615 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
616
617 for sgd in get_iterable(indata.get("scaling-group-descriptor")):
618 for sp in get_iterable(sgd.get("scaling-policy")):
619 for sc in get_iterable(sp.get("scaling-criteria")):
620 for mp in get_iterable(indata.get("monitoring-param")):
621 if mp["id"] == get_iterable(sc.get("vnf-monitoring-param-ref")):
622 break
623 else:
624 raise EngineException("scaling-group-descriptor[name='{}']:scaling-criteria[name='{}']:"
625 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param"
626 .format(sgd["name"], sc["name"], sc["vnf-monitoring-param-ref"]),
627 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
628 for sgd_vdu in get_iterable(sgd.get("vdu")):
629 sgd_vdu_mark = False
630 for vdu in get_iterable(indata.get("vdu")):
631 if vdu["id"] == sgd_vdu["vdu-id-ref"]:
632 sgd_vdu_mark = True
633 break
634 if sgd_vdu_mark:
635 break
636 else:
637 raise EngineException("scaling-group-descriptor[name='{}']:vdu-id-ref={} does not match any vdu"
638 .format(sgd["name"], sgd_vdu["vdu-id-ref"]),
639 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
640 for sca in get_iterable(sgd.get("scaling-config-action")):
641 if not indata.get("vnf-configuration"):
642 raise EngineException("'vnf-configuration' not defined in the descriptor but it is referenced by "
643 "scaling-group-descriptor[name='{}']:scaling-config-action"
644 .format(sgd["name"]),
645 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
646 for primitive in get_iterable(indata["vnf-configuration"].get("config-primitive")):
647 if primitive["name"] == sca["vnf-config-primitive-name-ref"]:
648 break
649 else:
650 raise EngineException("scaling-group-descriptor[name='{}']:scaling-config-action:vnf-config-"
651 "primitive-name-ref='{}' does not match any "
652 "vnf-configuration:config-primitive:name"
653 .format(sgd["name"], sca["vnf-config-primitive-name-ref"]),
654 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
655 return indata
656
657 def _validate_input_edit(self, indata, force=False):
658 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
659 return indata
660
661 def _validate_package_folders(self, storage_params, folder, file=None):
662 if not storage_params or not storage_params.get("pkg-dir"):
663 return False
664 else:
665 if self.fs.file_exists("{}_".format(storage_params["folder"]), 'dir'):
666 f = "{}_/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
667 else:
668 f = "{}/{}/{}".format(storage_params["folder"], storage_params["pkg-dir"], folder)
669 if file:
670 return self.fs.file_exists("{}/{}".format(f, file), 'file')
671 else:
672 if self.fs.file_exists(f, 'dir'):
673 if self.fs.dir_ls(f):
674 return True
675 return False
676
677
678 class NsdTopic(DescriptorTopic):
679 topic = "nsds"
680 topic_msg = "nsd"
681
682 def __init__(self, db, fs, msg, auth):
683 DescriptorTopic.__init__(self, db, fs, msg, auth)
684
685 @staticmethod
686 def _remove_envelop(indata=None):
687 if not indata:
688 return {}
689 clean_indata = indata
690
691 if clean_indata.get('nsd:nsd-catalog'):
692 clean_indata = clean_indata['nsd:nsd-catalog']
693 elif clean_indata.get('nsd-catalog'):
694 clean_indata = clean_indata['nsd-catalog']
695 if clean_indata.get('nsd'):
696 if not isinstance(clean_indata['nsd'], list) or len(clean_indata['nsd']) != 1:
697 raise EngineException("'nsd' must be a list of only one element")
698 clean_indata = clean_indata['nsd'][0]
699 elif clean_indata.get('nsd:nsd'):
700 if not isinstance(clean_indata['nsd:nsd'], list) or len(clean_indata['nsd:nsd']) != 1:
701 raise EngineException("'nsd:nsd' must be a list of only one element")
702 clean_indata = clean_indata['nsd:nsd'][0]
703 return clean_indata
704
705 def _validate_input_new(self, indata, storage_params, force=False):
706 indata = self.pyangbind_validation("nsds", indata, force)
707 # Cross references validation in the descriptor
708 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
709 for vld in get_iterable(indata.get("vld")):
710 if vld.get("mgmt-network") and vld.get("ip-profile-ref"):
711 raise EngineException("Error at vld[id='{}']:ip-profile-ref"
712 " You cannot set an ip-profile when mgmt-network is True"
713 .format(vld["id"]), http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
714 for vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
715 for constituent_vnfd in get_iterable(indata.get("constituent-vnfd")):
716 if vnfd_cp["member-vnf-index-ref"] == constituent_vnfd["member-vnf-index"]:
717 if vnfd_cp.get("vnfd-id-ref") and vnfd_cp["vnfd-id-ref"] != constituent_vnfd["vnfd-id-ref"]:
718 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[vnfd-id-ref='{}'] "
719 "does not match constituent-vnfd[member-vnf-index='{}']:vnfd-id-ref"
720 " '{}'".format(vld["id"], vnfd_cp["vnfd-id-ref"],
721 constituent_vnfd["member-vnf-index"],
722 constituent_vnfd["vnfd-id-ref"]),
723 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
724 break
725 else:
726 raise EngineException("Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}'] "
727 "does not match any constituent-vnfd:member-vnf-index"
728 .format(vld["id"], vnfd_cp["member-vnf-index-ref"]),
729 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
730 # Check VNFFGD
731 for fgd in get_iterable(indata.get("vnffgd")):
732 for cls in get_iterable(fgd.get("classifier")):
733 rspref = cls.get("rsp-id-ref")
734 for rsp in get_iterable(fgd.get("rsp")):
735 rspid = rsp.get("id")
736 if rspid and rspref and rspid == rspref:
737 break
738 else:
739 raise EngineException(
740 "Error at vnffgd[id='{}']:classifier[id='{}']:rsp-id-ref '{}' does not match any rsp:id"
741 .format(fgd["id"], cls["id"], rspref),
742 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
743 return indata
744
745 def _validate_input_edit(self, indata, force=False):
746 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
747 return indata
748
749 def _check_descriptor_dependencies(self, session, descriptor):
750 """
751 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
752 connection points are ok
753 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
754 :param descriptor: descriptor to be inserted or edit
755 :return: None or raises exception
756 """
757 if session["force"]:
758 return
759 member_vnfd_index = {}
760 if descriptor.get("constituent-vnfd") and not session["force"]:
761 for vnf in descriptor["constituent-vnfd"]:
762 vnfd_id = vnf["vnfd-id-ref"]
763 filter_q = self._get_project_filter(session)
764 filter_q["id"] = vnfd_id
765 vnf_list = self.db.get_list("vnfds", filter_q)
766 if not vnf_list:
767 raise EngineException("Descriptor error at 'constituent-vnfd':'vnfd-id-ref'='{}' references a non "
768 "existing vnfd".format(vnfd_id), http_code=HTTPStatus.CONFLICT)
769 # elif len(vnf_list) > 1:
770 # raise EngineException("More than one vnfd found for id='{}'".format(vnfd_id),
771 # http_code=HTTPStatus.CONFLICT)
772 member_vnfd_index[vnf["member-vnf-index"]] = vnf_list[0]
773
774 # Cross references validation in the descriptor and vnfd connection point validation
775 for vld in get_iterable(descriptor.get("vld")):
776 for referenced_vnfd_cp in get_iterable(vld.get("vnfd-connection-point-ref")):
777 # look if this vnfd contains this connection point
778 vnfd = member_vnfd_index.get(referenced_vnfd_cp["member-vnf-index-ref"])
779 for vnfd_cp in get_iterable(vnfd.get("connection-point")):
780 if referenced_vnfd_cp.get("vnfd-connection-point-ref") == vnfd_cp["name"]:
781 break
782 else:
783 raise EngineException(
784 "Error at vld[id='{}']:vnfd-connection-point-ref[member-vnf-index-ref='{}']:vnfd-"
785 "connection-point-ref='{}' references a non existing conection-point:name inside vnfd '{}'"
786 .format(vld["id"], referenced_vnfd_cp["member-vnf-index-ref"],
787 referenced_vnfd_cp["vnfd-connection-point-ref"], vnfd["id"]),
788 http_code=HTTPStatus.UNPROCESSABLE_ENTITY)
789
790 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
791 super().check_conflict_on_edit(session, final_content, edit_content, _id)
792
793 self._check_descriptor_dependencies(session, final_content)
794
795 def check_conflict_on_del(self, session, _id, db_content):
796 """
797 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
798 that NSD can be public and be used by other projects.
799 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
800 :param _id: nsd internal id
801 :param db_content: The database content of the _id
802 :return: None or raises EngineException with the conflict
803 """
804 if session["force"]:
805 return
806 descriptor = db_content
807 descriptor_id = descriptor.get("id")
808 if not descriptor_id: # empty nsd not uploaded
809 return
810
811 # check NSD used by NS
812 _filter = self._get_project_filter(session)
813 _filter["nsd-id"] = _id
814 if self.db.get_list("nsrs", _filter):
815 raise EngineException("There is at least one NS using this descriptor", http_code=HTTPStatus.CONFLICT)
816
817 # check NSD referenced by NST
818 del _filter["nsd-id"]
819 _filter["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
820 if self.db.get_list("nsts", _filter):
821 raise EngineException("There is at least one NetSlice Template referencing this descriptor",
822 http_code=HTTPStatus.CONFLICT)
823
824
825 class NstTopic(DescriptorTopic):
826 topic = "nsts"
827 topic_msg = "nst"
828
829 def __init__(self, db, fs, msg, auth):
830 DescriptorTopic.__init__(self, db, fs, msg, auth)
831
832 @staticmethod
833 def _remove_envelop(indata=None):
834 if not indata:
835 return {}
836 clean_indata = indata
837
838 if clean_indata.get('nst'):
839 if not isinstance(clean_indata['nst'], list) or len(clean_indata['nst']) != 1:
840 raise EngineException("'nst' must be a list only one element")
841 clean_indata = clean_indata['nst'][0]
842 elif clean_indata.get('nst:nst'):
843 if not isinstance(clean_indata['nst:nst'], list) or len(clean_indata['nst:nst']) != 1:
844 raise EngineException("'nst:nst' must be a list only one element")
845 clean_indata = clean_indata['nst:nst'][0]
846 return clean_indata
847
848 def _validate_input_edit(self, indata, force=False):
849 # TODO validate with pyangbind, serialize
850 return indata
851
852 def _validate_input_new(self, indata, storage_params, force=False):
853 indata = self.pyangbind_validation("nsts", indata, force)
854 return indata.copy()
855
856 def _check_descriptor_dependencies(self, session, descriptor):
857 """
858 Check that the dependent descriptors exist on a new descriptor or edition
859 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
860 :param descriptor: descriptor to be inserted or edit
861 :return: None or raises exception
862 """
863 if not descriptor.get("netslice-subnet"):
864 return
865 for nsd in descriptor["netslice-subnet"]:
866 nsd_id = nsd["nsd-ref"]
867 filter_q = self._get_project_filter(session)
868 filter_q["id"] = nsd_id
869 if not self.db.get_list("nsds", filter_q):
870 raise EngineException("Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
871 "existing nsd".format(nsd_id), http_code=HTTPStatus.CONFLICT)
872
873 def check_conflict_on_edit(self, session, final_content, edit_content, _id):
874 super().check_conflict_on_edit(session, final_content, edit_content, _id)
875
876 self._check_descriptor_dependencies(session, final_content)
877
878 def check_conflict_on_del(self, session, _id, db_content):
879 """
880 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
881 that NST can be public and be used by other projects.
882 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
883 :param _id: nst internal id
884 :param db_content: The database content of the _id.
885 :return: None or raises EngineException with the conflict
886 """
887 # TODO: Check this method
888 if session["force"]:
889 return
890 # Get Network Slice Template from Database
891 _filter = self._get_project_filter(session)
892 _filter["_admin.nst-id"] = _id
893 if self.db.get_list("nsis", _filter):
894 raise EngineException("there is at least one Netslice Instance using this descriptor",
895 http_code=HTTPStatus.CONFLICT)
896
897
898 class PduTopic(BaseTopic):
899 topic = "pdus"
900 topic_msg = "pdu"
901 schema_new = pdu_new_schema
902 schema_edit = pdu_edit_schema
903
904 def __init__(self, db, fs, msg, auth):
905 BaseTopic.__init__(self, db, fs, msg, auth)
906
907 @staticmethod
908 def format_on_new(content, project_id=None, make_public=False):
909 BaseTopic.format_on_new(content, project_id=project_id, make_public=make_public)
910 content["_admin"]["onboardingState"] = "CREATED"
911 content["_admin"]["operationalState"] = "ENABLED"
912 content["_admin"]["usageState"] = "NOT_IN_USE"
913
914 def check_conflict_on_del(self, session, _id, db_content):
915 """
916 Check that there is not any vnfr that uses this PDU
917 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
918 :param _id: pdu internal id
919 :param db_content: The database content of the _id.
920 :return: None or raises EngineException with the conflict
921 """
922 if session["force"]:
923 return
924
925 _filter = self._get_project_filter(session)
926 _filter["vdur.pdu-id"] = _id
927 if self.db.get_list("vnfrs", _filter):
928 raise EngineException("There is at least one VNF using this PDU", http_code=HTTPStatus.CONFLICT)