1 # -*- coding: utf-8 -*-
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
25 from deepdiff
import DeepDiff
26 from hashlib
import md5
27 from osm_common
.dbbase
import DbException
, deep_update_rfc7396
28 from http
import HTTPStatus
30 from uuid
import uuid4
31 from re
import fullmatch
32 from zipfile
import ZipFile
33 from osm_nbi
.validation
import (
40 from osm_nbi
.base_topic
import (
44 detect_descriptor_usage
,
46 from osm_im
import etsi_nfv_vnfd
, etsi_nfv_nsd
47 from osm_im
.nst
import nst
as nst_im
48 from pyangbind
.lib
.serialise
import pybindJSONDecoder
49 import pyangbind
.lib
.pybindJSON
as pybindJSON
50 from osm_nbi
import utils
52 __author__
= "Alfonso Tierno <alfonso.tiernosepulveda@telefonica.com>"
55 class DescriptorTopic(BaseTopic
):
56 def __init__(self
, db
, fs
, msg
, auth
):
58 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
60 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
61 final_content
= super().check_conflict_on_edit(
62 session
, final_content
, edit_content
, _id
65 def _check_unique_id_name(descriptor
, position
=""):
66 for desc_key
, desc_item
in descriptor
.items():
67 if isinstance(desc_item
, list) and desc_item
:
70 for index
, list_item
in enumerate(desc_item
):
71 if isinstance(list_item
, dict):
72 _check_unique_id_name(
73 list_item
, "{}.{}[{}]".format(position
, desc_key
, index
)
77 list_item
.get("id") or list_item
.get("name")
79 desc_item_id
= "id" if list_item
.get("id") else "name"
80 if desc_item_id
and list_item
.get(desc_item_id
):
81 if list_item
[desc_item_id
] in used_ids
:
82 position
= "{}.{}[{}]".format(
83 position
, desc_key
, index
85 raise EngineException(
86 "Error: identifier {} '{}' is not unique and repeats at '{}'".format(
88 list_item
[desc_item_id
],
91 HTTPStatus
.UNPROCESSABLE_ENTITY
,
93 used_ids
.append(list_item
[desc_item_id
])
95 _check_unique_id_name(final_content
)
96 # 1. validate again with pyangbind
97 # 1.1. remove internal keys
99 for k
in ("_id", "_admin"):
100 if k
in final_content
:
101 internal_keys
[k
] = final_content
.pop(k
)
102 storage_params
= internal_keys
["_admin"].get("storage")
103 serialized
= self
._validate
_input
_new
(
104 final_content
, storage_params
, session
["force"]
107 # 1.2. modify final_content with a serialized version
108 final_content
= copy
.deepcopy(serialized
)
109 # 1.3. restore internal keys
110 for k
, v
in internal_keys
.items():
115 # 2. check that this id is not present
116 if "id" in edit_content
:
117 _filter
= self
._get
_project
_filter
(session
)
119 _filter
["id"] = final_content
["id"]
120 _filter
["_id.neq"] = _id
122 if self
.db
.get_one(self
.topic
, _filter
, fail_on_empty
=False):
123 raise EngineException(
124 "{} with id '{}' already exists for this project".format(
125 self
.topic
[:-1], final_content
["id"]
133 def format_on_new(content
, project_id
=None, make_public
=False):
134 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
135 content
["_admin"]["onboardingState"] = "CREATED"
136 content
["_admin"]["operationalState"] = "DISABLED"
137 content
["_admin"]["usageState"] = "NOT_IN_USE"
139 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
141 Deletes file system storage associated with the descriptor
142 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
143 :param _id: server internal id
144 :param db_content: The database content of the descriptor
145 :param not_send_msg: To not send message (False) or store content (list) instead
146 :return: None if ok or raises EngineException with the problem
148 self
.fs
.file_delete(_id
, ignore_non_exist
=True)
149 self
.fs
.file_delete(_id
+ "_", ignore_non_exist
=True) # remove temp folder
150 # Remove file revisions
151 if "revision" in db_content
["_admin"]:
152 revision
= db_content
["_admin"]["revision"]
154 self
.fs
.file_delete(_id
+ ":" + str(revision
), ignore_non_exist
=True)
155 revision
= revision
- 1
159 def get_one_by_id(db
, session
, topic
, id):
160 # find owned by this project
161 _filter
= BaseTopic
._get
_project
_filter
(session
)
163 desc_list
= db
.get_list(topic
, _filter
)
164 if len(desc_list
) == 1:
166 elif len(desc_list
) > 1:
168 "Found more than one {} with id='{}' belonging to this project".format(
174 # not found any: try to find public
175 _filter
= BaseTopic
._get
_project
_filter
(session
)
177 desc_list
= db
.get_list(topic
, _filter
)
180 "Not found any {} with id='{}'".format(topic
[:-1], id),
181 HTTPStatus
.NOT_FOUND
,
183 elif len(desc_list
) == 1:
187 "Found more than one public {} with id='{}'; and no one belonging to this project".format(
193 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
195 Creates a new almost empty DISABLED entry into database. Due to SOL005, it does not follow normal procedure.
196 Creating a VNFD or NSD is done in two steps: 1. Creates an empty descriptor (this step) and 2) upload content
197 (self.upload_content)
198 :param rollback: list to append created items at database in case a rollback may to be done
199 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
200 :param indata: data to be inserted
201 :param kwargs: used to override the indata descriptor
202 :param headers: http request headers
203 :return: _id, None: identity of the inserted data; and None as there is not any operation
206 # No needed to capture exceptions
208 self
.check_quota(session
)
212 if "userDefinedData" in indata
:
213 indata
= indata
["userDefinedData"]
215 # Override descriptor with query string kwargs
216 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
217 # uncomment when this method is implemented.
218 # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors
219 # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"])
221 content
= {"_admin": {
222 "userDefinedData": indata
,
227 content
, session
["project_id"], make_public
=session
["public"]
229 _id
= self
.db
.create(self
.topic
, content
)
230 rollback
.append({"topic": self
.topic
, "_id": _id
})
231 self
._send
_msg
("created", {"_id": _id
})
234 def upload_content(self
, session
, _id
, indata
, kwargs
, headers
):
236 Used for receiving content by chunks (with a transaction_id header and/or gzip file. It will store and extract)
237 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
238 :param _id : the nsd,vnfd is already created, this is the id
239 :param indata: http body request
240 :param kwargs: user query string to override parameters. NOT USED
241 :param headers: http request headers
242 :return: True if package is completely uploaded or False if partial content has been uploded
243 Raise exception on error
245 # Check that _id exists and it is valid
246 current_desc
= self
.show(session
, _id
)
248 content_range_text
= headers
.get("Content-Range")
249 expected_md5
= headers
.get("Content-File-MD5")
251 content_type
= headers
.get("Content-Type")
254 and "application/gzip" in content_type
255 or "application/x-gzip" in content_type
260 and "application/zip" in content_type
263 filename
= headers
.get("Content-Filename")
264 if not filename
and compressed
:
265 filename
= "package.tar.gz" if compressed
== "gzip" else "package.zip"
270 if "revision" in current_desc
["_admin"]:
271 revision
= current_desc
["_admin"]["revision"] + 1
273 # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266
279 if content_range_text
:
281 content_range_text
.replace("-", " ").replace("/", " ").split()
284 content_range
[0] != "bytes"
285 ): # TODO check x<y not negative < total....
287 start
= int(content_range
[1])
288 end
= int(content_range
[2]) + 1
289 total
= int(content_range
[3])
292 # Rather than using a temp folder, we will store the package in a folder based on
293 # the current revision.
294 proposed_revision_path
= (
295 _id
+ ":" + str(revision
)
296 ) # all the content is upload here and if ok, it is rename from id_ to is folder
299 if not self
.fs
.file_exists(proposed_revision_path
, "dir"):
300 raise EngineException(
301 "invalid Transaction-Id header", HTTPStatus
.NOT_FOUND
304 self
.fs
.file_delete(proposed_revision_path
, ignore_non_exist
=True)
305 self
.fs
.mkdir(proposed_revision_path
)
306 fs_rollback
.append(proposed_revision_path
)
308 storage
= self
.fs
.get_params()
309 storage
["folder"] = proposed_revision_path
311 file_path
= (proposed_revision_path
, filename
)
312 if self
.fs
.file_exists(file_path
, "file"):
313 file_size
= self
.fs
.file_size(file_path
)
316 if file_size
!= start
:
317 raise EngineException(
318 "invalid Content-Range start sequence, expected '{}' but received '{}'".format(
321 HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
,
323 file_pkg
= self
.fs
.file_open(file_path
, "a+b")
324 if isinstance(indata
, dict):
325 indata_text
= yaml
.safe_dump(indata
, indent
=4, default_flow_style
=False)
326 file_pkg
.write(indata_text
.encode(encoding
="utf-8"))
330 indata_text
= indata
.read(4096)
331 indata_len
+= len(indata_text
)
334 file_pkg
.write(indata_text
)
335 if content_range_text
:
336 if indata_len
!= end
- start
:
337 raise EngineException(
338 "Mismatch between Content-Range header {}-{} and body length of {}".format(
339 start
, end
- 1, indata_len
341 HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
,
344 # TODO update to UPLOADING
351 chunk_data
= file_pkg
.read(1024)
353 file_md5
.update(chunk_data
)
354 chunk_data
= file_pkg
.read(1024)
355 if expected_md5
!= file_md5
.hexdigest():
356 raise EngineException("Error, MD5 mismatch", HTTPStatus
.CONFLICT
)
358 if compressed
== "gzip":
359 tar
= tarfile
.open(mode
="r", fileobj
=file_pkg
)
360 descriptor_file_name
= None
362 tarname
= tarinfo
.name
363 tarname_path
= tarname
.split("/")
365 not tarname_path
[0] or ".." in tarname_path
366 ): # if start with "/" means absolute path
367 raise EngineException(
368 "Absolute path or '..' are not allowed for package descriptor tar.gz"
370 if len(tarname_path
) == 1 and not tarinfo
.isdir():
371 raise EngineException(
372 "All files must be inside a dir for package descriptor tar.gz"
375 tarname
.endswith(".yaml")
376 or tarname
.endswith(".json")
377 or tarname
.endswith(".yml")
379 storage
["pkg-dir"] = tarname_path
[0]
380 if len(tarname_path
) == 2:
381 if descriptor_file_name
:
382 raise EngineException(
383 "Found more than one descriptor file at package descriptor tar.gz"
385 descriptor_file_name
= tarname
386 if not descriptor_file_name
:
387 raise EngineException(
388 "Not found any descriptor file at package descriptor tar.gz"
390 storage
["descriptor"] = descriptor_file_name
391 storage
["zipfile"] = filename
392 self
.fs
.file_extract(tar
, proposed_revision_path
)
393 with self
.fs
.file_open(
394 (proposed_revision_path
, descriptor_file_name
), "r"
395 ) as descriptor_file
:
396 content
= descriptor_file
.read()
397 elif compressed
== "zip":
398 zipfile
= ZipFile(file_pkg
)
399 descriptor_file_name
= None
400 for package_file
in zipfile
.infolist():
401 zipfilename
= package_file
.filename
402 file_path
= zipfilename
.split("/")
404 not file_path
[0] or ".." in zipfilename
405 ): # if start with "/" means absolute path
406 raise EngineException(
407 "Absolute path or '..' are not allowed for package descriptor zip"
412 zipfilename
.endswith(".yaml")
413 or zipfilename
.endswith(".json")
414 or zipfilename
.endswith(".yml")
416 zipfilename
.find("/") < 0
417 or zipfilename
.find("Definitions") >= 0
420 storage
["pkg-dir"] = ""
421 if descriptor_file_name
:
422 raise EngineException(
423 "Found more than one descriptor file at package descriptor zip"
425 descriptor_file_name
= zipfilename
426 if not descriptor_file_name
:
427 raise EngineException(
428 "Not found any descriptor file at package descriptor zip"
430 storage
["descriptor"] = descriptor_file_name
431 storage
["zipfile"] = filename
432 self
.fs
.file_extract(zipfile
, proposed_revision_path
)
434 with self
.fs
.file_open(
435 (proposed_revision_path
, descriptor_file_name
), "r"
436 ) as descriptor_file
:
437 content
= descriptor_file
.read()
439 content
= file_pkg
.read()
440 storage
["descriptor"] = descriptor_file_name
= filename
442 if descriptor_file_name
.endswith(".json"):
443 error_text
= "Invalid json format "
444 indata
= json
.load(content
)
446 error_text
= "Invalid yaml format "
447 indata
= yaml
.load(content
, Loader
=yaml
.SafeLoader
)
449 # Need to close the file package here so it can be copied from the
450 # revision to the current, unrevisioned record
455 # Fetch both the incoming, proposed revision and the original revision so we
456 # can call a validate method to compare them
457 current_revision_path
= _id
+ "/"
458 self
.fs
.sync(from_path
=current_revision_path
)
459 self
.fs
.sync(from_path
=proposed_revision_path
)
463 self
._validate
_descriptor
_changes
(
465 descriptor_file_name
,
466 current_revision_path
,
467 proposed_revision_path
,
469 except Exception as e
:
470 shutil
.rmtree(self
.fs
.path
+ current_revision_path
, ignore_errors
=True)
471 shutil
.rmtree(self
.fs
.path
+ proposed_revision_path
, ignore_errors
=True)
472 # Only delete the new revision. We need to keep the original version in place
473 # as it has not been changed.
474 self
.fs
.file_delete(proposed_revision_path
, ignore_non_exist
=True)
478 indata
= self
._remove
_envelop
(indata
)
480 # Override descriptor with query string kwargs
482 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
484 current_desc
["_admin"]["storage"] = storage
485 current_desc
["_admin"]["onboardingState"] = "ONBOARDED"
486 current_desc
["_admin"]["operationalState"] = "ENABLED"
487 current_desc
["_admin"]["modified"] = time()
488 current_desc
["_admin"]["revision"] = revision
490 deep_update_rfc7396(current_desc
, indata
)
491 current_desc
= self
.check_conflict_on_edit(
492 session
, current_desc
, indata
, _id
=_id
495 # Copy the revision to the active package name by its original id
496 shutil
.rmtree(self
.fs
.path
+ current_revision_path
, ignore_errors
=True)
497 os
.rename(self
.fs
.path
+ proposed_revision_path
, self
.fs
.path
+ current_revision_path
)
498 self
.fs
.file_delete(current_revision_path
, ignore_non_exist
=True)
499 self
.fs
.mkdir(current_revision_path
)
500 self
.fs
.reverse_sync(from_path
=current_revision_path
)
502 shutil
.rmtree(self
.fs
.path
+ _id
)
504 self
.db
.replace(self
.topic
, _id
, current_desc
)
506 # Store a copy of the package as a point in time revision
507 revision_desc
= dict(current_desc
)
508 revision_desc
["_id"] = _id
+ ":" + str(revision_desc
["_admin"]["revision"])
509 self
.db
.create(self
.topic
+ "_revisions", revision_desc
)
513 self
._send
_msg
("edited", indata
)
515 # TODO if descriptor has changed because kwargs update content and remove cached zip
516 # TODO if zip is not present creates one
519 except EngineException
:
522 raise EngineException(
523 "invalid Content-Range header format. Expected 'bytes start-end/total'",
524 HTTPStatus
.REQUESTED_RANGE_NOT_SATISFIABLE
,
527 raise EngineException(
528 "invalid upload transaction sequence: '{}'".format(e
),
529 HTTPStatus
.BAD_REQUEST
,
531 except tarfile
.ReadError
as e
:
532 raise EngineException(
533 "invalid file content {}".format(e
), HTTPStatus
.BAD_REQUEST
535 except (ValueError, yaml
.YAMLError
) as e
:
536 raise EngineException(error_text
+ str(e
))
537 except ValidationError
as e
:
538 raise EngineException(e
, HTTPStatus
.UNPROCESSABLE_ENTITY
)
542 for file in fs_rollback
:
543 self
.fs
.file_delete(file, ignore_non_exist
=True)
545 def get_file(self
, session
, _id
, path
=None, accept_header
=None):
547 Return the file content of a vnfd or nsd
548 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
549 :param _id: Identity of the vnfd, nsd
550 :param path: artifact path or "$DESCRIPTOR" or None
551 :param accept_header: Content of Accept header. Must contain applition/zip or/and text/plain
552 :return: opened file plus Accept format or raises an exception
554 accept_text
= accept_zip
= False
556 if "text/plain" in accept_header
or "*/*" in accept_header
:
558 if "application/zip" in accept_header
or "*/*" in accept_header
:
559 accept_zip
= "application/zip"
560 elif "application/gzip" in accept_header
:
561 accept_zip
= "application/gzip"
563 if not accept_text
and not accept_zip
:
564 raise EngineException(
565 "provide request header 'Accept' with 'application/zip' or 'text/plain'",
566 http_code
=HTTPStatus
.NOT_ACCEPTABLE
,
569 content
= self
.show(session
, _id
)
570 if content
["_admin"]["onboardingState"] != "ONBOARDED":
571 raise EngineException(
572 "Cannot get content because this resource is not at 'ONBOARDED' state. "
573 "onboardingState is {}".format(content
["_admin"]["onboardingState"]),
574 http_code
=HTTPStatus
.CONFLICT
,
576 storage
= content
["_admin"]["storage"]
577 if path
is not None and path
!= "$DESCRIPTOR": # artifacts
578 if not storage
.get("pkg-dir") and not storage
.get("folder"):
579 raise EngineException(
580 "Packages does not contains artifacts",
581 http_code
=HTTPStatus
.BAD_REQUEST
,
583 if self
.fs
.file_exists(
584 (storage
["folder"], storage
["pkg-dir"], *path
), "dir"
586 folder_content
= self
.fs
.dir_ls(
587 (storage
["folder"], storage
["pkg-dir"], *path
)
589 return folder_content
, "text/plain"
590 # TODO manage folders in http
594 (storage
["folder"], storage
["pkg-dir"], *path
), "rb"
596 "application/octet-stream",
599 # pkgtype accept ZIP TEXT -> result
600 # manyfiles yes X -> zip
602 # onefile yes no -> zip
604 contain_many_files
= False
605 if storage
.get("pkg-dir"):
606 # check if there are more than one file in the package, ignoring checksums.txt.
607 pkg_files
= self
.fs
.dir_ls((storage
["folder"], storage
["pkg-dir"]))
608 if len(pkg_files
) >= 3 or (
609 len(pkg_files
) == 2 and "checksums.txt" not in pkg_files
611 contain_many_files
= True
612 if accept_text
and (not contain_many_files
or path
== "$DESCRIPTOR"):
614 self
.fs
.file_open((storage
["folder"], storage
["descriptor"]), "r"),
617 elif contain_many_files
and not accept_zip
:
618 raise EngineException(
619 "Packages that contains several files need to be retrieved with 'application/zip'"
621 http_code
=HTTPStatus
.NOT_ACCEPTABLE
,
624 if not storage
.get("zipfile"):
625 # TODO generate zipfile if not present
626 raise EngineException(
627 "Only allowed 'text/plain' Accept header for this descriptor. To be solved in "
629 http_code
=HTTPStatus
.NOT_ACCEPTABLE
,
632 self
.fs
.file_open((storage
["folder"], storage
["zipfile"]), "rb"),
636 def _remove_yang_prefixes_from_descriptor(self
, descriptor
):
638 for k
, v
in descriptor
.items():
640 if isinstance(v
, dict):
641 new_v
= self
._remove
_yang
_prefixes
_from
_descriptor
(v
)
642 elif isinstance(v
, list):
645 if isinstance(x
, dict):
646 new_v
.append(self
._remove
_yang
_prefixes
_from
_descriptor
(x
))
649 new_descriptor
[k
.split(":")[-1]] = new_v
650 return new_descriptor
652 def pyangbind_validation(self
, item
, data
, force
=False):
653 raise EngineException(
654 "Not possible to validate '{}' item".format(item
),
655 http_code
=HTTPStatus
.INTERNAL_SERVER_ERROR
,
658 def _validate_input_edit(self
, indata
, content
, force
=False):
659 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
662 if "_admin" not in indata
:
663 indata
["_admin"] = {}
665 if "operationalState" in indata
:
666 if indata
["operationalState"] in ("ENABLED", "DISABLED"):
667 indata
["_admin"]["operationalState"] = indata
.pop("operationalState")
669 raise EngineException(
670 "State '{}' is not a valid operational state".format(
671 indata
["operationalState"]
673 http_code
=HTTPStatus
.BAD_REQUEST
,
676 # In the case of user defined data, we need to put the data in the root of the object
677 # to preserve current expected behaviour
678 if "userDefinedData" in indata
:
679 data
= indata
.pop("userDefinedData")
680 if type(data
) == dict:
681 indata
["_admin"]["userDefinedData"] = data
683 raise EngineException(
684 "userDefinedData should be an object, but is '{}' instead".format(
687 http_code
=HTTPStatus
.BAD_REQUEST
,
691 "operationalState" in indata
["_admin"]
692 and content
["_admin"]["operationalState"]
693 == indata
["_admin"]["operationalState"]
695 raise EngineException(
696 "operationalState already {}".format(
697 content
["_admin"]["operationalState"]
699 http_code
=HTTPStatus
.CONFLICT
,
704 def _validate_descriptor_changes(
707 descriptor_file_name
,
708 old_descriptor_directory
,
709 new_descriptor_directory
712 # raise EngineException(
713 # "Error in validating new descriptor: <NODE> cannot be modified",
714 # http_code=HTTPStatus.UNPROCESSABLE_ENTITY,
718 class VnfdTopic(DescriptorTopic
):
722 def __init__(self
, db
, fs
, msg
, auth
):
723 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
725 def pyangbind_validation(self
, item
, data
, force
=False):
726 if self
._descriptor
_data
_is
_in
_old
_format
(data
):
727 raise EngineException(
728 "ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
729 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
732 myvnfd
= etsi_nfv_vnfd
.etsi_nfv_vnfd()
733 pybindJSONDecoder
.load_ietf_json(
734 {"etsi-nfv-vnfd:vnfd": data
},
741 out
= pybindJSON
.dumps(myvnfd
, mode
="ietf")
742 desc_out
= self
._remove
_envelop
(yaml
.safe_load(out
))
743 desc_out
= self
._remove
_yang
_prefixes
_from
_descriptor
(desc_out
)
744 return utils
.deep_update_dict(data
, desc_out
)
745 except Exception as e
:
746 raise EngineException(
747 "Error in pyangbind validation: {}".format(str(e
)),
748 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
752 def _descriptor_data_is_in_old_format(data
):
753 return ("vnfd-catalog" in data
) or ("vnfd:vnfd-catalog" in data
)
756 def _remove_envelop(indata
=None):
759 clean_indata
= indata
761 if clean_indata
.get("etsi-nfv-vnfd:vnfd"):
762 if not isinstance(clean_indata
["etsi-nfv-vnfd:vnfd"], dict):
763 raise EngineException("'etsi-nfv-vnfd:vnfd' must be a dict")
764 clean_indata
= clean_indata
["etsi-nfv-vnfd:vnfd"]
765 elif clean_indata
.get("vnfd"):
766 if not isinstance(clean_indata
["vnfd"], dict):
767 raise EngineException("'vnfd' must be dict")
768 clean_indata
= clean_indata
["vnfd"]
772 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
773 final_content
= super().check_conflict_on_edit(
774 session
, final_content
, edit_content
, _id
780 for vdu
in get_iterable(final_content
.get("vdu")):
781 if vdu
.get("pdu-type"):
786 final_content
["_admin"]["type"] = "hnfd" if contains_vdu
else "pnfd"
788 final_content
["_admin"]["type"] = "vnfd"
789 # if neither vud nor pdu do not fill type
792 def check_conflict_on_del(self
, session
, _id
, db_content
):
794 Check that there is not any NSD that uses this VNFD. Only NSDs belonging to this project are considered. Note
795 that VNFD can be public and be used by NSD of other projects. Also check there are not deployments, or vnfr
797 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
798 :param _id: vnfd internal id
799 :param db_content: The database content of the _id.
800 :return: None or raises EngineException with the conflict
804 descriptor
= db_content
805 descriptor_id
= descriptor
.get("id")
806 if not descriptor_id
: # empty vnfd not uploaded
809 _filter
= self
._get
_project
_filter
(session
)
811 # check vnfrs using this vnfd
812 _filter
["vnfd-id"] = _id
813 if self
.db
.get_list("vnfrs", _filter
):
814 raise EngineException(
815 "There is at least one VNF instance using this descriptor",
816 http_code
=HTTPStatus
.CONFLICT
,
819 # check NSD referencing this VNFD
820 del _filter
["vnfd-id"]
821 _filter
["vnfd-id"] = descriptor_id
822 if self
.db
.get_list("nsds", _filter
):
823 raise EngineException(
824 "There is at least one NS package referencing this descriptor",
825 http_code
=HTTPStatus
.CONFLICT
,
828 def _validate_input_new(self
, indata
, storage_params
, force
=False):
829 indata
.pop("onboardingState", None)
830 indata
.pop("operationalState", None)
831 indata
.pop("usageState", None)
832 indata
.pop("links", None)
834 indata
= self
.pyangbind_validation("vnfds", indata
, force
)
835 # Cross references validation in the descriptor
837 self
.validate_mgmt_interface_connection_point(indata
)
839 for vdu
in get_iterable(indata
.get("vdu")):
840 self
.validate_vdu_internal_connection_points(vdu
)
841 self
._validate
_vdu
_cloud
_init
_in
_package
(storage_params
, vdu
, indata
)
842 self
._validate
_vdu
_charms
_in
_package
(storage_params
, indata
)
844 self
._validate
_vnf
_charms
_in
_package
(storage_params
, indata
)
846 self
.validate_external_connection_points(indata
)
847 self
.validate_internal_virtual_links(indata
)
848 self
.validate_monitoring_params(indata
)
849 self
.validate_scaling_group_descriptor(indata
)
854 def validate_mgmt_interface_connection_point(indata
):
855 if not indata
.get("vdu"):
857 if not indata
.get("mgmt-cp"):
858 raise EngineException(
859 "'mgmt-cp' is a mandatory field and it is not defined",
860 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
863 for cp
in get_iterable(indata
.get("ext-cpd")):
864 if cp
["id"] == indata
["mgmt-cp"]:
867 raise EngineException(
868 "mgmt-cp='{}' must match an existing ext-cpd".format(indata
["mgmt-cp"]),
869 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
873 def validate_vdu_internal_connection_points(vdu
):
875 for cpd
in get_iterable(vdu
.get("int-cpd")):
876 cpd_id
= cpd
.get("id")
877 if cpd_id
and cpd_id
in int_cpds
:
878 raise EngineException(
879 "vdu[id='{}']:int-cpd[id='{}'] is already used by other int-cpd".format(
882 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
887 def validate_external_connection_points(indata
):
888 all_vdus_int_cpds
= set()
889 for vdu
in get_iterable(indata
.get("vdu")):
890 for int_cpd
in get_iterable(vdu
.get("int-cpd")):
891 all_vdus_int_cpds
.add((vdu
.get("id"), int_cpd
.get("id")))
894 for cpd
in get_iterable(indata
.get("ext-cpd")):
895 cpd_id
= cpd
.get("id")
896 if cpd_id
and cpd_id
in ext_cpds
:
897 raise EngineException(
898 "ext-cpd[id='{}'] is already used by other ext-cpd".format(cpd_id
),
899 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
903 int_cpd
= cpd
.get("int-cpd")
905 if (int_cpd
.get("vdu-id"), int_cpd
.get("cpd")) not in all_vdus_int_cpds
:
906 raise EngineException(
907 "ext-cpd[id='{}']:int-cpd must match an existing vdu int-cpd".format(
910 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
912 # TODO: Validate k8s-cluster-net points to a valid k8s-cluster:nets ?
914 def _validate_vdu_charms_in_package(self
, storage_params
, indata
):
915 for df
in indata
["df"]:
917 "lcm-operations-configuration" in df
918 and "operate-vnf-op-config" in df
["lcm-operations-configuration"]
920 configs
= df
["lcm-operations-configuration"][
921 "operate-vnf-op-config"
923 vdus
= df
.get("vdu-profile", [])
925 for config
in configs
:
926 if config
["id"] == vdu
["id"] and utils
.find_in_list(
927 config
.get("execution-environment-list", []),
928 lambda ee
: "juju" in ee
,
930 if not self
._validate
_package
_folders
(
931 storage_params
, "charms"
932 ) and not self
._validate
_package
_folders
(
933 storage_params
, "Scripts/charms"
935 raise EngineException(
936 "Charm defined in vnf[id={}] but not present in "
937 "package".format(indata
["id"])
940 def _validate_vdu_cloud_init_in_package(self
, storage_params
, vdu
, indata
):
941 if not vdu
.get("cloud-init-file"):
943 if not self
._validate
_package
_folders
(
944 storage_params
, "cloud_init", vdu
["cloud-init-file"]
945 ) and not self
._validate
_package
_folders
(
946 storage_params
, "Scripts/cloud_init", vdu
["cloud-init-file"]
948 raise EngineException(
949 "Cloud-init defined in vnf[id={}]:vdu[id={}] but not present in "
950 "package".format(indata
["id"], vdu
["id"])
953 def _validate_vnf_charms_in_package(self
, storage_params
, indata
):
954 # Get VNF configuration through new container
955 for deployment_flavor
in indata
.get("df", []):
956 if "lcm-operations-configuration" not in deployment_flavor
:
959 "operate-vnf-op-config"
960 not in deployment_flavor
["lcm-operations-configuration"]
963 for day_1_2_config
in deployment_flavor
["lcm-operations-configuration"][
964 "operate-vnf-op-config"
966 if day_1_2_config
["id"] == indata
["id"]:
967 if utils
.find_in_list(
968 day_1_2_config
.get("execution-environment-list", []),
969 lambda ee
: "juju" in ee
,
971 if not self
._validate
_package
_folders
(
972 storage_params
, "charms"
973 ) and not self
._validate
_package
_folders
(
974 storage_params
, "Scripts/charms"
976 raise EngineException(
977 "Charm defined in vnf[id={}] but not present in "
978 "package".format(indata
["id"])
981 def _validate_package_folders(self
, storage_params
, folder
, file=None):
982 if not storage_params
:
984 elif not storage_params
.get("pkg-dir"):
985 if self
.fs
.file_exists("{}_".format(storage_params
["folder"]), "dir"):
987 storage_params
["folder"], folder
991 storage_params
["folder"], folder
994 return self
.fs
.file_exists("{}/{}".format(f
, file), "file")
996 if self
.fs
.file_exists(f
, "dir"):
997 if self
.fs
.dir_ls(f
):
1001 if self
.fs
.file_exists("{}_".format(storage_params
["folder"]), "dir"):
1002 f
= "{}_/{}/{}".format(
1003 storage_params
["folder"], storage_params
["pkg-dir"], folder
1006 f
= "{}/{}/{}".format(
1007 storage_params
["folder"], storage_params
["pkg-dir"], folder
1010 return self
.fs
.file_exists("{}/{}".format(f
, file), "file")
1012 if self
.fs
.file_exists(f
, "dir"):
1013 if self
.fs
.dir_ls(f
):
1018 def validate_internal_virtual_links(indata
):
1019 all_ivld_ids
= set()
1020 for ivld
in get_iterable(indata
.get("int-virtual-link-desc")):
1021 ivld_id
= ivld
.get("id")
1022 if ivld_id
and ivld_id
in all_ivld_ids
:
1023 raise EngineException(
1024 "Duplicated VLD id in int-virtual-link-desc[id={}]".format(ivld_id
),
1025 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1028 all_ivld_ids
.add(ivld_id
)
1030 for vdu
in get_iterable(indata
.get("vdu")):
1031 for int_cpd
in get_iterable(vdu
.get("int-cpd")):
1032 int_cpd_ivld_id
= int_cpd
.get("int-virtual-link-desc")
1033 if int_cpd_ivld_id
and int_cpd_ivld_id
not in all_ivld_ids
:
1034 raise EngineException(
1035 "vdu[id='{}']:int-cpd[id='{}']:int-virtual-link-desc='{}' must match an existing "
1036 "int-virtual-link-desc".format(
1037 vdu
["id"], int_cpd
["id"], int_cpd_ivld_id
1039 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1042 for df
in get_iterable(indata
.get("df")):
1043 for vlp
in get_iterable(df
.get("virtual-link-profile")):
1044 vlp_ivld_id
= vlp
.get("id")
1045 if vlp_ivld_id
and vlp_ivld_id
not in all_ivld_ids
:
1046 raise EngineException(
1047 "df[id='{}']:virtual-link-profile='{}' must match an existing "
1048 "int-virtual-link-desc".format(df
["id"], vlp_ivld_id
),
1049 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1053 def validate_monitoring_params(indata
):
1054 all_monitoring_params
= set()
1055 for ivld
in get_iterable(indata
.get("int-virtual-link-desc")):
1056 for mp
in get_iterable(ivld
.get("monitoring-parameters")):
1057 mp_id
= mp
.get("id")
1058 if mp_id
and mp_id
in all_monitoring_params
:
1059 raise EngineException(
1060 "Duplicated monitoring-parameter id in "
1061 "int-virtual-link-desc[id='{}']:monitoring-parameters[id='{}']".format(
1064 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1067 all_monitoring_params
.add(mp_id
)
1069 for vdu
in get_iterable(indata
.get("vdu")):
1070 for mp
in get_iterable(vdu
.get("monitoring-parameter")):
1071 mp_id
= mp
.get("id")
1072 if mp_id
and mp_id
in all_monitoring_params
:
1073 raise EngineException(
1074 "Duplicated monitoring-parameter id in "
1075 "vdu[id='{}']:monitoring-parameter[id='{}']".format(
1078 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1081 all_monitoring_params
.add(mp_id
)
1083 for df
in get_iterable(indata
.get("df")):
1084 for mp
in get_iterable(df
.get("monitoring-parameter")):
1085 mp_id
= mp
.get("id")
1086 if mp_id
and mp_id
in all_monitoring_params
:
1087 raise EngineException(
1088 "Duplicated monitoring-parameter id in "
1089 "df[id='{}']:monitoring-parameter[id='{}']".format(
1092 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1095 all_monitoring_params
.add(mp_id
)
1098 def validate_scaling_group_descriptor(indata
):
1099 all_monitoring_params
= set()
1100 for ivld
in get_iterable(indata
.get("int-virtual-link-desc")):
1101 for mp
in get_iterable(ivld
.get("monitoring-parameters")):
1102 all_monitoring_params
.add(mp
.get("id"))
1104 for vdu
in get_iterable(indata
.get("vdu")):
1105 for mp
in get_iterable(vdu
.get("monitoring-parameter")):
1106 all_monitoring_params
.add(mp
.get("id"))
1108 for df
in get_iterable(indata
.get("df")):
1109 for mp
in get_iterable(df
.get("monitoring-parameter")):
1110 all_monitoring_params
.add(mp
.get("id"))
1112 for df
in get_iterable(indata
.get("df")):
1113 for sa
in get_iterable(df
.get("scaling-aspect")):
1114 for sp
in get_iterable(sa
.get("scaling-policy")):
1115 for sc
in get_iterable(sp
.get("scaling-criteria")):
1116 sc_monitoring_param
= sc
.get("vnf-monitoring-param-ref")
1119 and sc_monitoring_param
not in all_monitoring_params
1121 raise EngineException(
1122 "df[id='{}']:scaling-aspect[id='{}']:scaling-policy"
1123 "[name='{}']:scaling-criteria[name='{}']: "
1124 "vnf-monitoring-param-ref='{}' not defined in any monitoring-param".format(
1129 sc_monitoring_param
,
1131 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1134 for sca
in get_iterable(sa
.get("scaling-config-action")):
1136 "lcm-operations-configuration" not in df
1137 or "operate-vnf-op-config"
1138 not in df
["lcm-operations-configuration"]
1139 or not utils
.find_in_list(
1140 df
["lcm-operations-configuration"][
1141 "operate-vnf-op-config"
1142 ].get("day1-2", []),
1143 lambda config
: config
["id"] == indata
["id"],
1146 raise EngineException(
1147 "'day1-2 configuration' not defined in the descriptor but it is "
1148 "referenced by df[id='{}']:scaling-aspect[id='{}']:scaling-config-action".format(
1151 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1153 for configuration
in get_iterable(
1154 df
["lcm-operations-configuration"]["operate-vnf-op-config"].get(
1158 for primitive
in get_iterable(
1159 configuration
.get("config-primitive")
1163 == sca
["vnf-config-primitive-name-ref"]
1167 raise EngineException(
1168 "df[id='{}']:scaling-aspect[id='{}']:scaling-config-action:vnf-"
1169 "config-primitive-name-ref='{}' does not match any "
1170 "day1-2 configuration:config-primitive:name".format(
1173 sca
["vnf-config-primitive-name-ref"],
1175 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1178 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
1180 Deletes associate file system storage (via super)
1181 Deletes associated vnfpkgops from database.
1182 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1183 :param _id: server internal id
1184 :param db_content: The database content of the descriptor
1186 :raises: FsException in case of error while deleting associated storage
1188 super().delete_extra(session
, _id
, db_content
, not_send_msg
)
1189 self
.db
.del_list("vnfpkgops", {"vnfPkgId": _id
})
1190 self
.db
.del_list(self
.topic
+"_revisions", {"_id": {"$regex": _id
}})
1192 def sol005_projection(self
, data
):
1193 data
["onboardingState"] = data
["_admin"]["onboardingState"]
1194 data
["operationalState"] = data
["_admin"]["operationalState"]
1195 data
["usageState"] = data
["_admin"]["usageState"]
1198 links
["self"] = {"href": "/vnfpkgm/v1/vnf_packages/{}".format(data
["_id"])}
1199 links
["vnfd"] = {"href": "/vnfpkgm/v1/vnf_packages/{}/vnfd".format(data
["_id"])}
1200 links
["packageContent"] = {
1201 "href": "/vnfpkgm/v1/vnf_packages/{}/package_content".format(data
["_id"])
1203 data
["_links"] = links
1205 return super().sol005_projection(data
)
1208 def find_software_version(vnfd
: dict) -> str:
1209 """Find the sotware version in the VNFD descriptors
1212 vnfd (dict): Descriptor as a dictionary
1215 software-version (str)
1217 default_sw_version
= "1.0"
1218 if vnfd
.get("vnfd"):
1220 if vnfd
.get("software-version"):
1221 return vnfd
["software-version"]
1223 return default_sw_version
1226 def extract_policies(vnfd
: dict) -> dict:
1227 """Removes the policies from the VNFD descriptors
1230 vnfd (dict): Descriptor as a dictionary
1233 vnfd (dict): VNFD which does not include policies
1235 for df
in vnfd
.get("df", {}):
1236 for policy
in ["scaling-aspect", "healing-aspect"]:
1237 if (df
.get(policy
, {})):
1239 for vdu
in vnfd
.get("vdu", {}):
1240 for alarm_policy
in ["alarm", "monitoring-parameter"]:
1241 if (vdu
.get(alarm_policy
, {})):
1242 vdu
.pop(alarm_policy
)
1246 def extract_day12_primitives(vnfd
: dict) -> dict:
1247 """Removes the day12 primitives from the VNFD descriptors
1250 vnfd (dict): Descriptor as a dictionary
1255 for df_id
, df
in enumerate(vnfd
.get("df", {})):
1257 df
.get("lcm-operations-configuration", {})
1258 .get("operate-vnf-op-config", {})
1261 day12
= df
["lcm-operations-configuration"]["operate-vnf-op-config"].get(
1264 for config_id
, config
in enumerate(day12
):
1266 "initial-config-primitive",
1268 "terminate-config-primitive",
1270 config
.pop(key
, None)
1271 day12
[config_id
] = config
1272 df
["lcm-operations-configuration"]["operate-vnf-op-config"][
1275 vnfd
["df"][df_id
] = df
1278 def remove_modifiable_items(self
, vnfd
: dict) -> dict:
1279 """Removes the modifiable parts from the VNFD descriptors
1281 It calls different extract functions according to different update types
1282 to clear all the modifiable items from VNFD
1285 vnfd (dict): Descriptor as a dictionary
1288 vnfd (dict): Descriptor which does not include modifiable contents
1290 if vnfd
.get("vnfd"):
1292 vnfd
.pop("_admin", None)
1293 # If the other extractions need to be done from VNFD,
1294 # the new extract methods could be appended to below list.
1295 for extract_function
in [self
.extract_day12_primitives
, self
.extract_policies
]:
1296 vnfd_temp
= extract_function(vnfd
)
1300 def _validate_descriptor_changes(
1303 descriptor_file_name
: str,
1304 old_descriptor_directory
: str,
1305 new_descriptor_directory
: str,
1307 """Compares the old and new VNFD descriptors and validates the new descriptor.
1310 old_descriptor_directory (str): Directory of descriptor which is in-use
1311 new_descriptor_directory (str): Directory of descriptor which is proposed to update (new revision)
1317 EngineException: In case of error when there are unallowed changes
1320 # If VNFD does not exist in DB or it is not in use by any NS,
1321 # validation is not required.
1322 vnfd
= self
.db
.get_one("vnfds", {"_id": descriptor_id
})
1323 if not vnfd
or not detect_descriptor_usage(vnfd
, "vnfds", self
.db
):
1326 # Get the old and new descriptor contents in order to compare them.
1327 with self
.fs
.file_open(
1328 (old_descriptor_directory
.rstrip("/"), descriptor_file_name
), "r"
1329 ) as old_descriptor_file
:
1331 with self
.fs
.file_open(
1332 (new_descriptor_directory
.rstrip("/"), descriptor_file_name
), "r"
1333 ) as new_descriptor_file
:
1335 old_content
= yaml
.safe_load(old_descriptor_file
.read())
1336 new_content
= yaml
.safe_load(new_descriptor_file
.read())
1338 # If software version has changed, we do not need to validate
1339 # the differences anymore.
1340 if old_content
and new_content
:
1341 if self
.find_software_version(
1343 ) != self
.find_software_version(new_content
):
1346 disallowed_change
= DeepDiff(
1347 self
.remove_modifiable_items(old_content
),
1348 self
.remove_modifiable_items(new_content
),
1351 if disallowed_change
:
1352 changed_nodes
= functools
.reduce(
1353 lambda a
, b
: a
+ " , " + b
,
1356 for node
in disallowed_change
.get(
1362 raise EngineException(
1363 f
"Error in validating new descriptor: {changed_nodes} cannot be modified, "
1364 "there are disallowed changes in the vnf descriptor.",
1365 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1375 "VNF Descriptor could not be processed with error: {}.".format(e
)
1379 class NsdTopic(DescriptorTopic
):
1383 def __init__(self
, db
, fs
, msg
, auth
):
1384 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
1386 def pyangbind_validation(self
, item
, data
, force
=False):
1387 if self
._descriptor
_data
_is
_in
_old
_format
(data
):
1388 raise EngineException(
1389 "ERROR: Unsupported descriptor format. Please, use an ETSI SOL006 descriptor.",
1390 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1393 nsd_vnf_profiles
= data
.get("df", [{}])[0].get("vnf-profile", [])
1394 mynsd
= etsi_nfv_nsd
.etsi_nfv_nsd()
1395 pybindJSONDecoder
.load_ietf_json(
1396 {"nsd": {"nsd": [data
]}},
1403 out
= pybindJSON
.dumps(mynsd
, mode
="ietf")
1404 desc_out
= self
._remove
_envelop
(yaml
.safe_load(out
))
1405 desc_out
= self
._remove
_yang
_prefixes
_from
_descriptor
(desc_out
)
1406 if nsd_vnf_profiles
:
1407 desc_out
["df"][0]["vnf-profile"] = nsd_vnf_profiles
1409 except Exception as e
:
1410 raise EngineException(
1411 "Error in pyangbind validation: {}".format(str(e
)),
1412 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1416 def _descriptor_data_is_in_old_format(data
):
1417 return ("nsd-catalog" in data
) or ("nsd:nsd-catalog" in data
)
1420 def _remove_envelop(indata
=None):
1423 clean_indata
= indata
1425 if clean_indata
.get("nsd"):
1426 clean_indata
= clean_indata
["nsd"]
1427 elif clean_indata
.get("etsi-nfv-nsd:nsd"):
1428 clean_indata
= clean_indata
["etsi-nfv-nsd:nsd"]
1429 if clean_indata
.get("nsd"):
1431 not isinstance(clean_indata
["nsd"], list)
1432 or len(clean_indata
["nsd"]) != 1
1434 raise EngineException("'nsd' must be a list of only one element")
1435 clean_indata
= clean_indata
["nsd"][0]
1438 def _validate_input_new(self
, indata
, storage_params
, force
=False):
1439 indata
.pop("nsdOnboardingState", None)
1440 indata
.pop("nsdOperationalState", None)
1441 indata
.pop("nsdUsageState", None)
1443 indata
.pop("links", None)
1445 indata
= self
.pyangbind_validation("nsds", indata
, force
)
1446 # Cross references validation in the descriptor
1447 # TODO validata that if contains cloud-init-file or charms, have artifacts _admin.storage."pkg-dir" is not none
1448 for vld
in get_iterable(indata
.get("virtual-link-desc")):
1449 self
.validate_vld_mgmt_network_with_virtual_link_protocol_data(vld
, indata
)
1451 self
.validate_vnf_profiles_vnfd_id(indata
)
1456 def validate_vld_mgmt_network_with_virtual_link_protocol_data(vld
, indata
):
1457 if not vld
.get("mgmt-network"):
1459 vld_id
= vld
.get("id")
1460 for df
in get_iterable(indata
.get("df")):
1461 for vlp
in get_iterable(df
.get("virtual-link-profile")):
1462 if vld_id
and vld_id
== vlp
.get("virtual-link-desc-id"):
1463 if vlp
.get("virtual-link-protocol-data"):
1464 raise EngineException(
1465 "Error at df[id='{}']:virtual-link-profile[id='{}']:virtual-link-"
1466 "protocol-data You cannot set a virtual-link-protocol-data "
1467 "when mgmt-network is True".format(df
["id"], vlp
["id"]),
1468 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1472 def validate_vnf_profiles_vnfd_id(indata
):
1473 all_vnfd_ids
= set(get_iterable(indata
.get("vnfd-id")))
1474 for df
in get_iterable(indata
.get("df")):
1475 for vnf_profile
in get_iterable(df
.get("vnf-profile")):
1476 vnfd_id
= vnf_profile
.get("vnfd-id")
1477 if vnfd_id
and vnfd_id
not in all_vnfd_ids
:
1478 raise EngineException(
1479 "Error at df[id='{}']:vnf_profile[id='{}']:vnfd-id='{}' "
1480 "does not match any vnfd-id".format(
1481 df
["id"], vnf_profile
["id"], vnfd_id
1483 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1486 def _validate_input_edit(self
, indata
, content
, force
=False):
1487 # not needed to validate with pyangbind becuase it will be validated at check_conflict_on_edit
1489 indata looks as follows:
1490 - In the new case (conformant)
1491 {'nsdOperationalState': 'DISABLED', 'userDefinedData': {'id': 'string23',
1492 '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}}
1493 - In the old case (backwards-compatible)
1494 {'id': 'string23', '_id': 'c6ddc544-cede-4b94-9ebe-be07b298a3c1', 'name': 'simon46'}
1496 if "_admin" not in indata
:
1497 indata
["_admin"] = {}
1499 if "nsdOperationalState" in indata
:
1500 if indata
["nsdOperationalState"] in ("ENABLED", "DISABLED"):
1501 indata
["_admin"]["operationalState"] = indata
.pop("nsdOperationalState")
1503 raise EngineException(
1504 "State '{}' is not a valid operational state".format(
1505 indata
["nsdOperationalState"]
1507 http_code
=HTTPStatus
.BAD_REQUEST
,
1510 # In the case of user defined data, we need to put the data in the root of the object
1511 # to preserve current expected behaviour
1512 if "userDefinedData" in indata
:
1513 data
= indata
.pop("userDefinedData")
1514 if type(data
) == dict:
1515 indata
["_admin"]["userDefinedData"] = data
1517 raise EngineException(
1518 "userDefinedData should be an object, but is '{}' instead".format(
1521 http_code
=HTTPStatus
.BAD_REQUEST
,
1524 "operationalState" in indata
["_admin"]
1525 and content
["_admin"]["operationalState"]
1526 == indata
["_admin"]["operationalState"]
1528 raise EngineException(
1529 "nsdOperationalState already {}".format(
1530 content
["_admin"]["operationalState"]
1532 http_code
=HTTPStatus
.CONFLICT
,
1536 def _check_descriptor_dependencies(self
, session
, descriptor
):
1538 Check that the dependent descriptors exist on a new descriptor or edition. Also checks references to vnfd
1539 connection points are ok
1540 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1541 :param descriptor: descriptor to be inserted or edit
1542 :return: None or raises exception
1544 if session
["force"]:
1546 vnfds_index
= self
._get
_descriptor
_constituent
_vnfds
_index
(session
, descriptor
)
1548 # Cross references validation in the descriptor and vnfd connection point validation
1549 for df
in get_iterable(descriptor
.get("df")):
1550 self
.validate_df_vnf_profiles_constituent_connection_points(df
, vnfds_index
)
1552 def _get_descriptor_constituent_vnfds_index(self
, session
, descriptor
):
1554 if descriptor
.get("vnfd-id") and not session
["force"]:
1555 for vnfd_id
in get_iterable(descriptor
.get("vnfd-id")):
1556 query_filter
= self
._get
_project
_filter
(session
)
1557 query_filter
["id"] = vnfd_id
1558 vnf_list
= self
.db
.get_list("vnfds", query_filter
)
1560 raise EngineException(
1561 "Descriptor error at 'vnfd-id'='{}' references a non "
1562 "existing vnfd".format(vnfd_id
),
1563 http_code
=HTTPStatus
.CONFLICT
,
1565 vnfds_index
[vnfd_id
] = vnf_list
[0]
1569 def validate_df_vnf_profiles_constituent_connection_points(df
, vnfds_index
):
1570 for vnf_profile
in get_iterable(df
.get("vnf-profile")):
1571 vnfd
= vnfds_index
.get(vnf_profile
["vnfd-id"])
1572 all_vnfd_ext_cpds
= set()
1573 for ext_cpd
in get_iterable(vnfd
.get("ext-cpd")):
1574 if ext_cpd
.get("id"):
1575 all_vnfd_ext_cpds
.add(ext_cpd
.get("id"))
1577 for virtual_link
in get_iterable(
1578 vnf_profile
.get("virtual-link-connectivity")
1580 for vl_cpd
in get_iterable(virtual_link
.get("constituent-cpd-id")):
1581 vl_cpd_id
= vl_cpd
.get("constituent-cpd-id")
1582 if vl_cpd_id
and vl_cpd_id
not in all_vnfd_ext_cpds
:
1583 raise EngineException(
1584 "Error at df[id='{}']:vnf-profile[id='{}']:virtual-link-connectivity"
1585 "[virtual-link-profile-id='{}']:constituent-cpd-id='{}' references a "
1586 "non existing ext-cpd:id inside vnfd '{}'".format(
1589 virtual_link
["virtual-link-profile-id"],
1593 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1596 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
1597 final_content
= super().check_conflict_on_edit(
1598 session
, final_content
, edit_content
, _id
1601 self
._check
_descriptor
_dependencies
(session
, final_content
)
1603 return final_content
1605 def check_conflict_on_del(self
, session
, _id
, db_content
):
1607 Check that there is not any NSR that uses this NSD. Only NSRs belonging to this project are considered. Note
1608 that NSD can be public and be used by other projects.
1609 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1610 :param _id: nsd internal id
1611 :param db_content: The database content of the _id
1612 :return: None or raises EngineException with the conflict
1614 if session
["force"]:
1616 descriptor
= db_content
1617 descriptor_id
= descriptor
.get("id")
1618 if not descriptor_id
: # empty nsd not uploaded
1621 # check NSD used by NS
1622 _filter
= self
._get
_project
_filter
(session
)
1623 _filter
["nsd-id"] = _id
1624 if self
.db
.get_list("nsrs", _filter
):
1625 raise EngineException(
1626 "There is at least one NS instance using this descriptor",
1627 http_code
=HTTPStatus
.CONFLICT
,
1630 # check NSD referenced by NST
1631 del _filter
["nsd-id"]
1632 _filter
["netslice-subnet.ANYINDEX.nsd-ref"] = descriptor_id
1633 if self
.db
.get_list("nsts", _filter
):
1634 raise EngineException(
1635 "There is at least one NetSlice Template referencing this descriptor",
1636 http_code
=HTTPStatus
.CONFLICT
,
1639 def delete_extra(self
, session
, _id
, db_content
, not_send_msg
=None):
1641 Deletes associate file system storage (via super)
1642 Deletes associated vnfpkgops from database.
1643 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1644 :param _id: server internal id
1645 :param db_content: The database content of the descriptor
1647 :raises: FsException in case of error while deleting associated storage
1649 super().delete_extra(session
, _id
, db_content
, not_send_msg
)
1650 self
.db
.del_list(self
.topic
+"_revisions", { "_id": { "$regex": _id
}})
1653 def extract_day12_primitives(nsd
: dict) -> dict:
1654 """Removes the day12 primitives from the NSD descriptors
1657 nsd (dict): Descriptor as a dictionary
1660 nsd (dict): Cleared NSD
1662 if nsd
.get("ns-configuration"):
1665 "initial-config-primitive",
1666 "terminate-config-primitive",
1668 nsd
["ns-configuration"].pop(key
, None)
1671 def remove_modifiable_items(self
, nsd
: dict) -> dict:
1672 """Removes the modifiable parts from the VNFD descriptors
1674 It calls different extract functions according to different update types
1675 to clear all the modifiable items from NSD
1678 nsd (dict): Descriptor as a dictionary
1681 nsd (dict): Descriptor which does not include modifiable contents
1683 while isinstance(nsd
, dict) and nsd
.get("nsd"):
1685 if isinstance(nsd
, list):
1687 nsd
.pop("_admin", None)
1688 # If the more extractions need to be done from NSD,
1689 # the new extract methods could be appended to below list.
1690 for extract_function
in [self
.extract_day12_primitives
]:
1691 nsd_temp
= extract_function(nsd
)
1695 def _validate_descriptor_changes(
1698 descriptor_file_name
: str,
1699 old_descriptor_directory
: str,
1700 new_descriptor_directory
: str,
1702 """Compares the old and new NSD descriptors and validates the new descriptor
1705 old_descriptor_directory: Directory of descriptor which is in-use
1706 new_descriptor_directory: Directory of descriptor which is proposed to update (new revision)
1712 EngineException: In case of error if the changes are not allowed
1716 # If NSD does not exist in DB, or it is not in use by any NS,
1717 # validation is not required.
1718 nsd
= self
.db
.get_one("nsds", {"_id": descriptor_id
}, fail_on_empty
=False)
1719 if not nsd
or not detect_descriptor_usage(nsd
, "nsds", self
.db
):
1722 # Get the old and new descriptor contents in order to compare them.
1723 with self
.fs
.file_open(
1724 (old_descriptor_directory
.rstrip("/"), descriptor_file_name
), "r"
1725 ) as old_descriptor_file
:
1727 with self
.fs
.file_open(
1728 (new_descriptor_directory
.rstrip("/"), descriptor_file_name
), "r"
1729 ) as new_descriptor_file
:
1731 old_content
= yaml
.safe_load(old_descriptor_file
.read())
1732 new_content
= yaml
.safe_load(new_descriptor_file
.read())
1734 if old_content
and new_content
:
1735 disallowed_change
= DeepDiff(
1736 self
.remove_modifiable_items(old_content
),
1737 self
.remove_modifiable_items(new_content
),
1740 if disallowed_change
:
1741 changed_nodes
= functools
.reduce(
1742 lambda a
, b
: a
+ ", " + b
,
1745 for node
in disallowed_change
.get(
1751 raise EngineException(
1752 f
"Error in validating new descriptor: {changed_nodes} cannot be modified, "
1753 "there are disallowed changes in the ns descriptor. ",
1754 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1764 "NS Descriptor could not be processed with error: {}.".format(e
)
1767 def sol005_projection(self
, data
):
1768 data
["nsdOnboardingState"] = data
["_admin"]["onboardingState"]
1769 data
["nsdOperationalState"] = data
["_admin"]["operationalState"]
1770 data
["nsdUsageState"] = data
["_admin"]["usageState"]
1773 links
["self"] = {"href": "/nsd/v1/ns_descriptors/{}".format(data
["_id"])}
1774 links
["nsd_content"] = {
1775 "href": "/nsd/v1/ns_descriptors/{}/nsd_content".format(data
["_id"])
1777 data
["_links"] = links
1779 return super().sol005_projection(data
)
1782 class NstTopic(DescriptorTopic
):
1785 quota_name
= "slice_templates"
1787 def __init__(self
, db
, fs
, msg
, auth
):
1788 DescriptorTopic
.__init
__(self
, db
, fs
, msg
, auth
)
1790 def pyangbind_validation(self
, item
, data
, force
=False):
1793 pybindJSONDecoder
.load_ietf_json(
1801 out
= pybindJSON
.dumps(mynst
, mode
="ietf")
1802 desc_out
= self
._remove
_envelop
(yaml
.safe_load(out
))
1804 except Exception as e
:
1805 raise EngineException(
1806 "Error in pyangbind validation: {}".format(str(e
)),
1807 http_code
=HTTPStatus
.UNPROCESSABLE_ENTITY
,
1811 def _remove_envelop(indata
=None):
1814 clean_indata
= indata
1816 if clean_indata
.get("nst"):
1818 not isinstance(clean_indata
["nst"], list)
1819 or len(clean_indata
["nst"]) != 1
1821 raise EngineException("'nst' must be a list only one element")
1822 clean_indata
= clean_indata
["nst"][0]
1823 elif clean_indata
.get("nst:nst"):
1825 not isinstance(clean_indata
["nst:nst"], list)
1826 or len(clean_indata
["nst:nst"]) != 1
1828 raise EngineException("'nst:nst' must be a list only one element")
1829 clean_indata
= clean_indata
["nst:nst"][0]
1832 def _validate_input_new(self
, indata
, storage_params
, force
=False):
1833 indata
.pop("onboardingState", None)
1834 indata
.pop("operationalState", None)
1835 indata
.pop("usageState", None)
1836 indata
= self
.pyangbind_validation("nsts", indata
, force
)
1837 return indata
.copy()
1839 def _check_descriptor_dependencies(self
, session
, descriptor
):
1841 Check that the dependent descriptors exist on a new descriptor or edition
1842 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1843 :param descriptor: descriptor to be inserted or edit
1844 :return: None or raises exception
1846 if not descriptor
.get("netslice-subnet"):
1848 for nsd
in descriptor
["netslice-subnet"]:
1849 nsd_id
= nsd
["nsd-ref"]
1850 filter_q
= self
._get
_project
_filter
(session
)
1851 filter_q
["id"] = nsd_id
1852 if not self
.db
.get_list("nsds", filter_q
):
1853 raise EngineException(
1854 "Descriptor error at 'netslice-subnet':'nsd-ref'='{}' references a non "
1855 "existing nsd".format(nsd_id
),
1856 http_code
=HTTPStatus
.CONFLICT
,
1859 def check_conflict_on_edit(self
, session
, final_content
, edit_content
, _id
):
1860 final_content
= super().check_conflict_on_edit(
1861 session
, final_content
, edit_content
, _id
1864 self
._check
_descriptor
_dependencies
(session
, final_content
)
1865 return final_content
1867 def check_conflict_on_del(self
, session
, _id
, db_content
):
1869 Check that there is not any NSIR that uses this NST. Only NSIRs belonging to this project are considered. Note
1870 that NST can be public and be used by other projects.
1871 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1872 :param _id: nst internal id
1873 :param db_content: The database content of the _id.
1874 :return: None or raises EngineException with the conflict
1876 # TODO: Check this method
1877 if session
["force"]:
1879 # Get Network Slice Template from Database
1880 _filter
= self
._get
_project
_filter
(session
)
1881 _filter
["_admin.nst-id"] = _id
1882 if self
.db
.get_list("nsis", _filter
):
1883 raise EngineException(
1884 "there is at least one Netslice Instance using this descriptor",
1885 http_code
=HTTPStatus
.CONFLICT
,
1888 def sol005_projection(self
, data
):
1889 data
["onboardingState"] = data
["_admin"]["onboardingState"]
1890 data
["operationalState"] = data
["_admin"]["operationalState"]
1891 data
["usageState"] = data
["_admin"]["usageState"]
1894 links
["self"] = {"href": "/nst/v1/netslice_templates/{}".format(data
["_id"])}
1895 links
["nst"] = {"href": "/nst/v1/netslice_templates/{}/nst".format(data
["_id"])}
1896 data
["_links"] = links
1898 return super().sol005_projection(data
)
1901 class PduTopic(BaseTopic
):
1904 quota_name
= "pduds"
1905 schema_new
= pdu_new_schema
1906 schema_edit
= pdu_edit_schema
1908 def __init__(self
, db
, fs
, msg
, auth
):
1909 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
1912 def format_on_new(content
, project_id
=None, make_public
=False):
1913 BaseTopic
.format_on_new(content
, project_id
=project_id
, make_public
=make_public
)
1914 content
["_admin"]["onboardingState"] = "CREATED"
1915 content
["_admin"]["operationalState"] = "ENABLED"
1916 content
["_admin"]["usageState"] = "NOT_IN_USE"
1918 def check_conflict_on_del(self
, session
, _id
, db_content
):
1920 Check that there is not any vnfr that uses this PDU
1921 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1922 :param _id: pdu internal id
1923 :param db_content: The database content of the _id.
1924 :return: None or raises EngineException with the conflict
1926 if session
["force"]:
1929 _filter
= self
._get
_project
_filter
(session
)
1930 _filter
["vdur.pdu-id"] = _id
1931 if self
.db
.get_list("vnfrs", _filter
):
1932 raise EngineException(
1933 "There is at least one VNF instance using this PDU",
1934 http_code
=HTTPStatus
.CONFLICT
,
1938 class VnfPkgOpTopic(BaseTopic
):
1941 schema_new
= vnfpkgop_new_schema
1944 def __init__(self
, db
, fs
, msg
, auth
):
1945 BaseTopic
.__init
__(self
, db
, fs
, msg
, auth
)
1947 def edit(self
, session
, _id
, indata
=None, kwargs
=None, content
=None):
1948 raise EngineException(
1949 "Method 'edit' not allowed for topic '{}'".format(self
.topic
),
1950 HTTPStatus
.METHOD_NOT_ALLOWED
,
1953 def delete(self
, session
, _id
, dry_run
=False):
1954 raise EngineException(
1955 "Method 'delete' not allowed for topic '{}'".format(self
.topic
),
1956 HTTPStatus
.METHOD_NOT_ALLOWED
,
1959 def delete_list(self
, session
, filter_q
=None):
1960 raise EngineException(
1961 "Method 'delete_list' not allowed for topic '{}'".format(self
.topic
),
1962 HTTPStatus
.METHOD_NOT_ALLOWED
,
1965 def new(self
, rollback
, session
, indata
=None, kwargs
=None, headers
=None):
1967 Creates a new entry into database.
1968 :param rollback: list to append created items at database in case a rollback may to be done
1969 :param session: contains "username", "admin", "force", "public", "project_id", "set_project"
1970 :param indata: data to be inserted
1971 :param kwargs: used to override the indata descriptor
1972 :param headers: http request headers
1973 :return: _id, op_id:
1974 _id: identity of the inserted data.
1977 self
._update
_input
_with
_kwargs
(indata
, kwargs
)
1978 validate_input(indata
, self
.schema_new
)
1979 vnfpkg_id
= indata
["vnfPkgId"]
1980 filter_q
= BaseTopic
._get
_project
_filter
(session
)
1981 filter_q
["_id"] = vnfpkg_id
1982 vnfd
= self
.db
.get_one("vnfds", filter_q
)
1983 operation
= indata
["lcmOperationType"]
1984 kdu_name
= indata
["kdu_name"]
1985 for kdu
in vnfd
.get("kdu", []):
1986 if kdu
["name"] == kdu_name
:
1987 helm_chart
= kdu
.get("helm-chart")
1988 juju_bundle
= kdu
.get("juju-bundle")
1991 raise EngineException(
1992 "Not found vnfd[id='{}']:kdu[name='{}']".format(vnfpkg_id
, kdu_name
)
1995 indata
["helm-chart"] = helm_chart
1996 match
= fullmatch(r
"([^/]*)/([^/]*)", helm_chart
)
1997 repo_name
= match
.group(1) if match
else None
1999 indata
["juju-bundle"] = juju_bundle
2000 match
= fullmatch(r
"([^/]*)/([^/]*)", juju_bundle
)
2001 repo_name
= match
.group(1) if match
else None
2003 raise EngineException(
2004 "Found neither 'helm-chart' nor 'juju-bundle' in vnfd[id='{}']:kdu[name='{}']".format(
2010 filter_q
["name"] = repo_name
2011 repo
= self
.db
.get_one("k8srepos", filter_q
)
2012 k8srepo_id
= repo
.get("_id")
2013 k8srepo_url
= repo
.get("url")
2017 indata
["k8srepoId"] = k8srepo_id
2018 indata
["k8srepo_url"] = k8srepo_url
2019 vnfpkgop_id
= str(uuid4())
2022 "operationState": "PROCESSING",
2023 "vnfPkgId": vnfpkg_id
,
2024 "lcmOperationType": operation
,
2025 "isAutomaticInvocation": False,
2026 "isCancelPending": False,
2027 "operationParams": indata
,
2029 "self": "/osm/vnfpkgm/v1/vnfpkg_op_occs/" + vnfpkgop_id
,
2030 "vnfpkg": "/osm/vnfpkgm/v1/vnf_packages/" + vnfpkg_id
,
2034 vnfpkgop_desc
, session
["project_id"], make_public
=session
["public"]
2036 ctime
= vnfpkgop_desc
["_admin"]["created"]
2037 vnfpkgop_desc
["statusEnteredTime"] = ctime
2038 vnfpkgop_desc
["startTime"] = ctime
2039 self
.db
.create(self
.topic
, vnfpkgop_desc
)
2040 rollback
.append({"topic": self
.topic
, "_id": vnfpkgop_id
})
2041 self
.msg
.write(self
.topic_msg
, operation
, vnfpkgop_desc
)
2042 return vnfpkgop_id
, None