X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_nbi%2Fdescriptor_topics.py;h=ca10c424d3a559207685d0dba2f03486d4247a0c;hb=9af2a4785d3a77772fd205aa572cc6a64d4d1003;hp=ddf8ffc2f579e309cb71e778cbcaf5f29080a120;hpb=5be838c7196843b81ee79936886f61ac86fe691e;p=osm%2FNBI.git diff --git a/osm_nbi/descriptor_topics.py b/osm_nbi/descriptor_topics.py index ddf8ffc..ca10c42 100644 --- a/osm_nbi/descriptor_topics.py +++ b/osm_nbi/descriptor_topics.py @@ -37,7 +37,12 @@ from osm_nbi.validation import ( validate_input, vnfpkgop_new_schema, ) -from osm_nbi.base_topic import BaseTopic, EngineException, get_iterable +from osm_nbi.base_topic import ( + BaseTopic, + EngineException, + get_iterable, + detect_descriptor_usage, +) from osm_im import etsi_nfv_vnfd, etsi_nfv_nsd from osm_im.nst import nst as nst_im from pyangbind.lib.serialise import pybindJSONDecoder @@ -49,9 +54,11 @@ __author__ = "Alfonso Tierno " class DescriptorTopic(BaseTopic): def __init__(self, db, fs, msg, auth): - BaseTopic.__init__(self, db, fs, msg, auth) + def _validate_input_new(self, indata, storage_params, force=False): + return indata + def check_conflict_on_edit(self, session, final_content, edit_content, _id): final_content = super().check_conflict_on_edit( session, final_content, edit_content, _id @@ -117,7 +124,7 @@ class DescriptorTopic(BaseTopic): if self.db.get_one(self.topic, _filter, fail_on_empty=False): raise EngineException( "{} with id '{}' already exists for this project".format( - self.topic[:-1], final_content["id"] + (str(self.topic))[:-1], final_content["id"] ), HTTPStatus.CONFLICT, ) @@ -149,7 +156,6 @@ class DescriptorTopic(BaseTopic): self.fs.file_delete(_id + ":" + str(revision), ignore_non_exist=True) revision = revision - 1 - @staticmethod def get_one_by_id(db, session, topic, id): # find owned by this project @@ -213,10 +219,7 @@ class DescriptorTopic(BaseTopic): # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"]) - content = {"_admin": { - "userDefinedData": indata, - "revision": 0 - }} + content = {"_admin": {"userDefinedData": indata, "revision": 0}} self.format_on_new( content, session["project_id"], make_public=session["public"] @@ -250,10 +253,7 @@ class DescriptorTopic(BaseTopic): or "application/x-gzip" in content_type ): compressed = "gzip" - if ( - content_type - and "application/zip" in content_type - ): + if content_type and "application/zip" in content_type: compressed = "zip" filename = headers.get("Content-Filename") if not filename and compressed: @@ -403,14 +403,12 @@ class DescriptorTopic(BaseTopic): ) if ( - ( - zipfilename.endswith(".yaml") - or zipfilename.endswith(".json") - or zipfilename.endswith(".yml") - ) and ( - zipfilename.find("/") < 0 - or zipfilename.find("Definitions") >= 0 - ) + zipfilename.endswith(".yaml") + or zipfilename.endswith(".json") + or zipfilename.endswith(".yml") + ) and ( + zipfilename.find("/") < 0 + or zipfilename.find("Definitions") >= 0 ): storage["pkg-dir"] = "" if descriptor_file_name: @@ -439,7 +437,7 @@ class DescriptorTopic(BaseTopic): indata = json.load(content) else: error_text = "Invalid yaml format " - indata = yaml.load(content, Loader=yaml.SafeLoader) + indata = yaml.safe_load(content) # Need to close the file package here so it can be copied from the # revision to the current, unrevisioned record @@ -456,18 +454,23 @@ class DescriptorTopic(BaseTopic): if revision > 1: try: self._validate_descriptor_changes( + _id, descriptor_file_name, current_revision_path, - proposed_revision_path) + proposed_revision_path, + ) except Exception as e: - shutil.rmtree(self.fs.path + current_revision_path, ignore_errors=True) - shutil.rmtree(self.fs.path + proposed_revision_path, ignore_errors=True) + shutil.rmtree( + self.fs.path + current_revision_path, ignore_errors=True + ) + shutil.rmtree( + self.fs.path + proposed_revision_path, ignore_errors=True + ) # Only delete the new revision. We need to keep the original version in place # as it has not been changed. self.fs.file_delete(proposed_revision_path, ignore_non_exist=True) raise e - indata = self._remove_envelop(indata) # Override descriptor with query string kwargs @@ -487,7 +490,10 @@ class DescriptorTopic(BaseTopic): # Copy the revision to the active package name by its original id shutil.rmtree(self.fs.path + current_revision_path, ignore_errors=True) - os.rename(self.fs.path + proposed_revision_path, self.fs.path + current_revision_path) + os.rename( + self.fs.path + proposed_revision_path, + self.fs.path + current_revision_path, + ) self.fs.file_delete(current_revision_path, ignore_non_exist=True) self.fs.mkdir(current_revision_path) self.fs.reverse_sync(from_path=current_revision_path) @@ -694,11 +700,13 @@ class DescriptorTopic(BaseTopic): return indata - def _validate_descriptor_changes(self, + def _validate_descriptor_changes( + self, + descriptor_id, descriptor_file_name, old_descriptor_directory, - new_descriptor_directory): - # Todo: compare changes and throw a meaningful exception for the user to understand + new_descriptor_directory, + ): # Example: # raise EngineException( # "Error in validating new descriptor: cannot be modified", @@ -706,6 +714,7 @@ class DescriptorTopic(BaseTopic): # ) pass + class VnfdTopic(DescriptorTopic): topic = "vnfds" topic_msg = "vnfd" @@ -974,13 +983,9 @@ class VnfdTopic(DescriptorTopic): return False elif not storage_params.get("pkg-dir"): if self.fs.file_exists("{}_".format(storage_params["folder"]), "dir"): - f = "{}_/{}".format( - storage_params["folder"], folder - ) + f = "{}_/{}".format(storage_params["folder"], folder) else: - f = "{}/{}".format( - storage_params["folder"], folder - ) + f = "{}/{}".format(storage_params["folder"], folder) if file: return self.fs.file_exists("{}/{}".format(f, file), "file") else: @@ -1178,7 +1183,7 @@ class VnfdTopic(DescriptorTopic): """ super().delete_extra(session, _id, db_content, not_send_msg) self.db.del_list("vnfpkgops", {"vnfPkgId": _id}) - self.db.del_list(self.topic+"_revisions", {"_id": {"$regex": _id}}) + self.db.del_list(self.topic + "_revisions", {"_id": {"$regex": _id}}) def sol005_projection(self, data): data["onboardingState"] = data["_admin"]["onboardingState"] @@ -1225,11 +1230,11 @@ class VnfdTopic(DescriptorTopic): """ for df in vnfd.get("df", {}): for policy in ["scaling-aspect", "healing-aspect"]: - if (df.get(policy, {})): + if df.get(policy, {}): df.pop(policy) for vdu in vnfd.get("vdu", {}): for alarm_policy in ["alarm", "monitoring-parameter"]: - if (vdu.get(alarm_policy, {})): + if vdu.get(alarm_policy, {}): vdu.pop(alarm_policy) return vnfd @@ -1290,6 +1295,7 @@ class VnfdTopic(DescriptorTopic): def _validate_descriptor_changes( self, + descriptor_id: str, descriptor_file_name: str, old_descriptor_directory: str, new_descriptor_directory: str, @@ -1298,7 +1304,7 @@ class VnfdTopic(DescriptorTopic): Args: old_descriptor_directory (str): Directory of descriptor which is in-use - new_descriptor_directory (str): Directory of directory which is proposed to update (new revision) + new_descriptor_directory (str): Directory of descriptor which is proposed to update (new revision) Returns: None @@ -1307,27 +1313,35 @@ class VnfdTopic(DescriptorTopic): EngineException: In case of error when there are unallowed changes """ try: + # If VNFD does not exist in DB or it is not in use by any NS, + # validation is not required. + vnfd = self.db.get_one("vnfds", {"_id": descriptor_id}) + if not vnfd or not detect_descriptor_usage(vnfd, "vnfds", self.db): + return + + # Get the old and new descriptor contents in order to compare them. with self.fs.file_open( (old_descriptor_directory.rstrip("/"), descriptor_file_name), "r" ) as old_descriptor_file: with self.fs.file_open( - (new_descriptor_directory, descriptor_file_name), "r" + (new_descriptor_directory.rstrip("/"), descriptor_file_name), "r" ) as new_descriptor_file: - old_content = yaml.load( - old_descriptor_file.read(), Loader=yaml.SafeLoader - ) - new_content = yaml.load( - new_descriptor_file.read(), Loader=yaml.SafeLoader - ) + old_content = yaml.safe_load(old_descriptor_file.read()) + new_content = yaml.safe_load(new_descriptor_file.read()) + + # If software version has changed, we do not need to validate + # the differences anymore. if old_content and new_content: if self.find_software_version( old_content ) != self.find_software_version(new_content): return + disallowed_change = DeepDiff( self.remove_modifiable_items(old_content), self.remove_modifiable_items(new_content), ) + if disallowed_change: changed_nodes = functools.reduce( lambda a, b: a + " , " + b, @@ -1338,6 +1352,7 @@ class VnfdTopic(DescriptorTopic): ).keys() ], ) + raise EngineException( f"Error in validating new descriptor: {changed_nodes} cannot be modified, " "there are disallowed changes in the vnf descriptor.", @@ -1626,7 +1641,7 @@ class NsdTopic(DescriptorTopic): :raises: FsException in case of error while deleting associated storage """ super().delete_extra(session, _id, db_content, not_send_msg) - self.db.del_list(self.topic+"_revisions", { "_id": { "$regex": _id}}) + self.db.del_list(self.topic + "_revisions", {"_id": {"$regex": _id}}) @staticmethod def extract_day12_primitives(nsd: dict) -> dict: @@ -1673,6 +1688,7 @@ class NsdTopic(DescriptorTopic): def _validate_descriptor_changes( self, + descriptor_id: str, descriptor_file_name: str, old_descriptor_directory: str, new_descriptor_directory: str, @@ -1681,7 +1697,7 @@ class NsdTopic(DescriptorTopic): Args: old_descriptor_directory: Directory of descriptor which is in-use - new_descriptor_directory: Directory of directory which is proposed to update (new revision) + new_descriptor_directory: Directory of descriptor which is proposed to update (new revision) Returns: None @@ -1691,23 +1707,28 @@ class NsdTopic(DescriptorTopic): """ try: + # If NSD does not exist in DB, or it is not in use by any NS, + # validation is not required. + nsd = self.db.get_one("nsds", {"_id": descriptor_id}, fail_on_empty=False) + if not nsd or not detect_descriptor_usage(nsd, "nsds", self.db): + return + + # Get the old and new descriptor contents in order to compare them. with self.fs.file_open( - (old_descriptor_directory, descriptor_file_name), "r" + (old_descriptor_directory.rstrip("/"), descriptor_file_name), "r" ) as old_descriptor_file: with self.fs.file_open( (new_descriptor_directory.rstrip("/"), descriptor_file_name), "r" ) as new_descriptor_file: - old_content = yaml.load( - old_descriptor_file.read(), Loader=yaml.SafeLoader - ) - new_content = yaml.load( - new_descriptor_file.read(), Loader=yaml.SafeLoader - ) + old_content = yaml.safe_load(old_descriptor_file.read()) + new_content = yaml.safe_load(new_descriptor_file.read()) + if old_content and new_content: disallowed_change = DeepDiff( self.remove_modifiable_items(old_content), self.remove_modifiable_items(new_content), ) + if disallowed_change: changed_nodes = functools.reduce( lambda a, b: a + ", " + b, @@ -1718,6 +1739,7 @@ class NsdTopic(DescriptorTopic): ).keys() ], ) + raise EngineException( f"Error in validating new descriptor: {changed_nodes} cannot be modified, " "there are disallowed changes in the ns descriptor. ",