X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_nbi%2Fdescriptor_topics.py;h=590380a4b7aa99dac42b868d132754c3d0d43f01;hb=544a2ae8b0b950b55f29c3f0a223ffe4874285e5;hp=deae786f14ffdee736dd91fe792fa4eb4af29b91;hpb=c26740a54b2b1acf5663aab0b2c4753b9949504c;p=osm%2FNBI.git diff --git a/osm_nbi/descriptor_topics.py b/osm_nbi/descriptor_topics.py index deae786..590380a 100644 --- a/osm_nbi/descriptor_topics.py +++ b/osm_nbi/descriptor_topics.py @@ -17,8 +17,12 @@ import tarfile import yaml import json import copy +import os +import shutil +import functools # import logging +from deepdiff import DeepDiff from hashlib import md5 from osm_common.dbbase import DbException, deep_update_rfc7396 from http import HTTPStatus @@ -45,6 +49,7 @@ __author__ = "Alfonso Tierno " class DescriptorTopic(BaseTopic): def __init__(self, db, fs, msg, auth): + BaseTopic.__init__(self, db, fs, msg, auth) def check_conflict_on_edit(self, session, final_content, edit_content, _id): @@ -137,6 +142,13 @@ class DescriptorTopic(BaseTopic): """ self.fs.file_delete(_id, ignore_non_exist=True) self.fs.file_delete(_id + "_", ignore_non_exist=True) # remove temp folder + # Remove file revisions + if "revision" in db_content["_admin"]: + revision = db_content["_admin"]["revision"] + while revision > 0: + self.fs.file_delete(_id + ":" + str(revision), ignore_non_exist=True) + revision = revision - 1 + @staticmethod def get_one_by_id(db, session, topic, id): @@ -201,7 +213,11 @@ class DescriptorTopic(BaseTopic): # Avoid override in this case as the target is userDefinedData, but not vnfd,nsd descriptors # indata = DescriptorTopic._validate_input_new(self, indata, project_id=session["force"]) - content = {"_admin": {"userDefinedData": indata}} + content = {"_admin": { + "userDefinedData": indata, + "revision": 0 + }} + self.format_on_new( content, session["project_id"], make_public=session["public"] ) @@ -245,6 +261,10 @@ class DescriptorTopic(BaseTopic): elif not filename: filename = "package" + revision = 1 + if "revision" in current_desc["_admin"]: + revision = current_desc["_admin"]["revision"] + 1 + # TODO change to Content-Disposition filename https://tools.ietf.org/html/rfc6266 file_pkg = None error_text = "" @@ -262,23 +282,25 @@ class DescriptorTopic(BaseTopic): total = int(content_range[3]) else: start = 0 - temp_folder = ( - _id + "_" + # Rather than using a temp folder, we will store the package in a folder based on + # the current revision. + proposed_revision_path = ( + _id + ":" + str(revision) ) # all the content is upload here and if ok, it is rename from id_ to is folder if start: - if not self.fs.file_exists(temp_folder, "dir"): + if not self.fs.file_exists(proposed_revision_path, "dir"): raise EngineException( "invalid Transaction-Id header", HTTPStatus.NOT_FOUND ) else: - self.fs.file_delete(temp_folder, ignore_non_exist=True) - self.fs.mkdir(temp_folder) + self.fs.file_delete(proposed_revision_path, ignore_non_exist=True) + self.fs.mkdir(proposed_revision_path) storage = self.fs.get_params() storage["folder"] = _id - file_path = (temp_folder, filename) + file_path = (proposed_revision_path, filename) if self.fs.file_exists(file_path, "file"): file_size = self.fs.file_size(file_path) else: @@ -359,9 +381,9 @@ class DescriptorTopic(BaseTopic): ) storage["descriptor"] = descriptor_file_name storage["zipfile"] = filename - self.fs.file_extract(tar, temp_folder) + self.fs.file_extract(tar, proposed_revision_path) with self.fs.file_open( - (temp_folder, descriptor_file_name), "r" + (proposed_revision_path, descriptor_file_name), "r" ) as descriptor_file: content = descriptor_file.read() elif compressed == "zip": @@ -399,10 +421,10 @@ class DescriptorTopic(BaseTopic): ) storage["descriptor"] = descriptor_file_name storage["zipfile"] = filename - self.fs.file_extract(zipfile, temp_folder) + self.fs.file_extract(zipfile, proposed_revision_path) with self.fs.file_open( - (temp_folder, descriptor_file_name), "r" + (proposed_revision_path, descriptor_file_name), "r" ) as descriptor_file: content = descriptor_file.read() else: @@ -416,6 +438,40 @@ class DescriptorTopic(BaseTopic): error_text = "Invalid yaml format " indata = yaml.load(content, Loader=yaml.SafeLoader) + # Need to close the file package here so it can be copied from the + # revision to the current, unrevisioned record + if file_pkg: + file_pkg.close() + file_pkg = None + + # Fetch both the incoming, proposed revision and the original revision so we + # can call a validate method to compare them + current_revision_path = _id + "/" + self.fs.sync(from_path=current_revision_path) + self.fs.sync(from_path=proposed_revision_path) + + if revision > 1: + try: + self._validate_descriptor_changes( + descriptor_file_name, + current_revision_path, + proposed_revision_path) + except Exception as e: + shutil.rmtree(self.fs.path + current_revision_path, ignore_errors=True) + shutil.rmtree(self.fs.path + proposed_revision_path, ignore_errors=True) + # Only delete the new revision. We need to keep the original version in place + # as it has not been changed. + self.fs.file_delete(proposed_revision_path, ignore_non_exist=True) + raise e + + # Copy the revision to the active package name by its original id + shutil.rmtree(self.fs.path + current_revision_path, ignore_errors=True) + os.rename(self.fs.path + proposed_revision_path, self.fs.path + current_revision_path) + self.fs.file_delete(current_revision_path, ignore_non_exist=True) + self.fs.mkdir(current_revision_path) + self.fs.reverse_sync(from_path=current_revision_path) + shutil.rmtree(self.fs.path + _id) + current_desc["_admin"]["storage"] = storage current_desc["_admin"]["onboardingState"] = "ONBOARDED" current_desc["_admin"]["operationalState"] = "ENABLED" @@ -431,8 +487,13 @@ class DescriptorTopic(BaseTopic): session, current_desc, indata, _id=_id ) current_desc["_admin"]["modified"] = time() + current_desc["_admin"]["revision"] = revision self.db.replace(self.topic, _id, current_desc) - self.fs.dir_rename(temp_folder, _id) + + # Store a copy of the package as a point in time revision + revision_desc = dict(current_desc) + revision_desc["_id"] = _id + ":" + str(revision_desc["_admin"]["revision"]) + self.db.create(self.topic + "_revisions", revision_desc) indata["_id"] = _id self._send_msg("edited", indata) @@ -624,6 +685,17 @@ class DescriptorTopic(BaseTopic): return indata + def _validate_descriptor_changes(self, + descriptor_file_name, + old_descriptor_directory, + new_descriptor_directory): + # Todo: compare changes and throw a meaningful exception for the user to understand + # Example: + # raise EngineException( + # "Error in validating new descriptor: cannot be modified", + # http_code=HTTPStatus.UNPROCESSABLE_ENTITY, + # ) + pass class VnfdTopic(DescriptorTopic): topic = "vnfds" @@ -903,7 +975,6 @@ class VnfdTopic(DescriptorTopic): if file: return self.fs.file_exists("{}/{}".format(f, file), "file") else: - f = f+"/" if self.fs.file_exists(f, "dir"): if self.fs.dir_ls(f): return True @@ -1098,6 +1169,7 @@ class VnfdTopic(DescriptorTopic): """ super().delete_extra(session, _id, db_content, not_send_msg) self.db.del_list("vnfpkgops", {"vnfPkgId": _id}) + self.db.del_list(self.topic+"_revisions", {"_id": {"$regex": _id}}) def sol005_projection(self, data): data["onboardingState"] = data["_admin"]["onboardingState"] @@ -1114,6 +1186,158 @@ class VnfdTopic(DescriptorTopic): return super().sol005_projection(data) + @staticmethod + def find_software_version(vnfd: dict) -> str: + """Find the sotware version in the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + software-version (str) + """ + default_sw_version = "1.0" + if vnfd.get("vnfd"): + vnfd = vnfd["vnfd"] + if vnfd.get("software-version"): + return vnfd["software-version"] + else: + return default_sw_version + + @staticmethod + def extract_policies(vnfd: dict) -> dict: + """Removes the policies from the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict): VNFD which does not include policies + """ + # TODO: Extract the policy related parts from the VNFD + return vnfd + + @staticmethod + def extract_day12_primitives(vnfd: dict) -> dict: + """Removes the day12 primitives from the VNFD descriptors + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict) + """ + for df_id, df in enumerate(vnfd.get("df", {})): + if ( + df.get("lcm-operations-configuration", {}) + .get("operate-vnf-op-config", {}) + .get("day1-2") + ): + day12 = df["lcm-operations-configuration"]["operate-vnf-op-config"].get( + "day1-2" + ) + for config_id, config in enumerate(day12): + for key in [ + "initial-config-primitive", + "config-primitive", + "terminate-config-primitive", + ]: + config.pop(key, None) + day12[config_id] = config + df["lcm-operations-configuration"]["operate-vnf-op-config"][ + "day1-2" + ] = day12 + vnfd["df"][df_id] = df + return vnfd + + def remove_modifiable_items(self, vnfd: dict) -> dict: + """Removes the modifiable parts from the VNFD descriptors + + It calls different extract functions according to different update types + to clear all the modifiable items from VNFD + + Args: + vnfd (dict): Descriptor as a dictionary + + Returns: + vnfd (dict): Descriptor which does not include modifiable contents + """ + if vnfd.get("vnfd"): + vnfd = vnfd["vnfd"] + vnfd.pop("_admin", None) + # If the other extractions need to be done from VNFD, + # the new extract methods could be appended to below list. + for extract_function in [self.extract_day12_primitives, self.extract_policies]: + vnfd_temp = extract_function(vnfd) + vnfd = vnfd_temp + return vnfd + + def _validate_descriptor_changes( + self, + descriptor_file_name: str, + old_descriptor_directory: str, + new_descriptor_directory: str, + ): + """Compares the old and new VNFD descriptors and validates the new descriptor. + + Args: + old_descriptor_directory (str): Directory of descriptor which is in-use + new_descriptor_directory (str): Directory of directory which is proposed to update (new revision) + + Returns: + None + + Raises: + EngineException: In case of error when there are unallowed changes + """ + try: + with self.fs.file_open( + (old_descriptor_directory.rstrip("/"), descriptor_file_name), "r" + ) as old_descriptor_file: + with self.fs.file_open( + (new_descriptor_directory, descriptor_file_name), "r" + ) as new_descriptor_file: + old_content = yaml.load( + old_descriptor_file.read(), Loader=yaml.SafeLoader + ) + new_content = yaml.load( + new_descriptor_file.read(), Loader=yaml.SafeLoader + ) + if old_content and new_content: + if self.find_software_version( + old_content + ) != self.find_software_version(new_content): + return + disallowed_change = DeepDiff( + self.remove_modifiable_items(old_content), + self.remove_modifiable_items(new_content), + ) + if disallowed_change: + changed_nodes = functools.reduce( + lambda a, b: a + " , " + b, + [ + node.lstrip("root") + for node in disallowed_change.get( + "values_changed" + ).keys() + ], + ) + raise EngineException( + f"Error in validating new descriptor: {changed_nodes} cannot be modified, " + "there are disallowed changes in the vnf descriptor.", + http_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except ( + DbException, + AttributeError, + IndexError, + KeyError, + ValueError, + ) as e: + raise type(e)( + "VNF Descriptor could not be processed with error: {}.".format(e) + ) + class NsdTopic(DescriptorTopic): topic = "nsds" @@ -1375,6 +1599,125 @@ class NsdTopic(DescriptorTopic): http_code=HTTPStatus.CONFLICT, ) + def delete_extra(self, session, _id, db_content, not_send_msg=None): + """ + Deletes associate file system storage (via super) + Deletes associated vnfpkgops from database. + :param session: contains "username", "admin", "force", "public", "project_id", "set_project" + :param _id: server internal id + :param db_content: The database content of the descriptor + :return: None + :raises: FsException in case of error while deleting associated storage + """ + super().delete_extra(session, _id, db_content, not_send_msg) + self.db.del_list(self.topic+"_revisions", { "_id": { "$regex": _id}}) + + @staticmethod + def extract_day12_primitives(nsd: dict) -> dict: + """Removes the day12 primitives from the NSD descriptors + + Args: + nsd (dict): Descriptor as a dictionary + + Returns: + nsd (dict): Cleared NSD + """ + if nsd.get("ns-configuration"): + for key in [ + "config-primitive", + "initial-config-primitive", + "terminate-config-primitive", + ]: + nsd["ns-configuration"].pop(key, None) + return nsd + + def remove_modifiable_items(self, nsd: dict) -> dict: + """Removes the modifiable parts from the VNFD descriptors + + It calls different extract functions according to different update types + to clear all the modifiable items from NSD + + Args: + nsd (dict): Descriptor as a dictionary + + Returns: + nsd (dict): Descriptor which does not include modifiable contents + """ + while isinstance(nsd, dict) and nsd.get("nsd"): + nsd = nsd["nsd"] + if isinstance(nsd, list): + nsd = nsd[0] + nsd.pop("_admin", None) + # If the more extractions need to be done from NSD, + # the new extract methods could be appended to below list. + for extract_function in [self.extract_day12_primitives]: + nsd_temp = extract_function(nsd) + nsd = nsd_temp + return nsd + + def _validate_descriptor_changes( + self, + descriptor_file_name: str, + old_descriptor_directory: str, + new_descriptor_directory: str, + ): + """Compares the old and new NSD descriptors and validates the new descriptor + + Args: + old_descriptor_directory: Directory of descriptor which is in-use + new_descriptor_directory: Directory of directory which is proposed to update (new revision) + + Returns: + None + + Raises: + EngineException: In case of error if the changes are not allowed + """ + + try: + with self.fs.file_open( + (old_descriptor_directory, descriptor_file_name), "r" + ) as old_descriptor_file: + with self.fs.file_open( + (new_descriptor_directory.rstrip("/"), descriptor_file_name), "r" + ) as new_descriptor_file: + old_content = yaml.load( + old_descriptor_file.read(), Loader=yaml.SafeLoader + ) + new_content = yaml.load( + new_descriptor_file.read(), Loader=yaml.SafeLoader + ) + if old_content and new_content: + disallowed_change = DeepDiff( + self.remove_modifiable_items(old_content), + self.remove_modifiable_items(new_content), + ) + if disallowed_change: + changed_nodes = functools.reduce( + lambda a, b: a + ", " + b, + [ + node.lstrip("root") + for node in disallowed_change.get( + "values_changed" + ).keys() + ], + ) + raise EngineException( + f"Error in validating new descriptor: {changed_nodes} cannot be modified, " + "there are disallowed changes in the ns descriptor. ", + http_code=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + except ( + DbException, + AttributeError, + IndexError, + KeyError, + ValueError, + ) as e: + raise type(e)( + "NS Descriptor could not be processed with error: {}.".format(e) + ) + def sol005_projection(self, data): data["nsdOnboardingState"] = data["_admin"]["onboardingState"] data["nsdOperationalState"] = data["_admin"]["operationalState"]