From 3d82ba2cc4c3ebb340a88400aaa4a8d3683806a1 Mon Sep 17 00:00:00 2001 From: beierlm Date: Tue, 19 Apr 2022 14:12:50 -0400 Subject: [PATCH] Increase Logging, fix directory delete When deleting a directory, we need to do a rexep search of all files that start with the directory that we want to delete, but not revisions that happen to share the same UUID. Changes the regexp for directory delete to include the trailing '/' so it limits the scope. Adds logging where needed Change-Id: Ia60618c17a863417224fadd9c055be658fb4ba4a Signed-off-by: beierlm --- osm_common/fsmongo.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py index a057e37..d923842 100644 --- a/osm_common/fsmongo.py +++ b/osm_common/fsmongo.py @@ -210,6 +210,7 @@ class FsMongo(FsBase): for directory in dir_cursor: if from_path and not directory.filename.startswith(from_path): continue + self.logger.debug("Making dir {}".format(self.path + directory.filename)) os.makedirs(self.path + directory.filename, exist_ok=True) valid_paths.append(self.path + directory.filename) @@ -229,6 +230,7 @@ class FsMongo(FsBase): link = b.read().decode("utf-8") try: + self.logger.debug("Sync removing {}".format(file_path)) os.remove(file_path) except OSError as e: if e.errno != errno.ENOENT: @@ -238,8 +240,10 @@ class FsMongo(FsBase): else: folder = os.path.dirname(file_path) if folder not in valid_paths: + self.logger.debug("Sync local directory {}".format(file_path)) os.makedirs(folder, exist_ok=True) with open(file_path, "wb+") as file_stream: + self.logger.debug("Sync download {}".format(file_path)) self.fs.download_to_stream(writing_file._id, file_stream) if "permissions" in writing_file.metadata: os.chmod(file_path, writing_file.metadata["permissions"]) @@ -413,7 +417,7 @@ class FsMongo(FsBase): metadata = {"type": file_type, "permissions": member.mode} member.name = member.name.rstrip("/") - self.logger.debug("Uploading {}".format(member.name)) + self.logger.debug("Uploading {}/{}".format(f, member.name)) self.fs.upload_from_stream( f + "/" + member.name, stream, metadata=metadata ) @@ -434,7 +438,7 @@ class FsMongo(FsBase): metadata = {"type": file_type} member.filename = member.filename.rstrip("/") - self.logger.debug("Uploading {}".format(member.filename)) + self.logger.debug("Uploading {}/{}".format(f, member.filename)) self.fs.upload_from_stream( f + "/" + member.filename, stream, metadata=metadata ) @@ -526,18 +530,27 @@ class FsMongo(FsBase): exception_file = next(file_cursor, None) if exception_file: + self.logger.error( + "Cannot delete duplicate file: {} and {}".format( + requested_file.filename, exception_file.filename + ) + ) raise FsException( "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) if requested_file.metadata["type"] == "dir": - dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}}) + dir_cursor = self.fs.find( + {"filename": {"$regex": "^{}/".format(f)}} + ) for tmp in dir_cursor: + self.logger.debug("Deleting {}".format(tmp.filename)) self.fs.delete(tmp._id) - else: - self.fs.delete(requested_file._id) + + self.logger.debug("Deleting {}".format(requested_file.filename)) + self.fs.delete(requested_file._id) if not found and not ignore_non_exist: raise FsException( "File {} does not exist".format(storage), @@ -623,11 +636,13 @@ class FsMongo(FsBase): metadata = {"type": file_type, "permissions": mask} + self.logger.debug("Sync upload {}".format(rel_filename)) self.fs.upload_from_stream(rel_filename, stream, metadata=metadata) # delete old files if remote_file: for file in remote_file: + self.logger.debug("Sync deleting {}".format(file.filename)) self.fs.delete(file._id) finally: if fh: -- 2.25.1