From: beierlm <mark.beierl@canonical.com>
Date: Tue, 19 Apr 2022 18:12:50 +0000 (-0400)
Subject: Increase Logging, fix directory delete
X-Git-Tag: v12.0.0rc1~3
X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=3d82ba2cc4c3ebb340a88400aaa4a8d3683806a1;p=osm%2Fcommon.git

Increase Logging, fix directory delete

When deleting a directory, we need to do a rexep search of all files that start
with the directory that we want to delete, but not revisions that happen to
share the same UUID.  Changes the regexp for directory delete to include the
trailing '/' so it limits the scope.

Adds logging where needed

Change-Id: Ia60618c17a863417224fadd9c055be658fb4ba4a
Signed-off-by: beierlm <mark.beierl@canonical.com>
---

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index a057e37..d923842 100644
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -210,6 +210,7 @@ class FsMongo(FsBase):
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue
+            self.logger.debug("Making dir {}".format(self.path + directory.filename))
             os.makedirs(self.path + directory.filename, exist_ok=True)
             valid_paths.append(self.path + directory.filename)
 
@@ -229,6 +230,7 @@ class FsMongo(FsBase):
                     link = b.read().decode("utf-8")
 
                 try:
+                    self.logger.debug("Sync removing {}".format(file_path))
                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
@@ -238,8 +240,10 @@ class FsMongo(FsBase):
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:
+                    self.logger.debug("Sync local directory {}".format(file_path))
                     os.makedirs(folder, exist_ok=True)
                 with open(file_path, "wb+") as file_stream:
+                    self.logger.debug("Sync download {}".format(file_path))
                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
@@ -413,7 +417,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type, "permissions": member.mode}
                 member.name = member.name.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.name))
+                self.logger.debug("Uploading {}/{}".format(f, member.name))
                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )
@@ -434,7 +438,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type}
                 member.filename = member.filename.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.filename))
+                self.logger.debug("Uploading {}/{}".format(f, member.filename))
                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )
@@ -526,18 +530,27 @@ class FsMongo(FsBase):
                 exception_file = next(file_cursor, None)
 
                 if exception_file:
+                    self.logger.error(
+                        "Cannot delete duplicate file: {} and {}".format(
+                            requested_file.filename, exception_file.filename
+                        )
+                    )
                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":
-                    dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
+                    dir_cursor = self.fs.find(
+                        {"filename": {"$regex": "^{}/".format(f)}}
+                    )
 
                     for tmp in dir_cursor:
+                        self.logger.debug("Deleting {}".format(tmp.filename))
                         self.fs.delete(tmp._id)
-                else:
-                    self.fs.delete(requested_file._id)
+
+                self.logger.debug("Deleting {}".format(requested_file.filename))
+                self.fs.delete(requested_file._id)
             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),
@@ -623,11 +636,13 @@ class FsMongo(FsBase):
 
                     metadata = {"type": file_type, "permissions": mask}
 
+                    self.logger.debug("Sync upload {}".format(rel_filename))
                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:
+                            self.logger.debug("Sync deleting {}".format(file.filename))
                             self.fs.delete(file._id)
                 finally:
                     if fh: