Increase Logging, fix directory delete

[osm/common.git] / osm_common / fsmongo.py

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index a057e37..d923842 100644 (file)
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -210,6 +210,7 @@ class FsMongo(FsBase):
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue
+            self.logger.debug("Making dir {}".format(self.path + directory.filename))
             os.makedirs(self.path + directory.filename, exist_ok=True)
             valid_paths.append(self.path + directory.filename)
 
@@ -229,6 +230,7 @@ class FsMongo(FsBase):
                     link = b.read().decode("utf-8")
 
                 try:
+                    self.logger.debug("Sync removing {}".format(file_path))
                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
@@ -238,8 +240,10 @@ class FsMongo(FsBase):
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:
+                    self.logger.debug("Sync local directory {}".format(file_path))
                     os.makedirs(folder, exist_ok=True)
                 with open(file_path, "wb+") as file_stream:
+                    self.logger.debug("Sync download {}".format(file_path))
                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
@@ -413,7 +417,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type, "permissions": member.mode}
                 member.name = member.name.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.name))
+                self.logger.debug("Uploading {}/{}".format(f, member.name))
                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )
@@ -434,7 +438,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type}
                 member.filename = member.filename.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.filename))
+                self.logger.debug("Uploading {}/{}".format(f, member.filename))
                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )
@@ -526,18 +530,27 @@ class FsMongo(FsBase):
                 exception_file = next(file_cursor, None)
 
                 if exception_file:
+                    self.logger.error(
+                        "Cannot delete duplicate file: {} and {}".format(
+                            requested_file.filename, exception_file.filename
+                        )
+                    )
                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":
-                    dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
+                    dir_cursor = self.fs.find(
+                        {"filename": {"$regex": "^{}/".format(f)}}
+                    )
 
                     for tmp in dir_cursor:
+                        self.logger.debug("Deleting {}".format(tmp.filename))
                         self.fs.delete(tmp._id)
-                else:
-                    self.fs.delete(requested_file._id)
+
+                self.logger.debug("Deleting {}".format(requested_file.filename))
+                self.fs.delete(requested_file._id)
             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),
@@ -623,11 +636,13 @@ class FsMongo(FsBase):
 
                     metadata = {"type": file_type, "permissions": mask}
 
+                    self.logger.debug("Sync upload {}".format(rel_filename))
                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:
+                            self.logger.debug("Sync deleting {}".format(file.filename))
                             self.fs.delete(file._id)
                 finally:
                     if fh: