X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Ffsmongo.py;h=727410e14a5fb21c405db4d4e97663160203a971;hb=7da9795a4b73c72e81ac4880a9e9507e441aa90f;hp=a057e378d6da14bbd5a382972d079fa1a15bc12d;hpb=40f57c87816f6e3e99f495b4e6f8b5abe8cce7a9;p=osm%2Fcommon.git

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index a057e37..727410e 100644
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -15,17 +15,16 @@
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: eduardo.sousa@canonical.com
 ##
-
+import datetime
 import errno
 from http import HTTPStatus
 from io import BytesIO, StringIO
 import logging
 import os
-import datetime
 import tarfile
 import zipfile
 
-from gridfs import GridFSBucket, errors
+from gridfs import errors, GridFSBucket
 from osm_common.fsbase import FsBase, FsException
 from pymongo import MongoClient
 
@@ -210,6 +209,7 @@ class FsMongo(FsBase):
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue
+            self.logger.debug("Making dir {}".format(self.path + directory.filename))
             os.makedirs(self.path + directory.filename, exist_ok=True)
             valid_paths.append(self.path + directory.filename)
 
@@ -229,6 +229,7 @@ class FsMongo(FsBase):
                     link = b.read().decode("utf-8")
 
                 try:
+                    self.logger.debug("Sync removing {}".format(file_path))
                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
@@ -238,8 +239,10 @@ class FsMongo(FsBase):
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:
+                    self.logger.debug("Sync local directory {}".format(file_path))
                     os.makedirs(folder, exist_ok=True)
                 with open(file_path, "wb+") as file_stream:
+                    self.logger.debug("Sync download {}".format(file_path))
                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
@@ -272,14 +275,11 @@ class FsMongo(FsBase):
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])
-            elif all(key in config.keys() for key in ["host", "port", "collection"]):
-                self.client = MongoClient(config["host"], config["port"])
-                self.fs = GridFSBucket(self.client[config["collection"]])
             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:
-                    raise FsException('Missing parameters: "uri" or "host" + "port"')
+                    raise FsException('Missing parameters: "uri"')
         except FsException:
             raise
         except Exception as e:  # TODO refine
@@ -413,7 +413,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type, "permissions": member.mode}
                 member.name = member.name.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.name))
+                self.logger.debug("Uploading {}/{}".format(f, member.name))
                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )
@@ -434,7 +434,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type}
                 member.filename = member.filename.rstrip("/")
 
-                self.logger.debug("Uploading {}".format(member.filename))
+                self.logger.debug("Uploading {}/{}".format(f, member.filename))
                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )
@@ -526,18 +526,27 @@ class FsMongo(FsBase):
                 exception_file = next(file_cursor, None)
 
                 if exception_file:
+                    self.logger.error(
+                        "Cannot delete duplicate file: {} and {}".format(
+                            requested_file.filename, exception_file.filename
+                        )
+                    )
                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":
-                    dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
+                    dir_cursor = self.fs.find(
+                        {"filename": {"$regex": "^{}/".format(f)}}
+                    )
 
                     for tmp in dir_cursor:
+                        self.logger.debug("Deleting {}".format(tmp.filename))
                         self.fs.delete(tmp._id)
-                else:
-                    self.fs.delete(requested_file._id)
+
+                self.logger.debug("Deleting {}".format(requested_file.filename))
+                self.fs.delete(requested_file._id)
             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),
@@ -623,11 +632,13 @@ class FsMongo(FsBase):
 
                     metadata = {"type": file_type, "permissions": mask}
 
+                    self.logger.debug("Sync upload {}".format(rel_filename))
                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:
+                            self.logger.debug("Sync deleting {}".format(file.filename))
                             self.fs.delete(file._id)
                 finally:
                     if fh: