Code Cleanup and adding unit tests

[osm/common.git] / osm_common / fsmongo.py

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index a057e37..727410e 100644 (file)
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -15,17 +15,16 @@
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: eduardo.sousa@canonical.com
 ##
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: eduardo.sousa@canonical.com
 ##

-
+import datetime

 import errno
 from http import HTTPStatus
 from io import BytesIO, StringIO
 import logging
 import os
 import errno
 from http import HTTPStatus
 from io import BytesIO, StringIO
 import logging
 import os

-import datetime

 import tarfile
 import zipfile
 
 import tarfile
 import zipfile
 

-from gridfs import GridFSBucket, errors
+from gridfs import errors, GridFSBucket

 from osm_common.fsbase import FsBase, FsException
 from pymongo import MongoClient
 
 from osm_common.fsbase import FsBase, FsException
 from pymongo import MongoClient
 


@@ -210,6 +209,7 @@ class FsMongo(FsBase):
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue

+            self.logger.debug("Making dir {}".format(self.path + directory.filename))

             os.makedirs(self.path + directory.filename, exist_ok=True)
             valid_paths.append(self.path + directory.filename)
 
             os.makedirs(self.path + directory.filename, exist_ok=True)
             valid_paths.append(self.path + directory.filename)
 


@@ -229,6 +229,7 @@ class FsMongo(FsBase):
                     link = b.read().decode("utf-8")
 
                 try:
                     link = b.read().decode("utf-8")
 
                 try:

+                    self.logger.debug("Sync removing {}".format(file_path))

                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:


@@ -238,8 +239,10 @@ class FsMongo(FsBase):
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:

+                    self.logger.debug("Sync local directory {}".format(file_path))

                     os.makedirs(folder, exist_ok=True)
                 with open(file_path, "wb+") as file_stream:
                     os.makedirs(folder, exist_ok=True)
                 with open(file_path, "wb+") as file_stream:

+                    self.logger.debug("Sync download {}".format(file_path))

                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])


@@ -272,14 +275,11 @@ class FsMongo(FsBase):
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])

-            elif all(key in config.keys() for key in ["host", "port", "collection"]):
-                self.client = MongoClient(config["host"], config["port"])
-                self.fs = GridFSBucket(self.client[config["collection"]])

             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:
             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:

-                    raise FsException('Missing parameters: "uri" or "host" + "port"')
+                    raise FsException('Missing parameters: "uri"')

         except FsException:
             raise
         except Exception as e:  # TODO refine
         except FsException:
             raise
         except Exception as e:  # TODO refine


@@ -413,7 +413,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type, "permissions": member.mode}
                 member.name = member.name.rstrip("/")
 
                 metadata = {"type": file_type, "permissions": member.mode}
                 member.name = member.name.rstrip("/")
 

-                self.logger.debug("Uploading {}".format(member.name))
+                self.logger.debug("Uploading {}/{}".format(f, member.name))

                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )
                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )


@@ -434,7 +434,7 @@ class FsMongo(FsBase):
                 metadata = {"type": file_type}
                 member.filename = member.filename.rstrip("/")
 
                 metadata = {"type": file_type}
                 member.filename = member.filename.rstrip("/")
 

-                self.logger.debug("Uploading {}".format(member.filename))
+                self.logger.debug("Uploading {}/{}".format(f, member.filename))

                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )
                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )


@@ -526,18 +526,27 @@ class FsMongo(FsBase):
                 exception_file = next(file_cursor, None)
 
                 if exception_file:
                 exception_file = next(file_cursor, None)
 
                 if exception_file:

+                    self.logger.error(
+                        "Cannot delete duplicate file: {} and {}".format(
+                            requested_file.filename, exception_file.filename
+                        )
+                    )

                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":
                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":

-                    dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
+                    dir_cursor = self.fs.find(
+                        {"filename": {"$regex": "^{}/".format(f)}}
+                    )

 
                     for tmp in dir_cursor:
 
                     for tmp in dir_cursor:

+                        self.logger.debug("Deleting {}".format(tmp.filename))

                         self.fs.delete(tmp._id)
                         self.fs.delete(tmp._id)

-                else:
-                    self.fs.delete(requested_file._id)
+
+                self.logger.debug("Deleting {}".format(requested_file.filename))
+                self.fs.delete(requested_file._id)

             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),
             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),


@@ -623,11 +632,13 @@ class FsMongo(FsBase):
 
                     metadata = {"type": file_type, "permissions": mask}
 
 
                     metadata = {"type": file_type, "permissions": mask}
 

+                    self.logger.debug("Sync upload {}".format(rel_filename))

                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:
                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:

+                            self.logger.debug("Sync deleting {}".format(file.filename))

                             self.fs.delete(file._id)
                 finally:
                     if fh:
                             self.fs.delete(file._id)
                 finally:
                     if fh: