Enable parallel execution and output of tox env

[osm/common.git] / osm_common / fsmongo.py

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index 1ce5909..2e47039 100644 (file)
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -235,7 +235,9 @@ class FsMongo(FsBase):
                     if e.errno != errno.ENOENT:
                         # This is probably permission denied or worse
                         raise
-                os.symlink(link, file_path)
+                os.symlink(
+                    link, os.path.realpath(os.path.normpath(os.path.abspath(file_path)))
+                )
             else:
                 folder = os.path.dirname(file_path)
                 if folder not in valid_paths:
@@ -275,14 +277,11 @@ class FsMongo(FsBase):
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])
-            elif all(key in config.keys() for key in ["host", "port", "collection"]):
-                self.client = MongoClient(config["host"], config["port"])
-                self.fs = GridFSBucket(self.client[config["collection"]])
             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:
-                    raise FsException('Missing parameters: "uri" or "host" + "port"')
+                    raise FsException('Missing parameters: "uri"')
         except FsException:
             raise
         except Exception as e:  # TODO refine
@@ -573,9 +572,7 @@ class FsMongo(FsBase):
         self.__update_local_fs(from_path=from_path)
 
     def _update_mongo_fs(self, from_path):
-
         os_path = self.path + from_path
-
         # Obtain list of files and dirs in filesystem
         members = []
         for root, dirs, files in os.walk(os_path):
@@ -606,7 +603,9 @@ class FsMongo(FsBase):
 
             # convert to relative path
             rel_filename = os.path.relpath(member["filename"], self.path)
-            last_modified_date = datetime.datetime.fromtimestamp(
+            # get timestamp in UTC because mongo stores upload date in UTC:
+            # https://www.mongodb.com/docs/v4.0/tutorial/model-time-data/#overview
+            last_modified_date = datetime.datetime.utcfromtimestamp(
                 os.path.getmtime(member["filename"])
             )
 
@@ -618,7 +617,6 @@ class FsMongo(FsBase):
             remote_files.pop(rel_filename, None)
 
             if last_modified_date >= upload_date:
-
                 stream = None
                 fh = None
                 try:
@@ -649,13 +647,12 @@ class FsMongo(FsBase):
                     if stream:
                         stream.close()
 
-        # delete files that are not any more in local fs
+        # delete files that are not anymore in local fs
         for remote_file in remote_files.values():
             for file in remote_file:
                 self.fs.delete(file._id)
 
     def _get_mongo_files(self, from_path=None):
-
         file_dict = {}
         file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
         for file in file_cursor: