allow partial sync for fsmongo

[osm/common.git] / osm_common / fsmongo.py

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index cd7f7f0..07d4821 100644 (file)
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -22,7 +22,6 @@ from gridfs import GridFSBucket, errors
 import logging
 from http import HTTPStatus
 import os
-import stat
 from osm_common.fsbase import FsBase, FsException
 
 __author__ = "Eduardo Sousa <eduardo.sousa@canonical.com>"
@@ -35,6 +34,7 @@ class GridByteStream(BytesIO):
         self.filename = filename
         self.fs = fs
         self.mode = mode
+        self.file_type = "file"  # Set "file" as default file_type
 
         self.__initialize__()
 
@@ -110,6 +110,7 @@ class GridStringStream(StringIO):
         self.filename = filename
         self.fs = fs
         self.mode = mode
+        self.file_type = "file"  # Set "file" as default file_type
 
         self.__initialize__()
 
@@ -194,26 +195,31 @@ class FsMongo(FsBase):
         self.client = None
         self.fs = None
 
-    def __update_local_fs(self):
+    def __update_local_fs(self, from_path=None):
         dir_cursor = self.fs.find({"metadata.type": "dir"}, no_cursor_timeout=True)
 
         for directory in dir_cursor:
+            if from_path and not directory.filename.startswith(from_path):
+                continue
             os.makedirs(self.path + directory.filename, exist_ok=True)
 
-        file_cursor = self.fs.find({"metadata.type": {"$elemMatch": ["file", "sym"]}}, no_cursor_timeout=True)
+        file_cursor = self.fs.find({"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True)
 
         for writing_file in file_cursor:
+            if from_path and not writing_file.filename.startswith(from_path):
+                continue
             file_path = self.path + writing_file.filename
-            file_stream = open(file_path, 'wb+')
-            self.fs.download_to_stream(writing_file._id, file_stream)
-            file_stream.close()
-            if "permissions" in writing_file.metadata:
-                if writing_file.metadata["type"] == "sym":
-                    os.chmod(
-                        file_path, 
-                        writing_file.metadata["permissions"] | stat.S_IFLNK
-                    )
-                else:
+
+            if writing_file.metadata["type"] == "sym":
+                with BytesIO() as b:
+                    self.fs.download_to_stream(writing_file._id, b)
+                    b.seek(0)
+                    link = b.read().decode("utf-8")
+                os.symlink(link, file_path)
+            else:
+                with open(file_path, 'wb+') as file_stream:
+                    self.fs.download_to_stream(writing_file._id, file_stream)
+                if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
 
     def get_params(self):
@@ -347,6 +353,8 @@ class FsMongo(FsBase):
         for member in tar_object.getmembers():
             if member.isfile():
                 stream = tar_object.extractfile(member)
+            elif member.issym():
+                stream = BytesIO(member.linkname.encode("utf-8"))
             else:
                 stream = BytesIO()
 
@@ -448,8 +456,10 @@ class FsMongo(FsBase):
         except IOError as e:
             raise FsException("File {} cannot be deleted: {}".format(f, e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
 
-    def sync(self):
+    def sync(self, from_path=None):
         """
         Sync from FSMongo to local storage
+        :param from_path: if supplied, only copy content from this path, not all
+        :return: None
         """
-        self.__update_local_fs()
+        self.__update_local_fs(from_path=from_path)