From b07e4ef6e44bae6044be7420d982406ab0122275 Mon Sep 17 00:00:00 2001 From: tierno Date: Wed, 6 May 2020 14:22:48 +0000 Subject: [PATCH] allow partial sync for fsmongo Change-Id: I8c1d729bcff5a0fa2c58b25d57c5434ac0528668 Signed-off-by: tierno --- osm_common/fsbase.py | 2 +- osm_common/fslocal.py | 2 +- osm_common/fsmongo.py | 12 +++++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/osm_common/fsbase.py b/osm_common/fsbase.py index 7e69613..24eb994 100644 --- a/osm_common/fsbase.py +++ b/osm_common/fsbase.py @@ -83,5 +83,5 @@ class FsBase(object): def file_delete(self, storage, ignore_non_exist=False): raise FsException("Method 'file_delete' not implemented") - def sync(self): + def sync(self, from_path=None): raise FsException("Method 'sync' not implemented") diff --git a/osm_common/fslocal.py b/osm_common/fslocal.py index b90b25e..45ae828 100644 --- a/osm_common/fslocal.py +++ b/osm_common/fslocal.py @@ -180,5 +180,5 @@ class FsLocal(FsBase): except (IOError, PermissionError) as e: raise FsException("File {} cannot be deleted: {}".format(f, e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR) - def sync(self): + def sync(self, from_path=None): pass # Not needed in fslocal diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py index 6a96d44..07d4821 100644 --- a/osm_common/fsmongo.py +++ b/osm_common/fsmongo.py @@ -195,15 +195,19 @@ class FsMongo(FsBase): self.client = None self.fs = None - def __update_local_fs(self): + def __update_local_fs(self, from_path=None): dir_cursor = self.fs.find({"metadata.type": "dir"}, no_cursor_timeout=True) for directory in dir_cursor: + if from_path and not directory.filename.startswith(from_path): + continue os.makedirs(self.path + directory.filename, exist_ok=True) file_cursor = self.fs.find({"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True) for writing_file in file_cursor: + if from_path and not writing_file.filename.startswith(from_path): + continue file_path = self.path + writing_file.filename if writing_file.metadata["type"] == "sym": @@ -452,8 +456,10 @@ class FsMongo(FsBase): except IOError as e: raise FsException("File {} cannot be deleted: {}".format(f, e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR) - def sync(self): + def sync(self, from_path=None): """ Sync from FSMongo to local storage + :param from_path: if supplied, only copy content from this path, not all + :return: None """ - self.__update_local_fs() + self.__update_local_fs(from_path=from_path) -- 2.17.1