projects
/
osm
/
common.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
PyYAML bug workaround
[osm/common.git]
/
osm_common
/
fsmongo.py
diff --git
a/osm_common/fsmongo.py
b/osm_common/fsmongo.py
index
d923842
..
2e47039
100644
(file)
--- a/
osm_common/fsmongo.py
+++ b/
osm_common/fsmongo.py
@@
-15,17
+15,16
@@
# For those usages not covered by the Apache License, Version 2.0 please
# contact: eduardo.sousa@canonical.com
##
# For those usages not covered by the Apache License, Version 2.0 please
# contact: eduardo.sousa@canonical.com
##
-
+import datetime
import errno
from http import HTTPStatus
from io import BytesIO, StringIO
import logging
import os
import errno
from http import HTTPStatus
from io import BytesIO, StringIO
import logging
import os
-import datetime
import tarfile
import zipfile
import tarfile
import zipfile
-from gridfs import
GridFSBucket, errors
+from gridfs import
errors, GridFSBucket
from osm_common.fsbase import FsBase, FsException
from pymongo import MongoClient
from osm_common.fsbase import FsBase, FsException
from pymongo import MongoClient
@@
-236,7
+235,9
@@
class FsMongo(FsBase):
if e.errno != errno.ENOENT:
# This is probably permission denied or worse
raise
if e.errno != errno.ENOENT:
# This is probably permission denied or worse
raise
- os.symlink(link, file_path)
+ os.symlink(
+ link, os.path.realpath(os.path.normpath(os.path.abspath(file_path)))
+ )
else:
folder = os.path.dirname(file_path)
if folder not in valid_paths:
else:
folder = os.path.dirname(file_path)
if folder not in valid_paths:
@@
-276,14
+277,11
@@
class FsMongo(FsBase):
if all(key in config.keys() for key in ["uri", "collection"]):
self.client = MongoClient(config["uri"])
self.fs = GridFSBucket(self.client[config["collection"]])
if all(key in config.keys() for key in ["uri", "collection"]):
self.client = MongoClient(config["uri"])
self.fs = GridFSBucket(self.client[config["collection"]])
- elif all(key in config.keys() for key in ["host", "port", "collection"]):
- self.client = MongoClient(config["host"], config["port"])
- self.fs = GridFSBucket(self.client[config["collection"]])
else:
if "collection" not in config.keys():
raise FsException('Missing parameter "collection"')
else:
else:
if "collection" not in config.keys():
raise FsException('Missing parameter "collection"')
else:
- raise FsException('Missing parameters: "uri"
or "host" + "port"
')
+ raise FsException('Missing parameters: "uri"')
except FsException:
raise
except Exception as e: # TODO refine
except FsException:
raise
except Exception as e: # TODO refine
@@
-574,9
+572,7
@@
class FsMongo(FsBase):
self.__update_local_fs(from_path=from_path)
def _update_mongo_fs(self, from_path):
self.__update_local_fs(from_path=from_path)
def _update_mongo_fs(self, from_path):
-
os_path = self.path + from_path
os_path = self.path + from_path
-
# Obtain list of files and dirs in filesystem
members = []
for root, dirs, files in os.walk(os_path):
# Obtain list of files and dirs in filesystem
members = []
for root, dirs, files in os.walk(os_path):
@@
-607,7
+603,9
@@
class FsMongo(FsBase):
# convert to relative path
rel_filename = os.path.relpath(member["filename"], self.path)
# convert to relative path
rel_filename = os.path.relpath(member["filename"], self.path)
- last_modified_date = datetime.datetime.fromtimestamp(
+ # get timestamp in UTC because mongo stores upload date in UTC:
+ # https://www.mongodb.com/docs/v4.0/tutorial/model-time-data/#overview
+ last_modified_date = datetime.datetime.utcfromtimestamp(
os.path.getmtime(member["filename"])
)
os.path.getmtime(member["filename"])
)
@@
-619,7
+617,6
@@
class FsMongo(FsBase):
remote_files.pop(rel_filename, None)
if last_modified_date >= upload_date:
remote_files.pop(rel_filename, None)
if last_modified_date >= upload_date:
-
stream = None
fh = None
try:
stream = None
fh = None
try:
@@
-650,13
+647,12
@@
class FsMongo(FsBase):
if stream:
stream.close()
if stream:
stream.close()
- # delete files that are not any
more in local fs
+ # delete files that are not anymore in local fs
for remote_file in remote_files.values():
for file in remote_file:
self.fs.delete(file._id)
def _get_mongo_files(self, from_path=None):
for remote_file in remote_files.values():
for file in remote_file:
self.fs.delete(file._id)
def _get_mongo_files(self, from_path=None):
-
file_dict = {}
file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
for file in file_cursor:
file_dict = {}
file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
for file in file_cursor: