X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Ffsmongo.py;h=ff37c42203bf55f18a993720c55e8a3723c14762;hb=HEAD;hp=a057e378d6da14bbd5a382972d079fa1a15bc12d;hpb=40f57c87816f6e3e99f495b4e6f8b5abe8cce7a9;p=osm%2Fcommon.git diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py index a057e37..2e47039 100644 --- a/osm_common/fsmongo.py +++ b/osm_common/fsmongo.py @@ -15,17 +15,16 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: eduardo.sousa@canonical.com ## - +import datetime import errno from http import HTTPStatus from io import BytesIO, StringIO import logging import os -import datetime import tarfile import zipfile -from gridfs import GridFSBucket, errors +from gridfs import errors, GridFSBucket from osm_common.fsbase import FsBase, FsException from pymongo import MongoClient @@ -210,6 +209,7 @@ class FsMongo(FsBase): for directory in dir_cursor: if from_path and not directory.filename.startswith(from_path): continue + self.logger.debug("Making dir {}".format(self.path + directory.filename)) os.makedirs(self.path + directory.filename, exist_ok=True) valid_paths.append(self.path + directory.filename) @@ -229,17 +229,22 @@ class FsMongo(FsBase): link = b.read().decode("utf-8") try: + self.logger.debug("Sync removing {}".format(file_path)) os.remove(file_path) except OSError as e: if e.errno != errno.ENOENT: # This is probably permission denied or worse raise - os.symlink(link, file_path) + os.symlink( + link, os.path.realpath(os.path.normpath(os.path.abspath(file_path))) + ) else: folder = os.path.dirname(file_path) if folder not in valid_paths: + self.logger.debug("Sync local directory {}".format(file_path)) os.makedirs(folder, exist_ok=True) with open(file_path, "wb+") as file_stream: + self.logger.debug("Sync download {}".format(file_path)) self.fs.download_to_stream(writing_file._id, file_stream) if "permissions" in writing_file.metadata: os.chmod(file_path, writing_file.metadata["permissions"]) @@ -272,14 +277,11 @@ class FsMongo(FsBase): if all(key in config.keys() for key in ["uri", "collection"]): self.client = MongoClient(config["uri"]) self.fs = GridFSBucket(self.client[config["collection"]]) - elif all(key in config.keys() for key in ["host", "port", "collection"]): - self.client = MongoClient(config["host"], config["port"]) - self.fs = GridFSBucket(self.client[config["collection"]]) else: if "collection" not in config.keys(): raise FsException('Missing parameter "collection"') else: - raise FsException('Missing parameters: "uri" or "host" + "port"') + raise FsException('Missing parameters: "uri"') except FsException: raise except Exception as e: # TODO refine @@ -413,7 +415,7 @@ class FsMongo(FsBase): metadata = {"type": file_type, "permissions": member.mode} member.name = member.name.rstrip("/") - self.logger.debug("Uploading {}".format(member.name)) + self.logger.debug("Uploading {}/{}".format(f, member.name)) self.fs.upload_from_stream( f + "/" + member.name, stream, metadata=metadata ) @@ -434,7 +436,7 @@ class FsMongo(FsBase): metadata = {"type": file_type} member.filename = member.filename.rstrip("/") - self.logger.debug("Uploading {}".format(member.filename)) + self.logger.debug("Uploading {}/{}".format(f, member.filename)) self.fs.upload_from_stream( f + "/" + member.filename, stream, metadata=metadata ) @@ -526,18 +528,27 @@ class FsMongo(FsBase): exception_file = next(file_cursor, None) if exception_file: + self.logger.error( + "Cannot delete duplicate file: {} and {}".format( + requested_file.filename, exception_file.filename + ) + ) raise FsException( "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) if requested_file.metadata["type"] == "dir": - dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}}) + dir_cursor = self.fs.find( + {"filename": {"$regex": "^{}/".format(f)}} + ) for tmp in dir_cursor: + self.logger.debug("Deleting {}".format(tmp.filename)) self.fs.delete(tmp._id) - else: - self.fs.delete(requested_file._id) + + self.logger.debug("Deleting {}".format(requested_file.filename)) + self.fs.delete(requested_file._id) if not found and not ignore_non_exist: raise FsException( "File {} does not exist".format(storage), @@ -561,9 +572,7 @@ class FsMongo(FsBase): self.__update_local_fs(from_path=from_path) def _update_mongo_fs(self, from_path): - os_path = self.path + from_path - # Obtain list of files and dirs in filesystem members = [] for root, dirs, files in os.walk(os_path): @@ -594,7 +603,9 @@ class FsMongo(FsBase): # convert to relative path rel_filename = os.path.relpath(member["filename"], self.path) - last_modified_date = datetime.datetime.fromtimestamp( + # get timestamp in UTC because mongo stores upload date in UTC: + # https://www.mongodb.com/docs/v4.0/tutorial/model-time-data/#overview + last_modified_date = datetime.datetime.utcfromtimestamp( os.path.getmtime(member["filename"]) ) @@ -606,7 +617,6 @@ class FsMongo(FsBase): remote_files.pop(rel_filename, None) if last_modified_date >= upload_date: - stream = None fh = None try: @@ -623,11 +633,13 @@ class FsMongo(FsBase): metadata = {"type": file_type, "permissions": mask} + self.logger.debug("Sync upload {}".format(rel_filename)) self.fs.upload_from_stream(rel_filename, stream, metadata=metadata) # delete old files if remote_file: for file in remote_file: + self.logger.debug("Sync deleting {}".format(file.filename)) self.fs.delete(file._id) finally: if fh: @@ -635,13 +647,12 @@ class FsMongo(FsBase): if stream: stream.close() - # delete files that are not any more in local fs + # delete files that are not anymore in local fs for remote_file in remote_files.values(): for file in remote_file: self.fs.delete(file._id) def _get_mongo_files(self, from_path=None): - file_dict = {} file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)]) for file in file_cursor: