Reformat common to standardized format
Change-Id: I9ba54f3510e17de285e1e774d18ee5c7205818ad
Signed-off-by: garciadeblas <gerardo.garciadeblas@telefonica.com>
diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index f43dca6..ff37c42 100644
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -51,13 +51,17 @@
exception_file = next(cursor, None)
if exception_file:
- raise FsException("Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
if requested_file.metadata["type"] in ("file", "sym"):
grid_file = requested_file
self.file_type = requested_file.metadata["type"]
else:
- raise FsException("Type isn't file", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Type isn't file", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
if grid_file:
self._id = grid_file._id
@@ -74,29 +78,26 @@
if self._id:
self.fs.delete(self._id)
- cursor = self.fs.find({
- "filename": self.filename.split("/")[0],
- "metadata": {"type": "dir"}})
+ cursor = self.fs.find(
+ {"filename": self.filename.split("/")[0], "metadata": {"type": "dir"}}
+ )
parent_dir = next(cursor, None)
if not parent_dir:
parent_dir_name = self.filename.split("/")[0]
- self.filename = self.filename.replace(parent_dir_name, parent_dir_name[:-1], 1)
+ self.filename = self.filename.replace(
+ parent_dir_name, parent_dir_name[:-1], 1
+ )
self.seek(0, 0)
if self._id:
self.fs.upload_from_stream_with_id(
- self._id,
- self.filename,
- self,
- metadata={"type": self.file_type}
+ self._id, self.filename, self, metadata={"type": self.file_type}
)
else:
self.fs.upload_from_stream(
- self.filename,
- self,
- metadata={"type": self.file_type}
+ self.filename, self, metadata={"type": self.file_type}
)
super(GridByteStream, self).close()
@@ -127,13 +128,17 @@
exception_file = next(cursor, None)
if exception_file:
- raise FsException("Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
if requested_file.metadata["type"] in ("file", "dir"):
grid_file = requested_file
self.file_type = requested_file.metadata["type"]
else:
- raise FsException("File type isn't file", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "File type isn't file", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
if grid_file:
stream = BytesIO()
@@ -154,15 +159,17 @@
if self._id:
self.fs.delete(self._id)
- cursor = self.fs.find({
- "filename": self.filename.split("/")[0],
- "metadata": {"type": "dir"}})
+ cursor = self.fs.find(
+ {"filename": self.filename.split("/")[0], "metadata": {"type": "dir"}}
+ )
parent_dir = next(cursor, None)
if not parent_dir:
parent_dir_name = self.filename.split("/")[0]
- self.filename = self.filename.replace(parent_dir_name, parent_dir_name[:-1], 1)
+ self.filename = self.filename.replace(
+ parent_dir_name, parent_dir_name[:-1], 1
+ )
self.seek(0, 0)
stream = BytesIO()
@@ -170,16 +177,11 @@
stream.seek(0, 0)
if self._id:
self.fs.upload_from_stream_with_id(
- self._id,
- self.filename,
- stream,
- metadata={"type": self.file_type}
+ self._id, self.filename, stream, metadata={"type": self.file_type}
)
else:
self.fs.upload_from_stream(
- self.filename,
- stream,
- metadata={"type": self.file_type}
+ self.filename, stream, metadata={"type": self.file_type}
)
stream.close()
super(GridStringStream, self).close()
@@ -192,8 +194,7 @@
class FsMongo(FsBase):
-
- def __init__(self, logger_name='fs', lock=False):
+ def __init__(self, logger_name="fs", lock=False):
super().__init__(logger_name, lock)
self.path = None
self.client = None
@@ -207,7 +208,9 @@
continue
os.makedirs(self.path + directory.filename, exist_ok=True)
- file_cursor = self.fs.find({"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True)
+ file_cursor = self.fs.find(
+ {"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True
+ )
for writing_file in file_cursor:
if from_path and not writing_file.filename.startswith(from_path):
@@ -228,7 +231,7 @@
raise
os.symlink(link, file_path)
else:
- with open(file_path, 'wb+') as file_stream:
+ with open(file_path, "wb+") as file_stream:
self.fs.download_to_stream(writing_file._id, file_stream)
if "permissions" in writing_file.metadata:
os.chmod(file_path, writing_file.metadata["permissions"])
@@ -243,15 +246,21 @@
if "path" in config:
self.path = config["path"]
else:
- raise FsException("Missing parameter \"path\"")
+ raise FsException('Missing parameter "path"')
if not self.path.endswith("/"):
self.path += "/"
if not os.path.exists(self.path):
- raise FsException("Invalid configuration param at '[storage]': path '{}' does not exist".format(
- config["path"]))
+ raise FsException(
+ "Invalid configuration param at '[storage]': path '{}' does not exist".format(
+ config["path"]
+ )
+ )
elif not os.access(self.path, os.W_OK):
- raise FsException("Invalid configuration param at '[storage]': path '{}' is not writable".format(
- config["path"]))
+ raise FsException(
+ "Invalid configuration param at '[storage]': path '{}' is not writable".format(
+ config["path"]
+ )
+ )
if all(key in config.keys() for key in ["uri", "collection"]):
self.client = MongoClient(config["uri"])
self.fs = GridFSBucket(self.client[config["collection"]])
@@ -260,9 +269,9 @@
self.fs = GridFSBucket(self.client[config["collection"]])
else:
if "collection" not in config.keys():
- raise FsException("Missing parameter \"collection\"")
+ raise FsException('Missing parameter "collection"')
else:
- raise FsException("Missing parameters: \"uri\" or \"host\" + \"port\"")
+ raise FsException('Missing parameters: "uri" or "host" + "port"')
except FsException:
raise
except Exception as e: # TODO refine
@@ -278,8 +287,7 @@
:return: None or raises an exception
"""
try:
- self.fs.upload_from_stream(
- folder, BytesIO(), metadata={"type": "dir"})
+ self.fs.upload_from_stream(folder, BytesIO(), metadata={"type": "dir"})
except errors.FileExists: # make it idempotent
pass
except Exception as e:
@@ -294,15 +302,15 @@
"""
try:
dst_cursor = self.fs.find(
- {"filename": {"$regex": "^{}(/|$)".format(dst)}},
- no_cursor_timeout=True)
+ {"filename": {"$regex": "^{}(/|$)".format(dst)}}, no_cursor_timeout=True
+ )
for dst_file in dst_cursor:
self.fs.delete(dst_file._id)
src_cursor = self.fs.find(
- {"filename": {"$regex": "^{}(/|$)".format(src)}},
- no_cursor_timeout=True)
+ {"filename": {"$regex": "^{}(/|$)".format(src)}}, no_cursor_timeout=True
+ )
for src_file in src_cursor:
self.fs.rename(src_file._id, src_file.filename.replace(src, dst, 1))
@@ -324,7 +332,9 @@
exception_file = next(cursor, None)
if exception_file:
- raise FsException("Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
# if no special mode is required just check it does exists
if not mode:
@@ -352,7 +362,9 @@
exception_file = next(cursor, None)
if exception_file:
- raise FsException("Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
+ )
return requested_file.length
@@ -380,16 +392,9 @@
else:
file_type = "dir"
- metadata = {
- "type": file_type,
- "permissions": member.mode
- }
+ metadata = {"type": file_type, "permissions": member.mode}
- self.fs.upload_from_stream(
- f + "/" + member.name,
- stream,
- metadata=metadata
- )
+ self.fs.upload_from_stream(f + "/" + member.name, stream, metadata=metadata)
stream.close()
@@ -408,9 +413,13 @@
else:
return GridStringStream(f, self.fs, mode)
except errors.NoFile:
- raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+ raise FsException(
+ "File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND
+ )
except IOError:
- raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
+ raise FsException(
+ "File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST
+ )
def dir_ls(self, storage):
"""
@@ -427,18 +436,28 @@
exception_dir = next(dir_cursor, None)
if exception_dir:
- raise FsException("Multiple directories found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple directories found",
+ http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+ )
if requested_dir.metadata["type"] != "dir":
- raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
+ raise FsException(
+ "File {} does not exist".format(f),
+ http_code=HTTPStatus.NOT_FOUND,
+ )
- files_cursor = self.fs.find({"filename": {"$regex": "^{}/([^/])*".format(f)}})
+ files_cursor = self.fs.find(
+ {"filename": {"$regex": "^{}/([^/])*".format(f)}}
+ )
for children_file in files_cursor:
- files += [children_file.filename.replace(f + '/', '', 1)]
+ files += [children_file.filename.replace(f + "/", "", 1)]
return files
except IOError:
- raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
+ raise FsException(
+ "File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST
+ )
def file_delete(self, storage, ignore_non_exist=False):
"""
@@ -457,7 +476,10 @@
exception_file = next(file_cursor, None)
if exception_file:
- raise FsException("Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "Multiple files found",
+ http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+ )
if requested_file.metadata["type"] == "dir":
dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
@@ -467,9 +489,15 @@
else:
self.fs.delete(requested_file._id)
if not found and not ignore_non_exist:
- raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.NOT_FOUND)
+ raise FsException(
+ "File {} does not exist".format(storage),
+ http_code=HTTPStatus.NOT_FOUND,
+ )
except IOError as e:
- raise FsException("File {} cannot be deleted: {}".format(f, e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
+ raise FsException(
+ "File {} cannot be deleted: {}".format(f, e),
+ http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+ )
def sync(self, from_path=None):
"""
@@ -490,10 +518,7 @@
members = []
for root, dirs, files in os.walk(os_path):
for folder in dirs:
- member = {
- "filename": os.path.join(root, folder),
- "type": "dir"
- }
+ member = {"filename": os.path.join(root, folder), "type": "dir"}
members.append(member)
for file in files:
filename = os.path.join(root, file)
@@ -501,10 +526,7 @@
file_type = "sym"
else:
file_type = "file"
- member = {
- "filename": os.path.join(root, file),
- "type": file_type
- }
+ member = {"filename": os.path.join(root, file), "type": file_type}
members.append(member)
# Obtain files in mongo dict
@@ -520,10 +542,14 @@
# convert to relative path
rel_filename = os.path.relpath(member["filename"], self.path)
- last_modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(member["filename"]))
+ last_modified_date = datetime.datetime.fromtimestamp(
+ os.path.getmtime(member["filename"])
+ )
remote_file = remote_files.get(rel_filename)
- upload_date = remote_file[0].uploadDate if remote_file else datetime.datetime.min
+ upload_date = (
+ remote_file[0].uploadDate if remote_file else datetime.datetime.min
+ )
# remove processed files from dict
remote_files.pop(rel_filename, None)
@@ -536,21 +562,16 @@
if file_type == "dir":
stream = BytesIO()
elif file_type == "sym":
- stream = BytesIO(os.readlink(member["filename"]).encode("utf-8"))
+ stream = BytesIO(
+ os.readlink(member["filename"]).encode("utf-8")
+ )
else:
fh = open(member["filename"], "rb")
stream = BytesIO(fh.read())
- metadata = {
- "type": file_type,
- "permissions": mask
- }
+ metadata = {"type": file_type, "permissions": mask}
- self.fs.upload_from_stream(
- rel_filename,
- stream,
- metadata=metadata
- )
+ self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
# delete old files
if remote_file:
@@ -570,7 +591,7 @@
def _get_mongo_files(self, from_path=None):
file_dict = {}
- file_cursor = self.fs.find(no_cursor_timeout=True, sort=[('uploadDate', -1)])
+ file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
for file in file_cursor:
if from_path and not file.filename.startswith(from_path):
continue