Disable the check of release notes

[osm/common.git] / osm_common / fsmongo.py

diff --git a/osm_common/fsmongo.py b/osm_common/fsmongo.py
index 7fb071a..2e47039 100644 (file)
--- a/osm_common/fsmongo.py
+++ b/osm_common/fsmongo.py
@@ -15,17 +15,16 @@
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: eduardo.sousa@canonical.com
 ##
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: eduardo.sousa@canonical.com
 ##

-
+import datetime

 import errno
 from http import HTTPStatus
 from io import BytesIO, StringIO
 import logging
 import os
 import errno
 from http import HTTPStatus
 from io import BytesIO, StringIO
 import logging
 import os

-import datetime

 import tarfile
 import zipfile
 
 import tarfile
 import zipfile
 

-from gridfs import GridFSBucket, errors
+from gridfs import errors, GridFSBucket

 from osm_common.fsbase import FsBase, FsException
 from pymongo import MongoClient
 
 from osm_common.fsbase import FsBase, FsException
 from pymongo import MongoClient
 


@@ -205,10 +204,14 @@ class FsMongo(FsBase):
     def __update_local_fs(self, from_path=None):
         dir_cursor = self.fs.find({"metadata.type": "dir"}, no_cursor_timeout=True)
 
     def __update_local_fs(self, from_path=None):
         dir_cursor = self.fs.find({"metadata.type": "dir"}, no_cursor_timeout=True)
 

+        valid_paths = []
+

         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue
         for directory in dir_cursor:
             if from_path and not directory.filename.startswith(from_path):
                 continue

+            self.logger.debug("Making dir {}".format(self.path + directory.filename))

             os.makedirs(self.path + directory.filename, exist_ok=True)
             os.makedirs(self.path + directory.filename, exist_ok=True)

+            valid_paths.append(self.path + directory.filename)

 
         file_cursor = self.fs.find(
             {"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True
 
         file_cursor = self.fs.find(
             {"metadata.type": {"$in": ["file", "sym"]}}, no_cursor_timeout=True


@@ -226,14 +229,22 @@ class FsMongo(FsBase):
                     link = b.read().decode("utf-8")
 
                 try:
                     link = b.read().decode("utf-8")
 
                 try:

+                    self.logger.debug("Sync removing {}".format(file_path))

                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         # This is probably permission denied or worse
                         raise
                     os.remove(file_path)
                 except OSError as e:
                     if e.errno != errno.ENOENT:
                         # This is probably permission denied or worse
                         raise

-                os.symlink(link, file_path)
+                os.symlink(
+                    link, os.path.realpath(os.path.normpath(os.path.abspath(file_path)))
+                )

             else:
             else:

+                folder = os.path.dirname(file_path)
+                if folder not in valid_paths:
+                    self.logger.debug("Sync local directory {}".format(file_path))
+                    os.makedirs(folder, exist_ok=True)

                 with open(file_path, "wb+") as file_stream:
                 with open(file_path, "wb+") as file_stream:

+                    self.logger.debug("Sync download {}".format(file_path))

                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])
                     self.fs.download_to_stream(writing_file._id, file_stream)
                 if "permissions" in writing_file.metadata:
                     os.chmod(file_path, writing_file.metadata["permissions"])


@@ -266,14 +277,11 @@ class FsMongo(FsBase):
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])
             if all(key in config.keys() for key in ["uri", "collection"]):
                 self.client = MongoClient(config["uri"])
                 self.fs = GridFSBucket(self.client[config["collection"]])

-            elif all(key in config.keys() for key in ["host", "port", "collection"]):
-                self.client = MongoClient(config["host"], config["port"])
-                self.fs = GridFSBucket(self.client[config["collection"]])

             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:
             else:
                 if "collection" not in config.keys():
                     raise FsException('Missing parameter "collection"')
                 else:

-                    raise FsException('Missing parameters: "uri" or "host" + "port"')
+                    raise FsException('Missing parameters: "uri"')

         except FsException:
             raise
         except Exception as e:  # TODO refine
         except FsException:
             raise
         except Exception as e:  # TODO refine


@@ -288,6 +296,7 @@ class FsMongo(FsBase):
         :param folder:
         :return: None or raises an exception
         """
         :param folder:
         :return: None or raises an exception
         """

+        folder = folder.rstrip("/")

         try:
             self.fs.upload_from_stream(folder, BytesIO(), metadata={"type": "dir"})
         except errors.FileExists:  # make it idempotent
         try:
             self.fs.upload_from_stream(folder, BytesIO(), metadata={"type": "dir"})
         except errors.FileExists:  # make it idempotent


@@ -302,6 +311,9 @@ class FsMongo(FsBase):
         :param dst: destination directory
         :return: None or raises and exception
         """
         :param dst: destination directory
         :return: None or raises and exception
         """

+        dst = dst.rstrip("/")
+        src = src.rstrip("/")
+

         try:
             dst_cursor = self.fs.find(
                 {"filename": {"$regex": "^{}(/|$)".format(dst)}}, no_cursor_timeout=True
         try:
             dst_cursor = self.fs.find(
                 {"filename": {"$regex": "^{}(/|$)".format(dst)}}, no_cursor_timeout=True


@@ -327,6 +339,7 @@ class FsMongo(FsBase):
         :return: True, False
         """
         f = storage if isinstance(storage, str) else "/".join(storage)
         :return: True, False
         """
         f = storage if isinstance(storage, str) else "/".join(storage)

+        f = f.rstrip("/")

 
         cursor = self.fs.find({"filename": f})
 
 
         cursor = self.fs.find({"filename": f})
 


@@ -338,7 +351,7 @@ class FsMongo(FsBase):
                     "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
                 )
 
                     "Multiple files found", http_code=HTTPStatus.INTERNAL_SERVER_ERROR
                 )
 

-            print(requested_file.metadata)
+            self.logger.debug("Entry {} metadata {}".format(f, requested_file.metadata))

 
             # if no special mode is required just check it does exists
             if not mode:
 
             # if no special mode is required just check it does exists
             if not mode:


@@ -359,6 +372,7 @@ class FsMongo(FsBase):
         :return: file size
         """
         f = storage if isinstance(storage, str) else "/".join(storage)
         :return: file size
         """
         f = storage if isinstance(storage, str) else "/".join(storage)

+        f = f.rstrip("/")

 
         cursor = self.fs.find({"filename": f})
 
 
         cursor = self.fs.find({"filename": f})
 


@@ -380,6 +394,7 @@ class FsMongo(FsBase):
         :return: None
         """
         f = path if isinstance(path, str) else "/".join(path)
         :return: None
         """
         f = path if isinstance(path, str) else "/".join(path)

+        f = f.rstrip("/")

 
         if type(compressed_object) is tarfile.TarFile:
             for member in compressed_object.getmembers():
 
         if type(compressed_object) is tarfile.TarFile:
             for member in compressed_object.getmembers():


@@ -398,7 +413,9 @@ class FsMongo(FsBase):
                     file_type = "dir"
 
                 metadata = {"type": file_type, "permissions": member.mode}
                     file_type = "dir"
 
                 metadata = {"type": file_type, "permissions": member.mode}

+                member.name = member.name.rstrip("/")

 
 

+                self.logger.debug("Uploading {}/{}".format(f, member.name))

                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )
                 self.fs.upload_from_stream(
                     f + "/" + member.name, stream, metadata=metadata
                 )


@@ -417,9 +434,9 @@ class FsMongo(FsBase):
                     file_type = "file"
 
                 metadata = {"type": file_type}
                     file_type = "file"
 
                 metadata = {"type": file_type}

+                member.filename = member.filename.rstrip("/")

 
 

-                print("Now uploading...")
-                print(f + "/" + member.filename)
+                self.logger.debug("Uploading {}/{}".format(f, member.filename))

                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )
                 self.fs.upload_from_stream(
                     f + "/" + member.filename, stream, metadata=metadata
                 )


@@ -436,6 +453,7 @@ class FsMongo(FsBase):
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)

+            f = f.rstrip("/")

 
             if "b" in mode:
                 return GridByteStream(f, self.fs, mode)
 
             if "b" in mode:
                 return GridByteStream(f, self.fs, mode)


@@ -458,6 +476,7 @@ class FsMongo(FsBase):
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)

+            f = f.rstrip("/")

 
             files = []
             dir_cursor = self.fs.find({"filename": f})
 
             files = []
             dir_cursor = self.fs.find({"filename": f})


@@ -500,6 +519,7 @@ class FsMongo(FsBase):
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)
         """
         try:
             f = storage if isinstance(storage, str) else "/".join(storage)

+            f = f.rstrip("/")

 
             file_cursor = self.fs.find({"filename": f})
             found = False
 
             file_cursor = self.fs.find({"filename": f})
             found = False


@@ -508,18 +528,27 @@ class FsMongo(FsBase):
                 exception_file = next(file_cursor, None)
 
                 if exception_file:
                 exception_file = next(file_cursor, None)
 
                 if exception_file:

+                    self.logger.error(
+                        "Cannot delete duplicate file: {} and {}".format(
+                            requested_file.filename, exception_file.filename
+                        )
+                    )

                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":
                     raise FsException(
                         "Multiple files found",
                         http_code=HTTPStatus.INTERNAL_SERVER_ERROR,
                     )
 
                 if requested_file.metadata["type"] == "dir":

-                    dir_cursor = self.fs.find({"filename": {"$regex": "^{}".format(f)}})
+                    dir_cursor = self.fs.find(
+                        {"filename": {"$regex": "^{}/".format(f)}}
+                    )

 
                     for tmp in dir_cursor:
 
                     for tmp in dir_cursor:

+                        self.logger.debug("Deleting {}".format(tmp.filename))

                         self.fs.delete(tmp._id)
                         self.fs.delete(tmp._id)

-                else:
-                    self.fs.delete(requested_file._id)
+
+                self.logger.debug("Deleting {}".format(requested_file.filename))
+                self.fs.delete(requested_file._id)

             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),
             if not found and not ignore_non_exist:
                 raise FsException(
                     "File {} does not exist".format(storage),


@@ -543,14 +572,14 @@ class FsMongo(FsBase):
         self.__update_local_fs(from_path=from_path)
 
     def _update_mongo_fs(self, from_path):
         self.__update_local_fs(from_path=from_path)
 
     def _update_mongo_fs(self, from_path):

-

         os_path = self.path + from_path
         os_path = self.path + from_path

-

         # Obtain list of files and dirs in filesystem
         members = []
         for root, dirs, files in os.walk(os_path):
             for folder in dirs:
                 member = {"filename": os.path.join(root, folder), "type": "dir"}
         # Obtain list of files and dirs in filesystem
         members = []
         for root, dirs, files in os.walk(os_path):
             for folder in dirs:
                 member = {"filename": os.path.join(root, folder), "type": "dir"}

+                if os.path.islink(member["filename"]):
+                    member["type"] = "sym"

                 members.append(member)
             for file in files:
                 filename = os.path.join(root, file)
                 members.append(member)
             for file in files:
                 filename = os.path.join(root, file)


@@ -574,7 +603,9 @@ class FsMongo(FsBase):
 
             # convert to relative path
             rel_filename = os.path.relpath(member["filename"], self.path)
 
             # convert to relative path
             rel_filename = os.path.relpath(member["filename"], self.path)

-            last_modified_date = datetime.datetime.fromtimestamp(
+            # get timestamp in UTC because mongo stores upload date in UTC:
+            # https://www.mongodb.com/docs/v4.0/tutorial/model-time-data/#overview
+            last_modified_date = datetime.datetime.utcfromtimestamp(

                 os.path.getmtime(member["filename"])
             )
 
                 os.path.getmtime(member["filename"])
             )
 


@@ -586,7 +617,6 @@ class FsMongo(FsBase):
             remote_files.pop(rel_filename, None)
 
             if last_modified_date >= upload_date:
             remote_files.pop(rel_filename, None)
 
             if last_modified_date >= upload_date:

-

                 stream = None
                 fh = None
                 try:
                 stream = None
                 fh = None
                 try:


@@ -603,11 +633,13 @@ class FsMongo(FsBase):
 
                     metadata = {"type": file_type, "permissions": mask}
 
 
                     metadata = {"type": file_type, "permissions": mask}
 

+                    self.logger.debug("Sync upload {}".format(rel_filename))

                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:
                     self.fs.upload_from_stream(rel_filename, stream, metadata=metadata)
 
                     # delete old files
                     if remote_file:
                         for file in remote_file:

+                            self.logger.debug("Sync deleting {}".format(file.filename))

                             self.fs.delete(file._id)
                 finally:
                     if fh:
                             self.fs.delete(file._id)
                 finally:
                     if fh:


@@ -615,13 +647,12 @@ class FsMongo(FsBase):
                     if stream:
                         stream.close()
 
                     if stream:
                         stream.close()
 

-        # delete files that are not any more in local fs
+        # delete files that are not anymore in local fs

         for remote_file in remote_files.values():
             for file in remote_file:
                 self.fs.delete(file._id)
 
     def _get_mongo_files(self, from_path=None):
         for remote_file in remote_files.values():
             for file in remote_file:
                 self.fs.delete(file._id)
 
     def _get_mongo_files(self, from_path=None):

-

         file_dict = {}
         file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
         for file in file_cursor:
         file_dict = {}
         file_cursor = self.fs.find(no_cursor_timeout=True, sort=[("uploadDate", -1)])
         for file in file_cursor: