X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Ftests%2Ftest_fsmongo.py;h=7e1e47ce7f4b01d9afa864364e47f3465e6c0299;hb=refs%2Fchanges%2F16%2F11816%2F6;hp=3b6256920ee2c0944b797369d035d66f55fd7b00;hpb=f296d2a468e38fef4145f526b5bc0726502d77cf;p=osm%2Fcommon.git diff --git a/osm_common/tests/test_fsmongo.py b/osm_common/tests/test_fsmongo.py index 3b62569..7e1e47c 100644 --- a/osm_common/tests/test_fsmongo.py +++ b/osm_common/tests/test_fsmongo.py @@ -16,38 +16,35 @@ # contact: eduardo.sousa@canonical.com ## +from io import BytesIO import logging -import pytest -import tempfile -import tarfile import os +from pathlib import Path import subprocess - -from pymongo import MongoClient -from gridfs import GridFSBucket - -from io import BytesIO - +import tarfile +import tempfile from unittest.mock import Mock +from gridfs import GridFSBucket from osm_common.fsbase import FsException from osm_common.fsmongo import FsMongo -from pathlib import Path +from pymongo import MongoClient +import pytest __author__ = "Eduardo Sousa " def valid_path(): - return tempfile.gettempdir() + '/' + return tempfile.gettempdir() + "/" def invalid_path(): - return '/#tweeter/' + return "/#tweeter/" @pytest.fixture(scope="function", params=[True, False]) def fs_mongo(request, monkeypatch): - def mock_mongoclient_constructor(a, b, c): + def mock_mongoclient_constructor(a, b): pass def mock_mongoclient_getitem(a, b): @@ -56,15 +53,11 @@ def fs_mongo(request, monkeypatch): def mock_gridfs_constructor(a, b): pass - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', mock_gridfs_constructor) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", mock_gridfs_constructor) fs = FsMongo(lock=request.param) - fs.fs_connect({ - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files'}) + fs.fs_connect({"path": valid_path(), "uri": "mongo:27017", "collection": "files"}) return fs @@ -73,26 +66,28 @@ def generic_fs_exception_message(message): def fs_connect_exception_message(path): - return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path) + return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format( + path + ) def file_open_file_not_found_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def file_open_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) def dir_ls_not_a_directory_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def dir_ls_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) @@ -102,14 +97,14 @@ def file_delete_exception_message(storage): def test_constructor_without_logger(): fs = FsMongo() - assert fs.logger == logging.getLogger('fs') + assert fs.logger == logging.getLogger("fs") assert fs.path is None assert fs.client is None assert fs.fs is None def test_constructor_with_logger(): - logger_name = 'fs_mongo' + logger_name = "fs_mongo" fs = FsMongo(logger_name=logger_name) assert fs.logger == logging.getLogger(logger_name) assert fs.path is None @@ -121,7 +116,7 @@ def test_get_params(fs_mongo, monkeypatch): def mock_gridfs_find(self, search_query, **kwargs): return [] - monkeypatch.setattr(GridFSBucket, 'find', mock_gridfs_find) + monkeypatch.setattr(GridFSBucket, "find", mock_gridfs_find) params = fs_mongo.get_params() assert len(params) == 2 assert "fs" in params @@ -130,79 +125,41 @@ def test_get_params(fs_mongo, monkeypatch): assert params["path"] == valid_path() -@pytest.mark.parametrize("config, exp_logger, exp_path", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - )]) +@pytest.mark.parametrize( + "config, exp_logger, exp_path", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + {"path": valid_path(), "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ( + {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ], +) def test_fs_connect_with_valid_config(config, exp_logger, exp_path): fs = FsMongo() fs.fs_connect(config) @@ -212,90 +169,43 @@ def test_fs_connect_with_valid_config(config, exp_logger, exp_path): assert type(fs.fs) == GridFSBucket -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': '/', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - generic_fs_exception_message( - "Invalid configuration param at '[storage]': path '/' is not writable" - ) - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": invalid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()), + ), + ( + { + "logger_name": "fs_mongo", + "path": invalid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()), + ), + ( + {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": "/", "uri": "mongo:27017", "collection": "files"}, + generic_fs_exception_message( + "Invalid configuration param at '[storage]': path '/' is not writable" + ), + ), + ], +) def test_fs_connect_with_invalid_path(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -303,67 +213,23 @@ def test_fs_connect_with_invalid_path(config, exp_exception_message): assert str(excinfo.value) == exp_exception_message -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'port': 27017, - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017' - }, - "Missing parameter \"collection\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - }, - "Missing parameter \"collection\"" - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"}, + 'Missing parameter "path"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"}, + 'Missing parameters: "uri"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"}, + 'Missing parameter "collection"', + ), + ], +) def test_fs_connect_with_missing_parameters(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -371,31 +237,27 @@ def test_fs_connect_with_missing_parameters(config, exp_exception_message): assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "MongoClient crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "MongoClient crashed" - )]) -def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monkeypatch): - def generate_exception(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "MongoClient crashed", + ), + ], +) +def test_fs_connect_with_invalid_mongoclient( + config, exp_exception_message, monkeypatch +): + def generate_exception(a, b=None): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -403,35 +265,31 @@ def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monk assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Collection unavailable" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Collection unavailable" - )]) -def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, monkeypatch): - def mock_mongoclient_constructor(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "Collection unavailable", + ), + ], +) +def test_fs_connect_with_invalid_mongo_collection( + config, exp_exception_message, monkeypatch +): + def mock_mongoclient_constructor(a, b=None): pass def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -439,28 +297,24 @@ def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "GridFsBucket crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "GridFsBucket crashed" - )]) -def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, monkeypatch): - def mock_mongoclient_constructor(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "GridFsBucket crashed", + ), + ], +) +def test_fs_connect_with_invalid_gridfsbucket( + config, exp_exception_message, monkeypatch +): + def mock_mongoclient_constructor(a, b=None): pass def mock_mongoclient_getitem(a, b): @@ -469,9 +323,9 @@ def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, mon def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -498,53 +352,55 @@ class FakeCursor: class FakeFS: - directory_metadata = {'type': 'dir', 'permissions': 509} - file_metadata = {'type': 'file', 'permissions': 436} - symlink_metadata = {'type': 'sym', 'permissions': 511} + directory_metadata = {"type": "dir", "permissions": 509} + file_metadata = {"type": "file", "permissions": 436} + symlink_metadata = {"type": "sym", "permissions": 511} tar_info = { 1: { - "cursor": FakeCursor(1, 'example_tar', directory_metadata), + "cursor": FakeCursor(1, "example_tar", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar', + "path": "./tmp/example_tar", }, 2: { - "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata), + "cursor": FakeCursor(2, "example_tar/directory", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/directory', + "path": "./tmp/example_tar/directory", }, 3: { - "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata), + "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/symlinks', + "path": "./tmp/example_tar/symlinks", }, 4: { - "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata), + "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata), "metadata": file_metadata, "stream_content": b"Example test", "stream_content_bad": b"Example test2", - "path": './tmp/example_tar/directory/file', + "path": "./tmp/example_tar/directory/file", }, 5: { - "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata), + "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata), "metadata": symlink_metadata, "stream_content": b"../directory/file", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/file_link', + "path": "./tmp/example_tar/symlinks/file_link", }, 6: { - "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata), + "cursor": FakeCursor( + 6, "example_tar/symlinks/directory_link", symlink_metadata + ), "metadata": symlink_metadata, "stream_content": b"../directory/", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/directory_link', - } + "path": "./tmp/example_tar/symlinks/directory_link", + }, } def upload_from_stream(self, f, stream, metadata=None): @@ -594,7 +450,7 @@ def test_file_extract(): tar = tarfile.open(tar_path, "r") fs = FsMongo() fs.fs = FakeFS() - fs.file_extract(tar_object=tar, path=".") + fs.file_extract(compressed_object=tar, path=".") finally: os.remove(tar_path) subprocess.call(["rm", "-rf", "./tmp"]) @@ -635,7 +491,9 @@ def test_upload_mongo_fs(): os.mkdir(path) os.mkdir("{}example_local".format(path)) os.mkdir("{}example_local/directory".format(path)) - with open("{}example_local/directory/test_file".format(path), "w+") as test_file: + with open( + "{}example_local/directory/test_file".format(path), "w+" + ) as test_file: test_file.write(file_content) fs.reverse_sync("example_local")