X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fcommon.git;a=blobdiff_plain;f=osm_common%2Ftests%2Ftest_fsmongo.py;h=6a8ec5838d615120e9d8844e754ecf76368ff1de;hp=3b6256920ee2c0944b797369d035d66f55fd7b00;hb=2644b76248a1b96f7a47013b414e31b4e3feecf8;hpb=fee977a304dbfadee797e25906df2db3cf51f1e3 diff --git a/osm_common/tests/test_fsmongo.py b/osm_common/tests/test_fsmongo.py index 3b62569..6a8ec58 100644 --- a/osm_common/tests/test_fsmongo.py +++ b/osm_common/tests/test_fsmongo.py @@ -38,11 +38,11 @@ __author__ = "Eduardo Sousa " def valid_path(): - return tempfile.gettempdir() + '/' + return tempfile.gettempdir() + "/" def invalid_path(): - return '/#tweeter/' + return "/#tweeter/" @pytest.fixture(scope="function", params=[True, False]) @@ -56,15 +56,13 @@ def fs_mongo(request, monkeypatch): def mock_gridfs_constructor(a, b): pass - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', mock_gridfs_constructor) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", mock_gridfs_constructor) fs = FsMongo(lock=request.param) - fs.fs_connect({ - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files'}) + fs.fs_connect( + {"path": valid_path(), "host": "mongo", "port": 27017, "collection": "files"} + ) return fs @@ -73,26 +71,28 @@ def generic_fs_exception_message(message): def fs_connect_exception_message(path): - return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path) + return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format( + path + ) def file_open_file_not_found_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def file_open_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) def dir_ls_not_a_directory_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def dir_ls_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) @@ -102,14 +102,14 @@ def file_delete_exception_message(storage): def test_constructor_without_logger(): fs = FsMongo() - assert fs.logger == logging.getLogger('fs') + assert fs.logger == logging.getLogger("fs") assert fs.path is None assert fs.client is None assert fs.fs is None def test_constructor_with_logger(): - logger_name = 'fs_mongo' + logger_name = "fs_mongo" fs = FsMongo(logger_name=logger_name) assert fs.logger == logging.getLogger(logger_name) assert fs.path is None @@ -121,7 +121,7 @@ def test_get_params(fs_mongo, monkeypatch): def mock_gridfs_find(self, search_query, **kwargs): return [] - monkeypatch.setattr(GridFSBucket, 'find', mock_gridfs_find) + monkeypatch.setattr(GridFSBucket, "find", mock_gridfs_find) params = fs_mongo.get_params() assert len(params) == 2 assert "fs" in params @@ -130,79 +130,83 @@ def test_get_params(fs_mongo, monkeypatch): assert params["path"] == valid_path() -@pytest.mark.parametrize("config, exp_logger, exp_path", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - )]) +@pytest.mark.parametrize( + "config, exp_logger, exp_path", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path()[:-1], + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + {"path": valid_path(), "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ( + { + "path": valid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "fs", + valid_path(), + ), + ( + {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ( + { + "path": valid_path()[:-1], + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "fs", + valid_path(), + ), + ], +) def test_fs_connect_with_valid_config(config, exp_logger, exp_path): fs = FsMongo() fs.fs_connect(config) @@ -212,90 +216,81 @@ def test_fs_connect_with_valid_config(config, exp_logger, exp_path): assert type(fs.fs) == GridFSBucket -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': '/', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - generic_fs_exception_message( - "Invalid configuration param at '[storage]': path '/' is not writable" - ) - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": invalid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()), + ), + ( + { + "logger_name": "fs_mongo", + "path": invalid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + fs_connect_exception_message(invalid_path()), + ), + ( + { + "logger_name": "fs_mongo", + "path": invalid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + { + "logger_name": "fs_mongo", + "path": invalid_path()[:-1], + "host": "mongo", + "port": 27017, + "collection": "files", + }, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()), + ), + ( + { + "path": invalid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + fs_connect_exception_message(invalid_path()), + ), + ( + {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + { + "path": invalid_path()[:-1], + "host": "mongo", + "port": 27017, + "collection": "files", + }, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": "/", "host": "mongo", "port": 27017, "collection": "files"}, + generic_fs_exception_message( + "Invalid configuration param at '[storage]': path '/' is not writable" + ), + ), + ], +) def test_fs_connect_with_invalid_path(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -303,67 +298,59 @@ def test_fs_connect_with_invalid_path(config, exp_exception_message): assert str(excinfo.value) == exp_exception_message -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'port': 27017, - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017' - }, - "Missing parameter \"collection\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - }, - "Missing parameter \"collection\"" - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"}, + 'Missing parameter "path"', + ), + ( + { + "logger_name": "fs_mongo", + "host": "mongo", + "port": 27017, + "collection": "files", + }, + 'Missing parameter "path"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"}, + 'Missing parameters: "uri" or "host" + "port"', + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "port": 27017, + "collection": "files", + }, + 'Missing parameters: "uri" or "host" + "port"', + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "collection": "files", + }, + 'Missing parameters: "uri" or "host" + "port"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"}, + 'Missing parameter "collection"', + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "port": 27017, + }, + 'Missing parameter "collection"', + ), + ], +) def test_fs_connect_with_missing_parameters(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -371,31 +358,37 @@ def test_fs_connect_with_missing_parameters(config, exp_exception_message): assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "MongoClient crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "MongoClient crashed" - )]) -def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monkeypatch): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "MongoClient crashed", + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "MongoClient crashed", + ), + ], +) +def test_fs_connect_with_invalid_mongoclient( + config, exp_exception_message, monkeypatch +): def generate_exception(a, b, c=None): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -403,35 +396,41 @@ def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monk assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Collection unavailable" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Collection unavailable" - )]) -def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, monkeypatch): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "Collection unavailable", + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "Collection unavailable", + ), + ], +) +def test_fs_connect_with_invalid_mongo_collection( + config, exp_exception_message, monkeypatch +): def mock_mongoclient_constructor(a, b, c=None): pass def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -439,27 +438,33 @@ def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "GridFsBucket crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "GridFsBucket crashed" - )]) -def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, monkeypatch): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "GridFsBucket crashed", + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "host": "mongo", + "port": 27017, + "collection": "files", + }, + "GridFsBucket crashed", + ), + ], +) +def test_fs_connect_with_invalid_gridfsbucket( + config, exp_exception_message, monkeypatch +): def mock_mongoclient_constructor(a, b, c=None): pass @@ -469,9 +474,9 @@ def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, mon def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -498,53 +503,55 @@ class FakeCursor: class FakeFS: - directory_metadata = {'type': 'dir', 'permissions': 509} - file_metadata = {'type': 'file', 'permissions': 436} - symlink_metadata = {'type': 'sym', 'permissions': 511} + directory_metadata = {"type": "dir", "permissions": 509} + file_metadata = {"type": "file", "permissions": 436} + symlink_metadata = {"type": "sym", "permissions": 511} tar_info = { 1: { - "cursor": FakeCursor(1, 'example_tar', directory_metadata), + "cursor": FakeCursor(1, "example_tar", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar', + "path": "./tmp/example_tar", }, 2: { - "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata), + "cursor": FakeCursor(2, "example_tar/directory", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/directory', + "path": "./tmp/example_tar/directory", }, 3: { - "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata), + "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/symlinks', + "path": "./tmp/example_tar/symlinks", }, 4: { - "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata), + "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata), "metadata": file_metadata, "stream_content": b"Example test", "stream_content_bad": b"Example test2", - "path": './tmp/example_tar/directory/file', + "path": "./tmp/example_tar/directory/file", }, 5: { - "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata), + "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata), "metadata": symlink_metadata, "stream_content": b"../directory/file", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/file_link', + "path": "./tmp/example_tar/symlinks/file_link", }, 6: { - "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata), + "cursor": FakeCursor( + 6, "example_tar/symlinks/directory_link", symlink_metadata + ), "metadata": symlink_metadata, "stream_content": b"../directory/", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/directory_link', - } + "path": "./tmp/example_tar/symlinks/directory_link", + }, } def upload_from_stream(self, f, stream, metadata=None): @@ -635,7 +642,9 @@ def test_upload_mongo_fs(): os.mkdir(path) os.mkdir("{}example_local".format(path)) os.mkdir("{}example_local/directory".format(path)) - with open("{}example_local/directory/test_file".format(path), "w+") as test_file: + with open( + "{}example_local/directory/test_file".format(path), "w+" + ) as test_file: test_file.write(file_content) fs.reverse_sync("example_local")