X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_common%2Ftests%2Ftest_fsmongo.py;h=7e1e47ce7f4b01d9afa864364e47f3465e6c0299;hb=7da9795a4b73c72e81ac4880a9e9507e441aa90f;hp=a14bff7545578d55e0992eae4df9f83a1297ecf3;hpb=8ab6cc621e77323982e19eaf4f4e1d8ddcd356df;p=osm%2Fcommon.git diff --git a/osm_common/tests/test_fsmongo.py b/osm_common/tests/test_fsmongo.py index a14bff7..7e1e47c 100644 --- a/osm_common/tests/test_fsmongo.py +++ b/osm_common/tests/test_fsmongo.py @@ -16,36 +16,35 @@ # contact: eduardo.sousa@canonical.com ## +from io import BytesIO import logging -import pytest -import tempfile -import tarfile import os +from pathlib import Path import subprocess +import tarfile +import tempfile +from unittest.mock import Mock -from pymongo import MongoClient from gridfs import GridFSBucket - -from io import BytesIO - from osm_common.fsbase import FsException from osm_common.fsmongo import FsMongo -from pathlib import Path +from pymongo import MongoClient +import pytest __author__ = "Eduardo Sousa " def valid_path(): - return tempfile.gettempdir() + '/' + return tempfile.gettempdir() + "/" def invalid_path(): - return '/#tweeter/' + return "/#tweeter/" @pytest.fixture(scope="function", params=[True, False]) def fs_mongo(request, monkeypatch): - def mock_mongoclient_constructor(a, b, c): + def mock_mongoclient_constructor(a, b): pass def mock_mongoclient_getitem(a, b): @@ -54,15 +53,11 @@ def fs_mongo(request, monkeypatch): def mock_gridfs_constructor(a, b): pass - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', mock_gridfs_constructor) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", mock_gridfs_constructor) fs = FsMongo(lock=request.param) - fs.fs_connect({ - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files'}) + fs.fs_connect({"path": valid_path(), "uri": "mongo:27017", "collection": "files"}) return fs @@ -71,26 +66,28 @@ def generic_fs_exception_message(message): def fs_connect_exception_message(path): - return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path) + return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format( + path + ) def file_open_file_not_found_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def file_open_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) def dir_ls_not_a_directory_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} does not exist".format(f) def dir_ls_io_exception(storage): - f = storage if isinstance(storage, str) else '/'.join(storage) + f = storage if isinstance(storage, str) else "/".join(storage) return "storage exception File {} cannot be opened".format(f) @@ -100,14 +97,14 @@ def file_delete_exception_message(storage): def test_constructor_without_logger(): fs = FsMongo() - assert fs.logger == logging.getLogger('fs') + assert fs.logger == logging.getLogger("fs") assert fs.path is None assert fs.client is None assert fs.fs is None def test_constructor_with_logger(): - logger_name = 'fs_mongo' + logger_name = "fs_mongo" fs = FsMongo(logger_name=logger_name) assert fs.logger == logging.getLogger(logger_name) assert fs.path is None @@ -119,7 +116,7 @@ def test_get_params(fs_mongo, monkeypatch): def mock_gridfs_find(self, search_query, **kwargs): return [] - monkeypatch.setattr(GridFSBucket, 'find', mock_gridfs_find) + monkeypatch.setattr(GridFSBucket, "find", mock_gridfs_find) params = fs_mongo.get_params() assert len(params) == 2 assert "fs" in params @@ -128,79 +125,41 @@ def test_get_params(fs_mongo, monkeypatch): assert params["path"] == valid_path() -@pytest.mark.parametrize("config, exp_logger, exp_path", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs_mongo', valid_path() - ), - ( - { - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - 'fs', valid_path() - ), - ( - { - 'path': valid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - 'fs', valid_path() - )]) +@pytest.mark.parametrize( + "config, exp_logger, exp_path", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + { + "logger_name": "fs_mongo", + "path": valid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + "fs_mongo", + valid_path(), + ), + ( + {"path": valid_path(), "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ( + {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + "fs", + valid_path(), + ), + ], +) def test_fs_connect_with_valid_config(config, exp_logger, exp_path): fs = FsMongo() fs.fs_connect(config) @@ -210,90 +169,43 @@ def test_fs_connect_with_valid_config(config, exp_logger, exp_path): assert type(fs.fs) == GridFSBucket -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()) - ), - ( - { - 'path': invalid_path()[:-1], - 'uri': 'mongo:27017', - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': invalid_path()[:-1], - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - fs_connect_exception_message(invalid_path()[:-1]) - ), - ( - { - 'path': '/', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - generic_fs_exception_message( - "Invalid configuration param at '[storage]': path '/' is not writable" - ) - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": invalid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()), + ), + ( + { + "logger_name": "fs_mongo", + "path": invalid_path()[:-1], + "uri": "mongo:27017", + "collection": "files", + }, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()), + ), + ( + {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"}, + fs_connect_exception_message(invalid_path()[:-1]), + ), + ( + {"path": "/", "uri": "mongo:27017", "collection": "files"}, + generic_fs_exception_message( + "Invalid configuration param at '[storage]': path '/' is not writable" + ), + ), + ], +) def test_fs_connect_with_invalid_path(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -301,67 +213,23 @@ def test_fs_connect_with_invalid_path(config, exp_exception_message): assert str(excinfo.value) == exp_exception_message -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Missing parameter \"path\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'port': 27017, - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'collection': 'files' - }, - "Missing parameters: \"uri\" or \"host\" + \"port\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017' - }, - "Missing parameter \"collection\"" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - }, - "Missing parameter \"collection\"" - )]) +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"}, + 'Missing parameter "path"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"}, + 'Missing parameters: "uri"', + ), + ( + {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"}, + 'Missing parameter "collection"', + ), + ], +) def test_fs_connect_with_missing_parameters(config, exp_exception_message): fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -369,31 +237,27 @@ def test_fs_connect_with_missing_parameters(config, exp_exception_message): assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "MongoClient crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "MongoClient crashed" - )]) -def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monkeypatch): - def generate_exception(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "MongoClient crashed", + ), + ], +) +def test_fs_connect_with_invalid_mongoclient( + config, exp_exception_message, monkeypatch +): + def generate_exception(a, b=None): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -401,35 +265,31 @@ def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monk assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "Collection unavailable" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "Collection unavailable" - )]) -def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, monkeypatch): - def mock_mongoclient_constructor(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "Collection unavailable", + ), + ], +) +def test_fs_connect_with_invalid_mongo_collection( + config, exp_exception_message, monkeypatch +): + def mock_mongoclient_constructor(a, b=None): pass def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -437,28 +297,24 @@ def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message) -@pytest.mark.parametrize("config, exp_exception_message", [ - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'uri': 'mongo:27017', - 'collection': 'files' - }, - "GridFsBucket crashed" - ), - ( - { - 'logger_name': 'fs_mongo', - 'path': valid_path(), - 'host': 'mongo', - 'port': 27017, - 'collection': 'files' - }, - "GridFsBucket crashed" - )]) -def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, monkeypatch): - def mock_mongoclient_constructor(a, b, c=None): +@pytest.mark.parametrize( + "config, exp_exception_message", + [ + ( + { + "logger_name": "fs_mongo", + "path": valid_path(), + "uri": "mongo:27017", + "collection": "files", + }, + "GridFsBucket crashed", + ), + ], +) +def test_fs_connect_with_invalid_gridfsbucket( + config, exp_exception_message, monkeypatch +): + def mock_mongoclient_constructor(a, b=None): pass def mock_mongoclient_getitem(a, b): @@ -467,9 +323,9 @@ def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, mon def generate_exception(a, b): raise Exception(exp_exception_message) - monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor) - monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem) - monkeypatch.setattr(GridFSBucket, '__init__', generate_exception) + monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor) + monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem) + monkeypatch.setattr(GridFSBucket, "__init__", generate_exception) fs = FsMongo() with pytest.raises(FsException) as excinfo: @@ -496,53 +352,55 @@ class FakeCursor: class FakeFS: - directory_metadata = {'type': 'dir', 'permissions': 509} - file_metadata = {'type': 'file', 'permissions': 436} - symlink_metadata = {'type': 'sym', 'permissions': 511} + directory_metadata = {"type": "dir", "permissions": 509} + file_metadata = {"type": "file", "permissions": 436} + symlink_metadata = {"type": "sym", "permissions": 511} tar_info = { 1: { - "cursor": FakeCursor(1, 'example_tar', directory_metadata), + "cursor": FakeCursor(1, "example_tar", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar', + "path": "./tmp/example_tar", }, 2: { - "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata), + "cursor": FakeCursor(2, "example_tar/directory", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/directory', + "path": "./tmp/example_tar/directory", }, 3: { - "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata), + "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata), "metadata": directory_metadata, - "stream_content": b'', + "stream_content": b"", "stream_content_bad": b"Something", - "path": './tmp/example_tar/symlinks', + "path": "./tmp/example_tar/symlinks", }, 4: { - "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata), + "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata), "metadata": file_metadata, "stream_content": b"Example test", "stream_content_bad": b"Example test2", - "path": './tmp/example_tar/directory/file', + "path": "./tmp/example_tar/directory/file", }, 5: { - "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata), + "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata), "metadata": symlink_metadata, "stream_content": b"../directory/file", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/file_link', + "path": "./tmp/example_tar/symlinks/file_link", }, 6: { - "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata), + "cursor": FakeCursor( + 6, "example_tar/symlinks/directory_link", symlink_metadata + ), "metadata": symlink_metadata, "stream_content": b"../directory/", "stream_content_bad": b"", - "path": './tmp/example_tar/symlinks/directory_link', - } + "path": "./tmp/example_tar/symlinks/directory_link", + }, } def upload_from_stream(self, f, stream, metadata=None): @@ -557,7 +415,7 @@ class FakeFS: continue assert found - def find(self, type, no_cursor_timeout=True): + def find(self, type, no_cursor_timeout=True, sort=None): list = [] for i, v in self.tar_info.items(): if type["metadata.type"] == "dir": @@ -592,7 +450,7 @@ def test_file_extract(): tar = tarfile.open(tar_path, "r") fs = FsMongo() fs.fs = FakeFS() - fs.file_extract(tar_object=tar, path=".") + fs.file_extract(compressed_object=tar, path=".") finally: os.remove(tar_path) subprocess.call(["rm", "-rf", "./tmp"]) @@ -615,3 +473,44 @@ def test_upload_local_fs(): assert os.path.islink("{}example_tar/symlinks/directory_link".format(path)) finally: subprocess.call(["rm", "-rf", path]) + + +def test_upload_mongo_fs(): + path = "./tmp/" + + subprocess.call(["rm", "-rf", path]) + try: + fs = FsMongo() + fs.path = path + fs.fs = Mock() + fs.fs.find.return_value = {} + + file_content = "Test file content" + + # Create local dir and upload content to fakefs + os.mkdir(path) + os.mkdir("{}example_local".format(path)) + os.mkdir("{}example_local/directory".format(path)) + with open( + "{}example_local/directory/test_file".format(path), "w+" + ) as test_file: + test_file.write(file_content) + fs.reverse_sync("example_local") + + assert fs.fs.upload_from_stream.call_count == 2 + + # first call to upload_from_stream, dir_name + dir_name = "example_local/directory" + call_args_0 = fs.fs.upload_from_stream.call_args_list[0] + assert call_args_0[0][0] == dir_name + assert call_args_0[1].get("metadata").get("type") == "dir" + + # second call to upload_from_stream, dir_name + file_name = "example_local/directory/test_file" + call_args_1 = fs.fs.upload_from_stream.call_args_list[1] + assert call_args_1[0][0] == file_name + assert call_args_1[1].get("metadata").get("type") == "file" + + finally: + subprocess.call(["rm", "-rf", path]) + pass