1 # Copyright 2019 Canonical
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # For those usages not covered by the Apache License, Version 2.0 please
16 # contact: eduardo.sousa@canonical.com
19 from io
import BytesIO
22 from pathlib
import Path
26 from unittest
.mock
import Mock
28 from gridfs
import GridFSBucket
29 from osm_common
.fsbase
import FsException
30 from osm_common
.fsmongo
import FsMongo
31 from pymongo
import MongoClient
34 __author__
= "Eduardo Sousa <eduardo.sousa@canonical.com>"
38 return tempfile
.gettempdir() + "/"
45 @pytest.fixture(scope
="function", params
=[True, False])
46 def fs_mongo(request
, monkeypatch
):
47 def mock_mongoclient_constructor(a
, b
):
50 def mock_mongoclient_getitem(a
, b
):
53 def mock_gridfs_constructor(a
, b
):
56 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
57 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
58 monkeypatch
.setattr(GridFSBucket
, "__init__", mock_gridfs_constructor
)
59 fs
= FsMongo(lock
=request
.param
)
60 fs
.fs_connect({"path": valid_path(), "uri": "mongo:27017", "collection": "files"})
64 def generic_fs_exception_message(message
):
65 return "storage exception {}".format(message
)
68 def fs_connect_exception_message(path
):
69 return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(
74 def file_open_file_not_found_exception(storage
):
75 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
76 return "storage exception File {} does not exist".format(f
)
79 def file_open_io_exception(storage
):
80 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
81 return "storage exception File {} cannot be opened".format(f
)
84 def dir_ls_not_a_directory_exception(storage
):
85 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
86 return "storage exception File {} does not exist".format(f
)
89 def dir_ls_io_exception(storage
):
90 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
91 return "storage exception File {} cannot be opened".format(f
)
94 def file_delete_exception_message(storage
):
95 return "storage exception File {} does not exist".format(storage
)
98 def test_constructor_without_logger():
100 assert fs
.logger
== logging
.getLogger("fs")
101 assert fs
.path
is None
102 assert fs
.client
is None
106 def test_constructor_with_logger():
107 logger_name
= "fs_mongo"
108 fs
= FsMongo(logger_name
=logger_name
)
109 assert fs
.logger
== logging
.getLogger(logger_name
)
110 assert fs
.path
is None
111 assert fs
.client
is None
115 def test_get_params(fs_mongo
, monkeypatch
):
116 def mock_gridfs_find(self
, search_query
, **kwargs
):
119 monkeypatch
.setattr(GridFSBucket
, "find", mock_gridfs_find
)
120 params
= fs_mongo
.get_params()
121 assert len(params
) == 2
122 assert "fs" in params
123 assert "path" in params
124 assert params
["fs"] == "mongo"
125 assert params
["path"] == valid_path()
128 @pytest.mark
.parametrize(
129 "config, exp_logger, exp_path",
133 "logger_name": "fs_mongo",
134 "path": valid_path(),
135 "uri": "mongo:27017",
136 "collection": "files",
143 "logger_name": "fs_mongo",
144 "path": valid_path()[:-1],
145 "uri": "mongo:27017",
146 "collection": "files",
152 {"path": valid_path(), "uri": "mongo:27017", "collection": "files"},
157 {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
163 def test_fs_connect_with_valid_config(config
, exp_logger
, exp_path
):
165 fs
.fs_connect(config
)
166 assert fs
.logger
== logging
.getLogger(exp_logger
)
167 assert fs
.path
== exp_path
168 assert type(fs
.client
) == MongoClient
169 assert type(fs
.fs
) == GridFSBucket
172 @pytest.mark
.parametrize(
173 "config, exp_exception_message",
177 "logger_name": "fs_mongo",
178 "path": invalid_path(),
179 "uri": "mongo:27017",
180 "collection": "files",
182 fs_connect_exception_message(invalid_path()),
186 "logger_name": "fs_mongo",
187 "path": invalid_path()[:-1],
188 "uri": "mongo:27017",
189 "collection": "files",
191 fs_connect_exception_message(invalid_path()[:-1]),
194 {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"},
195 fs_connect_exception_message(invalid_path()),
198 {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
199 fs_connect_exception_message(invalid_path()[:-1]),
202 {"path": "/", "uri": "mongo:27017", "collection": "files"},
203 generic_fs_exception_message(
204 "Invalid configuration param at '[storage]': path '/' is not writable"
209 def test_fs_connect_with_invalid_path(config
, exp_exception_message
):
211 with pytest
.raises(FsException
) as excinfo
:
212 fs
.fs_connect(config
)
213 assert str(excinfo
.value
) == exp_exception_message
216 @pytest.mark
.parametrize(
217 "config, exp_exception_message",
220 {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"},
221 'Missing parameter "path"',
224 {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"},
225 'Missing parameters: "uri"',
228 {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"},
229 'Missing parameter "collection"',
233 def test_fs_connect_with_missing_parameters(config
, exp_exception_message
):
235 with pytest
.raises(FsException
) as excinfo
:
236 fs
.fs_connect(config
)
237 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
240 @pytest.mark
.parametrize(
241 "config, exp_exception_message",
245 "logger_name": "fs_mongo",
246 "path": valid_path(),
247 "uri": "mongo:27017",
248 "collection": "files",
250 "MongoClient crashed",
254 def test_fs_connect_with_invalid_mongoclient(
255 config
, exp_exception_message
, monkeypatch
257 def generate_exception(a
, b
=None):
258 raise Exception(exp_exception_message
)
260 monkeypatch
.setattr(MongoClient
, "__init__", generate_exception
)
263 with pytest
.raises(FsException
) as excinfo
:
264 fs
.fs_connect(config
)
265 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
268 @pytest.mark
.parametrize(
269 "config, exp_exception_message",
273 "logger_name": "fs_mongo",
274 "path": valid_path(),
275 "uri": "mongo:27017",
276 "collection": "files",
278 "Collection unavailable",
282 def test_fs_connect_with_invalid_mongo_collection(
283 config
, exp_exception_message
, monkeypatch
285 def mock_mongoclient_constructor(a
, b
=None):
288 def generate_exception(a
, b
):
289 raise Exception(exp_exception_message
)
291 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
292 monkeypatch
.setattr(MongoClient
, "__getitem__", generate_exception
)
295 with pytest
.raises(FsException
) as excinfo
:
296 fs
.fs_connect(config
)
297 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
300 @pytest.mark
.parametrize(
301 "config, exp_exception_message",
305 "logger_name": "fs_mongo",
306 "path": valid_path(),
307 "uri": "mongo:27017",
308 "collection": "files",
310 "GridFsBucket crashed",
314 def test_fs_connect_with_invalid_gridfsbucket(
315 config
, exp_exception_message
, monkeypatch
317 def mock_mongoclient_constructor(a
, b
=None):
320 def mock_mongoclient_getitem(a
, b
):
323 def generate_exception(a
, b
):
324 raise Exception(exp_exception_message
)
326 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
327 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
328 monkeypatch
.setattr(GridFSBucket
, "__init__", generate_exception
)
331 with pytest
.raises(FsException
) as excinfo
:
332 fs
.fs_connect(config
)
333 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
336 def test_fs_disconnect(fs_mongo
):
337 fs_mongo
.fs_disconnect()
342 # ├── directory
345 # ├── directory_link -> ../directory/
346 # └── file_link -> ../directory/file
348 def __init__(self
, id, filename
, metadata
):
350 self
.filename
= filename
351 self
.metadata
= metadata
355 directory_metadata
= {"type": "dir", "permissions": 509}
356 file_metadata
= {"type": "file", "permissions": 436}
357 symlink_metadata
= {"type": "sym", "permissions": 511}
361 "cursor": FakeCursor(1, "example_tar", directory_metadata
),
362 "metadata": directory_metadata
,
363 "stream_content": b
"",
364 "stream_content_bad": b
"Something",
365 "path": "./tmp/example_tar",
368 "cursor": FakeCursor(2, "example_tar/directory", directory_metadata
),
369 "metadata": directory_metadata
,
370 "stream_content": b
"",
371 "stream_content_bad": b
"Something",
372 "path": "./tmp/example_tar/directory",
375 "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata
),
376 "metadata": directory_metadata
,
377 "stream_content": b
"",
378 "stream_content_bad": b
"Something",
379 "path": "./tmp/example_tar/symlinks",
382 "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata
),
383 "metadata": file_metadata
,
384 "stream_content": b
"Example test",
385 "stream_content_bad": b
"Example test2",
386 "path": "./tmp/example_tar/directory/file",
389 "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata
),
390 "metadata": symlink_metadata
,
391 "stream_content": b
"../directory/file",
392 "stream_content_bad": b
"",
393 "path": "./tmp/example_tar/symlinks/file_link",
396 "cursor": FakeCursor(
397 6, "example_tar/symlinks/directory_link", symlink_metadata
399 "metadata": symlink_metadata
,
400 "stream_content": b
"../directory/",
401 "stream_content_bad": b
"",
402 "path": "./tmp/example_tar/symlinks/directory_link",
406 def upload_from_stream(self
, f
, stream
, metadata
=None):
408 for i
, v
in self
.tar_info
.items():
410 assert metadata
["type"] == v
["metadata"]["type"]
411 assert stream
.read() == BytesIO(v
["stream_content"]).read()
413 assert stream
.read() != BytesIO(v
["stream_content_bad"]).read()
418 def find(self
, type, no_cursor_timeout
=True, sort
=None):
420 for i
, v
in self
.tar_info
.items():
421 if type["metadata.type"] == "dir":
422 if v
["metadata"] == self
.directory_metadata
:
423 list.append(v
["cursor"])
425 if v
["metadata"] != self
.directory_metadata
:
426 list.append(v
["cursor"])
429 def download_to_stream(self
, id, file_stream
):
430 file_stream
.write(BytesIO(self
.tar_info
[id]["stream_content"]).read())
433 def test_file_extract():
434 tar_path
= "tmp/Example.tar.gz"
435 folder_path
= "tmp/example_tar"
438 subprocess
.call(["rm", "-rf", "./tmp"])
439 subprocess
.call(["mkdir", "-p", "{}/directory".format(folder_path
)])
440 subprocess
.call(["mkdir", "-p", "{}/symlinks".format(folder_path
)])
441 p
= Path("{}/directory/file".format(folder_path
))
442 p
.write_text("Example test")
443 os
.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path
))
444 os
.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path
))
445 if os
.path
.exists(tar_path
):
447 subprocess
.call(["tar", "-czvf", tar_path
, folder_path
])
450 tar
= tarfile
.open(tar_path
, "r")
453 fs
.file_extract(compressed_object
=tar
, path
=".")
456 subprocess
.call(["rm", "-rf", "./tmp"])
459 def test_upload_local_fs():
462 subprocess
.call(["rm", "-rf", path
])
468 assert os
.path
.isdir("{}example_tar".format(path
))
469 assert os
.path
.isdir("{}example_tar/directory".format(path
))
470 assert os
.path
.isdir("{}example_tar/symlinks".format(path
))
471 assert os
.path
.isfile("{}example_tar/directory/file".format(path
))
472 assert os
.path
.islink("{}example_tar/symlinks/file_link".format(path
))
473 assert os
.path
.islink("{}example_tar/symlinks/directory_link".format(path
))
475 subprocess
.call(["rm", "-rf", path
])
478 def test_upload_mongo_fs():
481 subprocess
.call(["rm", "-rf", path
])
486 fs
.fs
.find
.return_value
= {}
488 file_content
= "Test file content"
490 # Create local dir and upload content to fakefs
492 os
.mkdir("{}example_local".format(path
))
493 os
.mkdir("{}example_local/directory".format(path
))
495 "{}example_local/directory/test_file".format(path
), "w+"
497 test_file
.write(file_content
)
498 fs
.reverse_sync("example_local")
500 assert fs
.fs
.upload_from_stream
.call_count
== 2
502 # first call to upload_from_stream, dir_name
503 dir_name
= "example_local/directory"
504 call_args_0
= fs
.fs
.upload_from_stream
.call_args_list
[0]
505 assert call_args_0
[0][0] == dir_name
506 assert call_args_0
[1].get("metadata").get("type") == "dir"
508 # second call to upload_from_stream, dir_name
509 file_name
= "example_local/directory/test_file"
510 call_args_1
= fs
.fs
.upload_from_stream
.call_args_list
[1]
511 assert call_args_1
[0][0] == file_name
512 assert call_args_1
[1].get("metadata").get("type") == "file"
515 subprocess
.call(["rm", "-rf", path
])