1 # Copyright 2019 Canonical
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # For those usages not covered by the Apache License, Version 2.0 please
16 # contact: eduardo.sousa@canonical.com
26 from pymongo
import MongoClient
27 from gridfs
import GridFSBucket
29 from io
import BytesIO
31 from unittest
.mock
import Mock
33 from osm_common
.fsbase
import FsException
34 from osm_common
.fsmongo
import FsMongo
35 from pathlib
import Path
37 __author__
= "Eduardo Sousa <eduardo.sousa@canonical.com>"
41 return tempfile
.gettempdir() + "/"
48 @pytest.fixture(scope
="function", params
=[True, False])
49 def fs_mongo(request
, monkeypatch
):
50 def mock_mongoclient_constructor(a
, b
, c
):
53 def mock_mongoclient_getitem(a
, b
):
56 def mock_gridfs_constructor(a
, b
):
59 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
60 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
61 monkeypatch
.setattr(GridFSBucket
, "__init__", mock_gridfs_constructor
)
62 fs
= FsMongo(lock
=request
.param
)
64 {"path": valid_path(), "host": "mongo", "port": 27017, "collection": "files"}
69 def generic_fs_exception_message(message
):
70 return "storage exception {}".format(message
)
73 def fs_connect_exception_message(path
):
74 return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(
79 def file_open_file_not_found_exception(storage
):
80 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
81 return "storage exception File {} does not exist".format(f
)
84 def file_open_io_exception(storage
):
85 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
86 return "storage exception File {} cannot be opened".format(f
)
89 def dir_ls_not_a_directory_exception(storage
):
90 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
91 return "storage exception File {} does not exist".format(f
)
94 def dir_ls_io_exception(storage
):
95 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
96 return "storage exception File {} cannot be opened".format(f
)
99 def file_delete_exception_message(storage
):
100 return "storage exception File {} does not exist".format(storage
)
103 def test_constructor_without_logger():
105 assert fs
.logger
== logging
.getLogger("fs")
106 assert fs
.path
is None
107 assert fs
.client
is None
111 def test_constructor_with_logger():
112 logger_name
= "fs_mongo"
113 fs
= FsMongo(logger_name
=logger_name
)
114 assert fs
.logger
== logging
.getLogger(logger_name
)
115 assert fs
.path
is None
116 assert fs
.client
is None
120 def test_get_params(fs_mongo
, monkeypatch
):
121 def mock_gridfs_find(self
, search_query
, **kwargs
):
124 monkeypatch
.setattr(GridFSBucket
, "find", mock_gridfs_find
)
125 params
= fs_mongo
.get_params()
126 assert len(params
) == 2
127 assert "fs" in params
128 assert "path" in params
129 assert params
["fs"] == "mongo"
130 assert params
["path"] == valid_path()
133 @pytest.mark
.parametrize(
134 "config, exp_logger, exp_path",
138 "logger_name": "fs_mongo",
139 "path": valid_path(),
140 "uri": "mongo:27017",
141 "collection": "files",
148 "logger_name": "fs_mongo",
149 "path": valid_path(),
152 "collection": "files",
159 "logger_name": "fs_mongo",
160 "path": valid_path()[:-1],
161 "uri": "mongo:27017",
162 "collection": "files",
169 "logger_name": "fs_mongo",
170 "path": valid_path()[:-1],
173 "collection": "files",
179 {"path": valid_path(), "uri": "mongo:27017", "collection": "files"},
185 "path": valid_path(),
188 "collection": "files",
194 {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
200 "path": valid_path()[:-1],
203 "collection": "files",
210 def test_fs_connect_with_valid_config(config
, exp_logger
, exp_path
):
212 fs
.fs_connect(config
)
213 assert fs
.logger
== logging
.getLogger(exp_logger
)
214 assert fs
.path
== exp_path
215 assert type(fs
.client
) == MongoClient
216 assert type(fs
.fs
) == GridFSBucket
219 @pytest.mark
.parametrize(
220 "config, exp_exception_message",
224 "logger_name": "fs_mongo",
225 "path": invalid_path(),
226 "uri": "mongo:27017",
227 "collection": "files",
229 fs_connect_exception_message(invalid_path()),
233 "logger_name": "fs_mongo",
234 "path": invalid_path(),
237 "collection": "files",
239 fs_connect_exception_message(invalid_path()),
243 "logger_name": "fs_mongo",
244 "path": invalid_path()[:-1],
245 "uri": "mongo:27017",
246 "collection": "files",
248 fs_connect_exception_message(invalid_path()[:-1]),
252 "logger_name": "fs_mongo",
253 "path": invalid_path()[:-1],
256 "collection": "files",
258 fs_connect_exception_message(invalid_path()[:-1]),
261 {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"},
262 fs_connect_exception_message(invalid_path()),
266 "path": invalid_path(),
269 "collection": "files",
271 fs_connect_exception_message(invalid_path()),
274 {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
275 fs_connect_exception_message(invalid_path()[:-1]),
279 "path": invalid_path()[:-1],
282 "collection": "files",
284 fs_connect_exception_message(invalid_path()[:-1]),
287 {"path": "/", "host": "mongo", "port": 27017, "collection": "files"},
288 generic_fs_exception_message(
289 "Invalid configuration param at '[storage]': path '/' is not writable"
294 def test_fs_connect_with_invalid_path(config
, exp_exception_message
):
296 with pytest
.raises(FsException
) as excinfo
:
297 fs
.fs_connect(config
)
298 assert str(excinfo
.value
) == exp_exception_message
301 @pytest.mark
.parametrize(
302 "config, exp_exception_message",
305 {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"},
306 'Missing parameter "path"',
310 "logger_name": "fs_mongo",
313 "collection": "files",
315 'Missing parameter "path"',
318 {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"},
319 'Missing parameters: "uri" or "host" + "port"',
323 "logger_name": "fs_mongo",
324 "path": valid_path(),
326 "collection": "files",
328 'Missing parameters: "uri" or "host" + "port"',
332 "logger_name": "fs_mongo",
333 "path": valid_path(),
335 "collection": "files",
337 'Missing parameters: "uri" or "host" + "port"',
340 {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"},
341 'Missing parameter "collection"',
345 "logger_name": "fs_mongo",
346 "path": valid_path(),
350 'Missing parameter "collection"',
354 def test_fs_connect_with_missing_parameters(config
, exp_exception_message
):
356 with pytest
.raises(FsException
) as excinfo
:
357 fs
.fs_connect(config
)
358 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
361 @pytest.mark
.parametrize(
362 "config, exp_exception_message",
366 "logger_name": "fs_mongo",
367 "path": valid_path(),
368 "uri": "mongo:27017",
369 "collection": "files",
371 "MongoClient crashed",
375 "logger_name": "fs_mongo",
376 "path": valid_path(),
379 "collection": "files",
381 "MongoClient crashed",
385 def test_fs_connect_with_invalid_mongoclient(
386 config
, exp_exception_message
, monkeypatch
388 def generate_exception(a
, b
, c
=None):
389 raise Exception(exp_exception_message
)
391 monkeypatch
.setattr(MongoClient
, "__init__", generate_exception
)
394 with pytest
.raises(FsException
) as excinfo
:
395 fs
.fs_connect(config
)
396 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
399 @pytest.mark
.parametrize(
400 "config, exp_exception_message",
404 "logger_name": "fs_mongo",
405 "path": valid_path(),
406 "uri": "mongo:27017",
407 "collection": "files",
409 "Collection unavailable",
413 "logger_name": "fs_mongo",
414 "path": valid_path(),
417 "collection": "files",
419 "Collection unavailable",
423 def test_fs_connect_with_invalid_mongo_collection(
424 config
, exp_exception_message
, monkeypatch
426 def mock_mongoclient_constructor(a
, b
, c
=None):
429 def generate_exception(a
, b
):
430 raise Exception(exp_exception_message
)
432 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
433 monkeypatch
.setattr(MongoClient
, "__getitem__", generate_exception
)
436 with pytest
.raises(FsException
) as excinfo
:
437 fs
.fs_connect(config
)
438 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
441 @pytest.mark
.parametrize(
442 "config, exp_exception_message",
446 "logger_name": "fs_mongo",
447 "path": valid_path(),
448 "uri": "mongo:27017",
449 "collection": "files",
451 "GridFsBucket crashed",
455 "logger_name": "fs_mongo",
456 "path": valid_path(),
459 "collection": "files",
461 "GridFsBucket crashed",
465 def test_fs_connect_with_invalid_gridfsbucket(
466 config
, exp_exception_message
, monkeypatch
468 def mock_mongoclient_constructor(a
, b
, c
=None):
471 def mock_mongoclient_getitem(a
, b
):
474 def generate_exception(a
, b
):
475 raise Exception(exp_exception_message
)
477 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
478 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
479 monkeypatch
.setattr(GridFSBucket
, "__init__", generate_exception
)
482 with pytest
.raises(FsException
) as excinfo
:
483 fs
.fs_connect(config
)
484 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
487 def test_fs_disconnect(fs_mongo
):
488 fs_mongo
.fs_disconnect()
496 # ├── directory_link -> ../directory/
497 # └── file_link -> ../directory/file
499 def __init__(self
, id, filename
, metadata
):
501 self
.filename
= filename
502 self
.metadata
= metadata
506 directory_metadata
= {"type": "dir", "permissions": 509}
507 file_metadata
= {"type": "file", "permissions": 436}
508 symlink_metadata
= {"type": "sym", "permissions": 511}
512 "cursor": FakeCursor(1, "example_tar", directory_metadata
),
513 "metadata": directory_metadata
,
514 "stream_content": b
"",
515 "stream_content_bad": b
"Something",
516 "path": "./tmp/example_tar",
519 "cursor": FakeCursor(2, "example_tar/directory", directory_metadata
),
520 "metadata": directory_metadata
,
521 "stream_content": b
"",
522 "stream_content_bad": b
"Something",
523 "path": "./tmp/example_tar/directory",
526 "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata
),
527 "metadata": directory_metadata
,
528 "stream_content": b
"",
529 "stream_content_bad": b
"Something",
530 "path": "./tmp/example_tar/symlinks",
533 "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata
),
534 "metadata": file_metadata
,
535 "stream_content": b
"Example test",
536 "stream_content_bad": b
"Example test2",
537 "path": "./tmp/example_tar/directory/file",
540 "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata
),
541 "metadata": symlink_metadata
,
542 "stream_content": b
"../directory/file",
543 "stream_content_bad": b
"",
544 "path": "./tmp/example_tar/symlinks/file_link",
547 "cursor": FakeCursor(
548 6, "example_tar/symlinks/directory_link", symlink_metadata
550 "metadata": symlink_metadata
,
551 "stream_content": b
"../directory/",
552 "stream_content_bad": b
"",
553 "path": "./tmp/example_tar/symlinks/directory_link",
557 def upload_from_stream(self
, f
, stream
, metadata
=None):
559 for i
, v
in self
.tar_info
.items():
561 assert metadata
["type"] == v
["metadata"]["type"]
562 assert stream
.read() == BytesIO(v
["stream_content"]).read()
564 assert stream
.read() != BytesIO(v
["stream_content_bad"]).read()
569 def find(self
, type, no_cursor_timeout
=True, sort
=None):
571 for i
, v
in self
.tar_info
.items():
572 if type["metadata.type"] == "dir":
573 if v
["metadata"] == self
.directory_metadata
:
574 list.append(v
["cursor"])
576 if v
["metadata"] != self
.directory_metadata
:
577 list.append(v
["cursor"])
580 def download_to_stream(self
, id, file_stream
):
581 file_stream
.write(BytesIO(self
.tar_info
[id]["stream_content"]).read())
584 def test_file_extract():
585 tar_path
= "tmp/Example.tar.gz"
586 folder_path
= "tmp/example_tar"
589 subprocess
.call(["rm", "-rf", "./tmp"])
590 subprocess
.call(["mkdir", "-p", "{}/directory".format(folder_path
)])
591 subprocess
.call(["mkdir", "-p", "{}/symlinks".format(folder_path
)])
592 p
= Path("{}/directory/file".format(folder_path
))
593 p
.write_text("Example test")
594 os
.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path
))
595 os
.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path
))
596 if os
.path
.exists(tar_path
):
598 subprocess
.call(["tar", "-czvf", tar_path
, folder_path
])
601 tar
= tarfile
.open(tar_path
, "r")
604 fs
.file_extract(compressed_object
=tar
, path
=".")
607 subprocess
.call(["rm", "-rf", "./tmp"])
610 def test_upload_local_fs():
613 subprocess
.call(["rm", "-rf", path
])
619 assert os
.path
.isdir("{}example_tar".format(path
))
620 assert os
.path
.isdir("{}example_tar/directory".format(path
))
621 assert os
.path
.isdir("{}example_tar/symlinks".format(path
))
622 assert os
.path
.isfile("{}example_tar/directory/file".format(path
))
623 assert os
.path
.islink("{}example_tar/symlinks/file_link".format(path
))
624 assert os
.path
.islink("{}example_tar/symlinks/directory_link".format(path
))
626 subprocess
.call(["rm", "-rf", path
])
629 def test_upload_mongo_fs():
632 subprocess
.call(["rm", "-rf", path
])
637 fs
.fs
.find
.return_value
= {}
639 file_content
= "Test file content"
641 # Create local dir and upload content to fakefs
643 os
.mkdir("{}example_local".format(path
))
644 os
.mkdir("{}example_local/directory".format(path
))
646 "{}example_local/directory/test_file".format(path
), "w+"
648 test_file
.write(file_content
)
649 fs
.reverse_sync("example_local")
651 assert fs
.fs
.upload_from_stream
.call_count
== 2
653 # first call to upload_from_stream, dir_name
654 dir_name
= "example_local/directory"
655 call_args_0
= fs
.fs
.upload_from_stream
.call_args_list
[0]
656 assert call_args_0
[0][0] == dir_name
657 assert call_args_0
[1].get("metadata").get("type") == "dir"
659 # second call to upload_from_stream, dir_name
660 file_name
= "example_local/directory/test_file"
661 call_args_1
= fs
.fs
.upload_from_stream
.call_args_list
[1]
662 assert call_args_1
[0][0] == file_name
663 assert call_args_1
[1].get("metadata").get("type") == "file"
666 subprocess
.call(["rm", "-rf", path
])