1 # Copyright 2019 Canonical
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # For those usages not covered by the Apache License, Version 2.0 please
16 # contact: eduardo.sousa@canonical.com
19 from io
import BytesIO
22 from pathlib
import Path
26 from unittest
.mock
import Mock
28 from gridfs
import GridFSBucket
29 from osm_common
.fsbase
import FsException
30 from osm_common
.fsmongo
import FsMongo
31 from pymongo
import MongoClient
34 __author__
= "Eduardo Sousa <eduardo.sousa@canonical.com>"
38 return tempfile
.gettempdir() + "/"
45 @pytest.fixture(scope
="function", params
=[True, False])
46 def fs_mongo(request
, monkeypatch
):
47 def mock_mongoclient_constructor(a
, b
, c
):
50 def mock_mongoclient_getitem(a
, b
):
53 def mock_gridfs_constructor(a
, b
):
56 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
57 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
58 monkeypatch
.setattr(GridFSBucket
, "__init__", mock_gridfs_constructor
)
59 fs
= FsMongo(lock
=request
.param
)
61 {"path": valid_path(), "host": "mongo", "port": 27017, "collection": "files"}
66 def generic_fs_exception_message(message
):
67 return "storage exception {}".format(message
)
70 def fs_connect_exception_message(path
):
71 return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(
76 def file_open_file_not_found_exception(storage
):
77 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
78 return "storage exception File {} does not exist".format(f
)
81 def file_open_io_exception(storage
):
82 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
83 return "storage exception File {} cannot be opened".format(f
)
86 def dir_ls_not_a_directory_exception(storage
):
87 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
88 return "storage exception File {} does not exist".format(f
)
91 def dir_ls_io_exception(storage
):
92 f
= storage
if isinstance(storage
, str) else "/".join(storage
)
93 return "storage exception File {} cannot be opened".format(f
)
96 def file_delete_exception_message(storage
):
97 return "storage exception File {} does not exist".format(storage
)
100 def test_constructor_without_logger():
102 assert fs
.logger
== logging
.getLogger("fs")
103 assert fs
.path
is None
104 assert fs
.client
is None
108 def test_constructor_with_logger():
109 logger_name
= "fs_mongo"
110 fs
= FsMongo(logger_name
=logger_name
)
111 assert fs
.logger
== logging
.getLogger(logger_name
)
112 assert fs
.path
is None
113 assert fs
.client
is None
117 def test_get_params(fs_mongo
, monkeypatch
):
118 def mock_gridfs_find(self
, search_query
, **kwargs
):
121 monkeypatch
.setattr(GridFSBucket
, "find", mock_gridfs_find
)
122 params
= fs_mongo
.get_params()
123 assert len(params
) == 2
124 assert "fs" in params
125 assert "path" in params
126 assert params
["fs"] == "mongo"
127 assert params
["path"] == valid_path()
130 @pytest.mark
.parametrize(
131 "config, exp_logger, exp_path",
135 "logger_name": "fs_mongo",
136 "path": valid_path(),
137 "uri": "mongo:27017",
138 "collection": "files",
145 "logger_name": "fs_mongo",
146 "path": valid_path(),
149 "collection": "files",
156 "logger_name": "fs_mongo",
157 "path": valid_path()[:-1],
158 "uri": "mongo:27017",
159 "collection": "files",
166 "logger_name": "fs_mongo",
167 "path": valid_path()[:-1],
170 "collection": "files",
176 {"path": valid_path(), "uri": "mongo:27017", "collection": "files"},
182 "path": valid_path(),
185 "collection": "files",
191 {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
197 "path": valid_path()[:-1],
200 "collection": "files",
207 def test_fs_connect_with_valid_config(config
, exp_logger
, exp_path
):
209 fs
.fs_connect(config
)
210 assert fs
.logger
== logging
.getLogger(exp_logger
)
211 assert fs
.path
== exp_path
212 assert type(fs
.client
) == MongoClient
213 assert type(fs
.fs
) == GridFSBucket
216 @pytest.mark
.parametrize(
217 "config, exp_exception_message",
221 "logger_name": "fs_mongo",
222 "path": invalid_path(),
223 "uri": "mongo:27017",
224 "collection": "files",
226 fs_connect_exception_message(invalid_path()),
230 "logger_name": "fs_mongo",
231 "path": invalid_path(),
234 "collection": "files",
236 fs_connect_exception_message(invalid_path()),
240 "logger_name": "fs_mongo",
241 "path": invalid_path()[:-1],
242 "uri": "mongo:27017",
243 "collection": "files",
245 fs_connect_exception_message(invalid_path()[:-1]),
249 "logger_name": "fs_mongo",
250 "path": invalid_path()[:-1],
253 "collection": "files",
255 fs_connect_exception_message(invalid_path()[:-1]),
258 {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"},
259 fs_connect_exception_message(invalid_path()),
263 "path": invalid_path(),
266 "collection": "files",
268 fs_connect_exception_message(invalid_path()),
271 {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
272 fs_connect_exception_message(invalid_path()[:-1]),
276 "path": invalid_path()[:-1],
279 "collection": "files",
281 fs_connect_exception_message(invalid_path()[:-1]),
284 {"path": "/", "host": "mongo", "port": 27017, "collection": "files"},
285 generic_fs_exception_message(
286 "Invalid configuration param at '[storage]': path '/' is not writable"
291 def test_fs_connect_with_invalid_path(config
, exp_exception_message
):
293 with pytest
.raises(FsException
) as excinfo
:
294 fs
.fs_connect(config
)
295 assert str(excinfo
.value
) == exp_exception_message
298 @pytest.mark
.parametrize(
299 "config, exp_exception_message",
302 {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"},
303 'Missing parameter "path"',
307 "logger_name": "fs_mongo",
310 "collection": "files",
312 'Missing parameter "path"',
315 {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"},
316 'Missing parameters: "uri" or "host" + "port"',
320 "logger_name": "fs_mongo",
321 "path": valid_path(),
323 "collection": "files",
325 'Missing parameters: "uri" or "host" + "port"',
329 "logger_name": "fs_mongo",
330 "path": valid_path(),
332 "collection": "files",
334 'Missing parameters: "uri" or "host" + "port"',
337 {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"},
338 'Missing parameter "collection"',
342 "logger_name": "fs_mongo",
343 "path": valid_path(),
347 'Missing parameter "collection"',
351 def test_fs_connect_with_missing_parameters(config
, exp_exception_message
):
353 with pytest
.raises(FsException
) as excinfo
:
354 fs
.fs_connect(config
)
355 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
358 @pytest.mark
.parametrize(
359 "config, exp_exception_message",
363 "logger_name": "fs_mongo",
364 "path": valid_path(),
365 "uri": "mongo:27017",
366 "collection": "files",
368 "MongoClient crashed",
372 "logger_name": "fs_mongo",
373 "path": valid_path(),
376 "collection": "files",
378 "MongoClient crashed",
382 def test_fs_connect_with_invalid_mongoclient(
383 config
, exp_exception_message
, monkeypatch
385 def generate_exception(a
, b
, c
=None):
386 raise Exception(exp_exception_message
)
388 monkeypatch
.setattr(MongoClient
, "__init__", generate_exception
)
391 with pytest
.raises(FsException
) as excinfo
:
392 fs
.fs_connect(config
)
393 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
396 @pytest.mark
.parametrize(
397 "config, exp_exception_message",
401 "logger_name": "fs_mongo",
402 "path": valid_path(),
403 "uri": "mongo:27017",
404 "collection": "files",
406 "Collection unavailable",
410 "logger_name": "fs_mongo",
411 "path": valid_path(),
414 "collection": "files",
416 "Collection unavailable",
420 def test_fs_connect_with_invalid_mongo_collection(
421 config
, exp_exception_message
, monkeypatch
423 def mock_mongoclient_constructor(a
, b
, c
=None):
426 def generate_exception(a
, b
):
427 raise Exception(exp_exception_message
)
429 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
430 monkeypatch
.setattr(MongoClient
, "__getitem__", generate_exception
)
433 with pytest
.raises(FsException
) as excinfo
:
434 fs
.fs_connect(config
)
435 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
438 @pytest.mark
.parametrize(
439 "config, exp_exception_message",
443 "logger_name": "fs_mongo",
444 "path": valid_path(),
445 "uri": "mongo:27017",
446 "collection": "files",
448 "GridFsBucket crashed",
452 "logger_name": "fs_mongo",
453 "path": valid_path(),
456 "collection": "files",
458 "GridFsBucket crashed",
462 def test_fs_connect_with_invalid_gridfsbucket(
463 config
, exp_exception_message
, monkeypatch
465 def mock_mongoclient_constructor(a
, b
, c
=None):
468 def mock_mongoclient_getitem(a
, b
):
471 def generate_exception(a
, b
):
472 raise Exception(exp_exception_message
)
474 monkeypatch
.setattr(MongoClient
, "__init__", mock_mongoclient_constructor
)
475 monkeypatch
.setattr(MongoClient
, "__getitem__", mock_mongoclient_getitem
)
476 monkeypatch
.setattr(GridFSBucket
, "__init__", generate_exception
)
479 with pytest
.raises(FsException
) as excinfo
:
480 fs
.fs_connect(config
)
481 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
484 def test_fs_disconnect(fs_mongo
):
485 fs_mongo
.fs_disconnect()
490 # ├── directory
493 # ├── directory_link -> ../directory/
494 # └── file_link -> ../directory/file
496 def __init__(self
, id, filename
, metadata
):
498 self
.filename
= filename
499 self
.metadata
= metadata
503 directory_metadata
= {"type": "dir", "permissions": 509}
504 file_metadata
= {"type": "file", "permissions": 436}
505 symlink_metadata
= {"type": "sym", "permissions": 511}
509 "cursor": FakeCursor(1, "example_tar", directory_metadata
),
510 "metadata": directory_metadata
,
511 "stream_content": b
"",
512 "stream_content_bad": b
"Something",
513 "path": "./tmp/example_tar",
516 "cursor": FakeCursor(2, "example_tar/directory", directory_metadata
),
517 "metadata": directory_metadata
,
518 "stream_content": b
"",
519 "stream_content_bad": b
"Something",
520 "path": "./tmp/example_tar/directory",
523 "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata
),
524 "metadata": directory_metadata
,
525 "stream_content": b
"",
526 "stream_content_bad": b
"Something",
527 "path": "./tmp/example_tar/symlinks",
530 "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata
),
531 "metadata": file_metadata
,
532 "stream_content": b
"Example test",
533 "stream_content_bad": b
"Example test2",
534 "path": "./tmp/example_tar/directory/file",
537 "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata
),
538 "metadata": symlink_metadata
,
539 "stream_content": b
"../directory/file",
540 "stream_content_bad": b
"",
541 "path": "./tmp/example_tar/symlinks/file_link",
544 "cursor": FakeCursor(
545 6, "example_tar/symlinks/directory_link", symlink_metadata
547 "metadata": symlink_metadata
,
548 "stream_content": b
"../directory/",
549 "stream_content_bad": b
"",
550 "path": "./tmp/example_tar/symlinks/directory_link",
554 def upload_from_stream(self
, f
, stream
, metadata
=None):
556 for i
, v
in self
.tar_info
.items():
558 assert metadata
["type"] == v
["metadata"]["type"]
559 assert stream
.read() == BytesIO(v
["stream_content"]).read()
561 assert stream
.read() != BytesIO(v
["stream_content_bad"]).read()
566 def find(self
, type, no_cursor_timeout
=True, sort
=None):
568 for i
, v
in self
.tar_info
.items():
569 if type["metadata.type"] == "dir":
570 if v
["metadata"] == self
.directory_metadata
:
571 list.append(v
["cursor"])
573 if v
["metadata"] != self
.directory_metadata
:
574 list.append(v
["cursor"])
577 def download_to_stream(self
, id, file_stream
):
578 file_stream
.write(BytesIO(self
.tar_info
[id]["stream_content"]).read())
581 def test_file_extract():
582 tar_path
= "tmp/Example.tar.gz"
583 folder_path
= "tmp/example_tar"
586 subprocess
.call(["rm", "-rf", "./tmp"])
587 subprocess
.call(["mkdir", "-p", "{}/directory".format(folder_path
)])
588 subprocess
.call(["mkdir", "-p", "{}/symlinks".format(folder_path
)])
589 p
= Path("{}/directory/file".format(folder_path
))
590 p
.write_text("Example test")
591 os
.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path
))
592 os
.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path
))
593 if os
.path
.exists(tar_path
):
595 subprocess
.call(["tar", "-czvf", tar_path
, folder_path
])
598 tar
= tarfile
.open(tar_path
, "r")
601 fs
.file_extract(compressed_object
=tar
, path
=".")
604 subprocess
.call(["rm", "-rf", "./tmp"])
607 def test_upload_local_fs():
610 subprocess
.call(["rm", "-rf", path
])
616 assert os
.path
.isdir("{}example_tar".format(path
))
617 assert os
.path
.isdir("{}example_tar/directory".format(path
))
618 assert os
.path
.isdir("{}example_tar/symlinks".format(path
))
619 assert os
.path
.isfile("{}example_tar/directory/file".format(path
))
620 assert os
.path
.islink("{}example_tar/symlinks/file_link".format(path
))
621 assert os
.path
.islink("{}example_tar/symlinks/directory_link".format(path
))
623 subprocess
.call(["rm", "-rf", path
])
626 def test_upload_mongo_fs():
629 subprocess
.call(["rm", "-rf", path
])
634 fs
.fs
.find
.return_value
= {}
636 file_content
= "Test file content"
638 # Create local dir and upload content to fakefs
640 os
.mkdir("{}example_local".format(path
))
641 os
.mkdir("{}example_local/directory".format(path
))
643 "{}example_local/directory/test_file".format(path
), "w+"
645 test_file
.write(file_content
)
646 fs
.reverse_sync("example_local")
648 assert fs
.fs
.upload_from_stream
.call_count
== 2
650 # first call to upload_from_stream, dir_name
651 dir_name
= "example_local/directory"
652 call_args_0
= fs
.fs
.upload_from_stream
.call_args_list
[0]
653 assert call_args_0
[0][0] == dir_name
654 assert call_args_0
[1].get("metadata").get("type") == "dir"
656 # second call to upload_from_stream, dir_name
657 file_name
= "example_local/directory/test_file"
658 call_args_1
= fs
.fs
.upload_from_stream
.call_args_list
[1]
659 assert call_args_1
[0][0] == file_name
660 assert call_args_1
[1].get("metadata").get("type") == "file"
663 subprocess
.call(["rm", "-rf", path
])