1 # Copyright 2019 Canonical
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # For those usages not covered by the Apache License, Version 2.0 please
16 # contact: eduardo.sousa@canonical.com
26 from pymongo
import MongoClient
27 from gridfs
import GridFSBucket
29 from io
import BytesIO
31 from osm_common
.fsbase
import FsException
32 from osm_common
.fsmongo
import FsMongo
33 from pathlib
import Path
35 __author__
= "Eduardo Sousa <eduardo.sousa@canonical.com>"
39 return tempfile
.gettempdir() + '/'
46 @pytest.fixture(scope
="function", params
=[True, False])
47 def fs_mongo(request
, monkeypatch
):
48 def mock_mongoclient_constructor(a
, b
, c
):
51 def mock_mongoclient_getitem(a
, b
):
54 def mock_gridfs_constructor(a
, b
):
57 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
58 monkeypatch
.setattr(MongoClient
, '__getitem__', mock_mongoclient_getitem
)
59 monkeypatch
.setattr(GridFSBucket
, '__init__', mock_gridfs_constructor
)
60 fs
= FsMongo(lock
=request
.param
)
65 'collection': 'files'})
69 def generic_fs_exception_message(message
):
70 return "storage exception {}".format(message
)
73 def fs_connect_exception_message(path
):
74 return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path
)
77 def file_open_file_not_found_exception(storage
):
78 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
79 return "storage exception File {} does not exist".format(f
)
82 def file_open_io_exception(storage
):
83 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
84 return "storage exception File {} cannot be opened".format(f
)
87 def dir_ls_not_a_directory_exception(storage
):
88 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
89 return "storage exception File {} does not exist".format(f
)
92 def dir_ls_io_exception(storage
):
93 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
94 return "storage exception File {} cannot be opened".format(f
)
97 def file_delete_exception_message(storage
):
98 return "storage exception File {} does not exist".format(storage
)
101 def test_constructor_without_logger():
103 assert fs
.logger
== logging
.getLogger('fs')
104 assert fs
.path
is None
105 assert fs
.client
is None
109 def test_constructor_with_logger():
110 logger_name
= 'fs_mongo'
111 fs
= FsMongo(logger_name
=logger_name
)
112 assert fs
.logger
== logging
.getLogger(logger_name
)
113 assert fs
.path
is None
114 assert fs
.client
is None
118 def test_get_params(fs_mongo
, monkeypatch
):
119 def mock_gridfs_find(self
, search_query
, **kwargs
):
122 monkeypatch
.setattr(GridFSBucket
, 'find', mock_gridfs_find
)
123 params
= fs_mongo
.get_params()
124 assert len(params
) == 2
125 assert "fs" in params
126 assert "path" in params
127 assert params
["fs"] == "mongo"
128 assert params
["path"] == valid_path()
131 @pytest.mark
.parametrize("config, exp_logger, exp_path", [
134 'logger_name': 'fs_mongo',
135 'path': valid_path(),
136 'uri': 'mongo:27017',
137 'collection': 'files'
139 'fs_mongo', valid_path()
143 'logger_name': 'fs_mongo',
144 'path': valid_path(),
147 'collection': 'files'
149 'fs_mongo', valid_path()
153 'logger_name': 'fs_mongo',
154 'path': valid_path()[:-1],
155 'uri': 'mongo:27017',
156 'collection': 'files'
158 'fs_mongo', valid_path()
162 'logger_name': 'fs_mongo',
163 'path': valid_path()[:-1],
166 'collection': 'files'
168 'fs_mongo', valid_path()
172 'path': valid_path(),
173 'uri': 'mongo:27017',
174 'collection': 'files'
180 'path': valid_path(),
183 'collection': 'files'
189 'path': valid_path()[:-1],
190 'uri': 'mongo:27017',
191 'collection': 'files'
197 'path': valid_path()[:-1],
200 'collection': 'files'
204 def test_fs_connect_with_valid_config(config
, exp_logger
, exp_path
):
206 fs
.fs_connect(config
)
207 assert fs
.logger
== logging
.getLogger(exp_logger
)
208 assert fs
.path
== exp_path
209 assert type(fs
.client
) == MongoClient
210 assert type(fs
.fs
) == GridFSBucket
213 @pytest.mark
.parametrize("config, exp_exception_message", [
216 'logger_name': 'fs_mongo',
217 'path': invalid_path(),
218 'uri': 'mongo:27017',
219 'collection': 'files'
221 fs_connect_exception_message(invalid_path())
225 'logger_name': 'fs_mongo',
226 'path': invalid_path(),
229 'collection': 'files'
231 fs_connect_exception_message(invalid_path())
235 'logger_name': 'fs_mongo',
236 'path': invalid_path()[:-1],
237 'uri': 'mongo:27017',
238 'collection': 'files'
240 fs_connect_exception_message(invalid_path()[:-1])
244 'logger_name': 'fs_mongo',
245 'path': invalid_path()[:-1],
248 'collection': 'files'
250 fs_connect_exception_message(invalid_path()[:-1])
254 'path': invalid_path(),
255 'uri': 'mongo:27017',
256 'collection': 'files'
258 fs_connect_exception_message(invalid_path())
262 'path': invalid_path(),
265 'collection': 'files'
267 fs_connect_exception_message(invalid_path())
271 'path': invalid_path()[:-1],
272 'uri': 'mongo:27017',
273 'collection': 'files'
275 fs_connect_exception_message(invalid_path()[:-1])
279 'path': invalid_path()[:-1],
282 'collection': 'files'
284 fs_connect_exception_message(invalid_path()[:-1])
291 'collection': 'files'
293 generic_fs_exception_message(
294 "Invalid configuration param at '[storage]': path '/' is not writable"
297 def test_fs_connect_with_invalid_path(config
, exp_exception_message
):
299 with pytest
.raises(FsException
) as excinfo
:
300 fs
.fs_connect(config
)
301 assert str(excinfo
.value
) == exp_exception_message
304 @pytest.mark
.parametrize("config, exp_exception_message", [
307 'logger_name': 'fs_mongo',
308 'uri': 'mongo:27017',
309 'collection': 'files'
311 "Missing parameter \"path\""
315 'logger_name': 'fs_mongo',
318 'collection': 'files'
320 "Missing parameter \"path\""
324 'logger_name': 'fs_mongo',
325 'path': valid_path(),
326 'collection': 'files'
328 "Missing parameters: \"uri\" or \"host\" + \"port\""
332 'logger_name': 'fs_mongo',
333 'path': valid_path(),
335 'collection': 'files'
337 "Missing parameters: \"uri\" or \"host\" + \"port\""
341 'logger_name': 'fs_mongo',
342 'path': valid_path(),
344 'collection': 'files'
346 "Missing parameters: \"uri\" or \"host\" + \"port\""
350 'logger_name': 'fs_mongo',
351 'path': valid_path(),
354 "Missing parameter \"collection\""
358 'logger_name': 'fs_mongo',
359 'path': valid_path(),
363 "Missing parameter \"collection\""
365 def test_fs_connect_with_missing_parameters(config
, exp_exception_message
):
367 with pytest
.raises(FsException
) as excinfo
:
368 fs
.fs_connect(config
)
369 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
372 @pytest.mark
.parametrize("config, exp_exception_message", [
375 'logger_name': 'fs_mongo',
376 'path': valid_path(),
377 'uri': 'mongo:27017',
378 'collection': 'files'
380 "MongoClient crashed"
384 'logger_name': 'fs_mongo',
385 'path': valid_path(),
388 'collection': 'files'
390 "MongoClient crashed"
392 def test_fs_connect_with_invalid_mongoclient(config
, exp_exception_message
, monkeypatch
):
393 def generate_exception(a
, b
, c
=None):
394 raise Exception(exp_exception_message
)
396 monkeypatch
.setattr(MongoClient
, '__init__', generate_exception
)
399 with pytest
.raises(FsException
) as excinfo
:
400 fs
.fs_connect(config
)
401 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
404 @pytest.mark
.parametrize("config, exp_exception_message", [
407 'logger_name': 'fs_mongo',
408 'path': valid_path(),
409 'uri': 'mongo:27017',
410 'collection': 'files'
412 "Collection unavailable"
416 'logger_name': 'fs_mongo',
417 'path': valid_path(),
420 'collection': 'files'
422 "Collection unavailable"
424 def test_fs_connect_with_invalid_mongo_collection(config
, exp_exception_message
, monkeypatch
):
425 def mock_mongoclient_constructor(a
, b
, c
=None):
428 def generate_exception(a
, b
):
429 raise Exception(exp_exception_message
)
431 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
432 monkeypatch
.setattr(MongoClient
, '__getitem__', generate_exception
)
435 with pytest
.raises(FsException
) as excinfo
:
436 fs
.fs_connect(config
)
437 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
440 @pytest.mark
.parametrize("config, exp_exception_message", [
443 'logger_name': 'fs_mongo',
444 'path': valid_path(),
445 'uri': 'mongo:27017',
446 'collection': 'files'
448 "GridFsBucket crashed"
452 'logger_name': 'fs_mongo',
453 'path': valid_path(),
456 'collection': 'files'
458 "GridFsBucket crashed"
460 def test_fs_connect_with_invalid_gridfsbucket(config
, exp_exception_message
, monkeypatch
):
461 def mock_mongoclient_constructor(a
, b
, c
=None):
464 def mock_mongoclient_getitem(a
, b
):
467 def generate_exception(a
, b
):
468 raise Exception(exp_exception_message
)
470 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
471 monkeypatch
.setattr(MongoClient
, '__getitem__', mock_mongoclient_getitem
)
472 monkeypatch
.setattr(GridFSBucket
, '__init__', generate_exception
)
475 with pytest
.raises(FsException
) as excinfo
:
476 fs
.fs_connect(config
)
477 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
480 def test_fs_disconnect(fs_mongo
):
481 fs_mongo
.fs_disconnect()
489 # ├── directory_link -> ../directory/
490 # └── file_link -> ../directory/file
492 def __init__(self
, id, filename
, metadata
):
494 self
.filename
= filename
495 self
.metadata
= metadata
499 directory_metadata
= {'type': 'dir', 'permissions': 509}
500 file_metadata
= {'type': 'file', 'permissions': 436}
501 symlink_metadata
= {'type': 'sym', 'permissions': 511}
505 "cursor": FakeCursor(1, 'example_tar', directory_metadata
),
506 "metadata": directory_metadata
,
507 "stream_content": b
'',
508 "stream_content_bad": b
"Something",
509 "path": './tmp/example_tar',
512 "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata
),
513 "metadata": directory_metadata
,
514 "stream_content": b
'',
515 "stream_content_bad": b
"Something",
516 "path": './tmp/example_tar/directory',
519 "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata
),
520 "metadata": directory_metadata
,
521 "stream_content": b
'',
522 "stream_content_bad": b
"Something",
523 "path": './tmp/example_tar/symlinks',
526 "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata
),
527 "metadata": file_metadata
,
528 "stream_content": b
"Example test",
529 "stream_content_bad": b
"Example test2",
530 "path": './tmp/example_tar/directory/file',
533 "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata
),
534 "metadata": symlink_metadata
,
535 "stream_content": b
"../directory/file",
536 "stream_content_bad": b
"",
537 "path": './tmp/example_tar/symlinks/file_link',
540 "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata
),
541 "metadata": symlink_metadata
,
542 "stream_content": b
"../directory/",
543 "stream_content_bad": b
"",
544 "path": './tmp/example_tar/symlinks/directory_link',
548 def upload_from_stream(self
, f
, stream
, metadata
=None):
550 for i
, v
in self
.tar_info
.items():
552 assert metadata
["type"] == v
["metadata"]["type"]
553 assert stream
.read() == BytesIO(v
["stream_content"]).read()
555 assert stream
.read() != BytesIO(v
["stream_content_bad"]).read()
560 def find(self
, type, no_cursor_timeout
=True):
562 for i
, v
in self
.tar_info
.items():
563 if type["metadata.type"] == "dir":
564 if v
["metadata"] == self
.directory_metadata
:
565 list.append(v
["cursor"])
567 if v
["metadata"] != self
.directory_metadata
:
568 list.append(v
["cursor"])
571 def download_to_stream(self
, id, file_stream
):
572 file_stream
.write(BytesIO(self
.tar_info
[id]["stream_content"]).read())
575 def test_file_extract():
576 tar_path
= "tmp/Example.tar.gz"
577 folder_path
= "tmp/example_tar"
580 subprocess
.call(["rm", "-rf", "./tmp"])
581 subprocess
.call(["mkdir", "-p", "{}/directory".format(folder_path
)])
582 subprocess
.call(["mkdir", "-p", "{}/symlinks".format(folder_path
)])
583 p
= Path("{}/directory/file".format(folder_path
))
584 p
.write_text("Example test")
585 os
.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path
))
586 os
.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path
))
587 if os
.path
.exists(tar_path
):
589 subprocess
.call(["tar", "-czvf", tar_path
, folder_path
])
592 tar
= tarfile
.open(tar_path
, "r")
595 fs
.file_extract(tar_object
=tar
, path
=".")
598 subprocess
.call(["rm", "-rf", "./tmp"])
601 def test_upload_local_fs():
604 subprocess
.call(["rm", "-rf", path
])
610 assert os
.path
.isdir("{}example_tar".format(path
))
611 assert os
.path
.isdir("{}example_tar/directory".format(path
))
612 assert os
.path
.isdir("{}example_tar/symlinks".format(path
))
613 assert os
.path
.isfile("{}example_tar/directory/file".format(path
))
614 assert os
.path
.islink("{}example_tar/symlinks/file_link".format(path
))
615 assert os
.path
.islink("{}example_tar/symlinks/directory_link".format(path
))
617 subprocess
.call(["rm", "-rf", path
])