1 # Copyright 2019 Canonical
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
15 # For those usages not covered by the Apache License, Version 2.0 please
16 # contact: eduardo.sousa@canonical.com
26 from pymongo
import MongoClient
27 from gridfs
import GridFSBucket
29 from io
import BytesIO
31 from unittest
.mock
import Mock
33 from osm_common
.fsbase
import FsException
34 from osm_common
.fsmongo
import FsMongo
35 from pathlib
import Path
37 __author__
= "Eduardo Sousa <eduardo.sousa@canonical.com>"
41 return tempfile
.gettempdir() + '/'
48 @pytest.fixture(scope
="function", params
=[True, False])
49 def fs_mongo(request
, monkeypatch
):
50 def mock_mongoclient_constructor(a
, b
, c
):
53 def mock_mongoclient_getitem(a
, b
):
56 def mock_gridfs_constructor(a
, b
):
59 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
60 monkeypatch
.setattr(MongoClient
, '__getitem__', mock_mongoclient_getitem
)
61 monkeypatch
.setattr(GridFSBucket
, '__init__', mock_gridfs_constructor
)
62 fs
= FsMongo(lock
=request
.param
)
67 'collection': 'files'})
71 def generic_fs_exception_message(message
):
72 return "storage exception {}".format(message
)
75 def fs_connect_exception_message(path
):
76 return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path
)
79 def file_open_file_not_found_exception(storage
):
80 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
81 return "storage exception File {} does not exist".format(f
)
84 def file_open_io_exception(storage
):
85 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
86 return "storage exception File {} cannot be opened".format(f
)
89 def dir_ls_not_a_directory_exception(storage
):
90 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
91 return "storage exception File {} does not exist".format(f
)
94 def dir_ls_io_exception(storage
):
95 f
= storage
if isinstance(storage
, str) else '/'.join(storage
)
96 return "storage exception File {} cannot be opened".format(f
)
99 def file_delete_exception_message(storage
):
100 return "storage exception File {} does not exist".format(storage
)
103 def test_constructor_without_logger():
105 assert fs
.logger
== logging
.getLogger('fs')
106 assert fs
.path
is None
107 assert fs
.client
is None
111 def test_constructor_with_logger():
112 logger_name
= 'fs_mongo'
113 fs
= FsMongo(logger_name
=logger_name
)
114 assert fs
.logger
== logging
.getLogger(logger_name
)
115 assert fs
.path
is None
116 assert fs
.client
is None
120 def test_get_params(fs_mongo
, monkeypatch
):
121 def mock_gridfs_find(self
, search_query
, **kwargs
):
124 monkeypatch
.setattr(GridFSBucket
, 'find', mock_gridfs_find
)
125 params
= fs_mongo
.get_params()
126 assert len(params
) == 2
127 assert "fs" in params
128 assert "path" in params
129 assert params
["fs"] == "mongo"
130 assert params
["path"] == valid_path()
133 @pytest.mark
.parametrize("config, exp_logger, exp_path", [
136 'logger_name': 'fs_mongo',
137 'path': valid_path(),
138 'uri': 'mongo:27017',
139 'collection': 'files'
141 'fs_mongo', valid_path()
145 'logger_name': 'fs_mongo',
146 'path': valid_path(),
149 'collection': 'files'
151 'fs_mongo', valid_path()
155 'logger_name': 'fs_mongo',
156 'path': valid_path()[:-1],
157 'uri': 'mongo:27017',
158 'collection': 'files'
160 'fs_mongo', valid_path()
164 'logger_name': 'fs_mongo',
165 'path': valid_path()[:-1],
168 'collection': 'files'
170 'fs_mongo', valid_path()
174 'path': valid_path(),
175 'uri': 'mongo:27017',
176 'collection': 'files'
182 'path': valid_path(),
185 'collection': 'files'
191 'path': valid_path()[:-1],
192 'uri': 'mongo:27017',
193 'collection': 'files'
199 'path': valid_path()[:-1],
202 'collection': 'files'
206 def test_fs_connect_with_valid_config(config
, exp_logger
, exp_path
):
208 fs
.fs_connect(config
)
209 assert fs
.logger
== logging
.getLogger(exp_logger
)
210 assert fs
.path
== exp_path
211 assert type(fs
.client
) == MongoClient
212 assert type(fs
.fs
) == GridFSBucket
215 @pytest.mark
.parametrize("config, exp_exception_message", [
218 'logger_name': 'fs_mongo',
219 'path': invalid_path(),
220 'uri': 'mongo:27017',
221 'collection': 'files'
223 fs_connect_exception_message(invalid_path())
227 'logger_name': 'fs_mongo',
228 'path': invalid_path(),
231 'collection': 'files'
233 fs_connect_exception_message(invalid_path())
237 'logger_name': 'fs_mongo',
238 'path': invalid_path()[:-1],
239 'uri': 'mongo:27017',
240 'collection': 'files'
242 fs_connect_exception_message(invalid_path()[:-1])
246 'logger_name': 'fs_mongo',
247 'path': invalid_path()[:-1],
250 'collection': 'files'
252 fs_connect_exception_message(invalid_path()[:-1])
256 'path': invalid_path(),
257 'uri': 'mongo:27017',
258 'collection': 'files'
260 fs_connect_exception_message(invalid_path())
264 'path': invalid_path(),
267 'collection': 'files'
269 fs_connect_exception_message(invalid_path())
273 'path': invalid_path()[:-1],
274 'uri': 'mongo:27017',
275 'collection': 'files'
277 fs_connect_exception_message(invalid_path()[:-1])
281 'path': invalid_path()[:-1],
284 'collection': 'files'
286 fs_connect_exception_message(invalid_path()[:-1])
293 'collection': 'files'
295 generic_fs_exception_message(
296 "Invalid configuration param at '[storage]': path '/' is not writable"
299 def test_fs_connect_with_invalid_path(config
, exp_exception_message
):
301 with pytest
.raises(FsException
) as excinfo
:
302 fs
.fs_connect(config
)
303 assert str(excinfo
.value
) == exp_exception_message
306 @pytest.mark
.parametrize("config, exp_exception_message", [
309 'logger_name': 'fs_mongo',
310 'uri': 'mongo:27017',
311 'collection': 'files'
313 "Missing parameter \"path\""
317 'logger_name': 'fs_mongo',
320 'collection': 'files'
322 "Missing parameter \"path\""
326 'logger_name': 'fs_mongo',
327 'path': valid_path(),
328 'collection': 'files'
330 "Missing parameters: \"uri\" or \"host\" + \"port\""
334 'logger_name': 'fs_mongo',
335 'path': valid_path(),
337 'collection': 'files'
339 "Missing parameters: \"uri\" or \"host\" + \"port\""
343 'logger_name': 'fs_mongo',
344 'path': valid_path(),
346 'collection': 'files'
348 "Missing parameters: \"uri\" or \"host\" + \"port\""
352 'logger_name': 'fs_mongo',
353 'path': valid_path(),
356 "Missing parameter \"collection\""
360 'logger_name': 'fs_mongo',
361 'path': valid_path(),
365 "Missing parameter \"collection\""
367 def test_fs_connect_with_missing_parameters(config
, exp_exception_message
):
369 with pytest
.raises(FsException
) as excinfo
:
370 fs
.fs_connect(config
)
371 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
374 @pytest.mark
.parametrize("config, exp_exception_message", [
377 'logger_name': 'fs_mongo',
378 'path': valid_path(),
379 'uri': 'mongo:27017',
380 'collection': 'files'
382 "MongoClient crashed"
386 'logger_name': 'fs_mongo',
387 'path': valid_path(),
390 'collection': 'files'
392 "MongoClient crashed"
394 def test_fs_connect_with_invalid_mongoclient(config
, exp_exception_message
, monkeypatch
):
395 def generate_exception(a
, b
, c
=None):
396 raise Exception(exp_exception_message
)
398 monkeypatch
.setattr(MongoClient
, '__init__', generate_exception
)
401 with pytest
.raises(FsException
) as excinfo
:
402 fs
.fs_connect(config
)
403 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
406 @pytest.mark
.parametrize("config, exp_exception_message", [
409 'logger_name': 'fs_mongo',
410 'path': valid_path(),
411 'uri': 'mongo:27017',
412 'collection': 'files'
414 "Collection unavailable"
418 'logger_name': 'fs_mongo',
419 'path': valid_path(),
422 'collection': 'files'
424 "Collection unavailable"
426 def test_fs_connect_with_invalid_mongo_collection(config
, exp_exception_message
, monkeypatch
):
427 def mock_mongoclient_constructor(a
, b
, c
=None):
430 def generate_exception(a
, b
):
431 raise Exception(exp_exception_message
)
433 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
434 monkeypatch
.setattr(MongoClient
, '__getitem__', generate_exception
)
437 with pytest
.raises(FsException
) as excinfo
:
438 fs
.fs_connect(config
)
439 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
442 @pytest.mark
.parametrize("config, exp_exception_message", [
445 'logger_name': 'fs_mongo',
446 'path': valid_path(),
447 'uri': 'mongo:27017',
448 'collection': 'files'
450 "GridFsBucket crashed"
454 'logger_name': 'fs_mongo',
455 'path': valid_path(),
458 'collection': 'files'
460 "GridFsBucket crashed"
462 def test_fs_connect_with_invalid_gridfsbucket(config
, exp_exception_message
, monkeypatch
):
463 def mock_mongoclient_constructor(a
, b
, c
=None):
466 def mock_mongoclient_getitem(a
, b
):
469 def generate_exception(a
, b
):
470 raise Exception(exp_exception_message
)
472 monkeypatch
.setattr(MongoClient
, '__init__', mock_mongoclient_constructor
)
473 monkeypatch
.setattr(MongoClient
, '__getitem__', mock_mongoclient_getitem
)
474 monkeypatch
.setattr(GridFSBucket
, '__init__', generate_exception
)
477 with pytest
.raises(FsException
) as excinfo
:
478 fs
.fs_connect(config
)
479 assert str(excinfo
.value
) == generic_fs_exception_message(exp_exception_message
)
482 def test_fs_disconnect(fs_mongo
):
483 fs_mongo
.fs_disconnect()
488 # ├── directory
491 # ├── directory_link -> ../directory/
492 # └── file_link -> ../directory/file
494 def __init__(self
, id, filename
, metadata
):
496 self
.filename
= filename
497 self
.metadata
= metadata
501 directory_metadata
= {'type': 'dir', 'permissions': 509}
502 file_metadata
= {'type': 'file', 'permissions': 436}
503 symlink_metadata
= {'type': 'sym', 'permissions': 511}
507 "cursor": FakeCursor(1, 'example_tar', directory_metadata
),
508 "metadata": directory_metadata
,
509 "stream_content": b
'',
510 "stream_content_bad": b
"Something",
511 "path": './tmp/example_tar',
514 "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata
),
515 "metadata": directory_metadata
,
516 "stream_content": b
'',
517 "stream_content_bad": b
"Something",
518 "path": './tmp/example_tar/directory',
521 "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata
),
522 "metadata": directory_metadata
,
523 "stream_content": b
'',
524 "stream_content_bad": b
"Something",
525 "path": './tmp/example_tar/symlinks',
528 "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata
),
529 "metadata": file_metadata
,
530 "stream_content": b
"Example test",
531 "stream_content_bad": b
"Example test2",
532 "path": './tmp/example_tar/directory/file',
535 "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata
),
536 "metadata": symlink_metadata
,
537 "stream_content": b
"../directory/file",
538 "stream_content_bad": b
"",
539 "path": './tmp/example_tar/symlinks/file_link',
542 "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata
),
543 "metadata": symlink_metadata
,
544 "stream_content": b
"../directory/",
545 "stream_content_bad": b
"",
546 "path": './tmp/example_tar/symlinks/directory_link',
550 def upload_from_stream(self
, f
, stream
, metadata
=None):
552 for i
, v
in self
.tar_info
.items():
554 assert metadata
["type"] == v
["metadata"]["type"]
555 assert stream
.read() == BytesIO(v
["stream_content"]).read()
557 assert stream
.read() != BytesIO(v
["stream_content_bad"]).read()
562 def find(self
, type, no_cursor_timeout
=True, sort
=None):
564 for i
, v
in self
.tar_info
.items():
565 if type["metadata.type"] == "dir":
566 if v
["metadata"] == self
.directory_metadata
:
567 list.append(v
["cursor"])
569 if v
["metadata"] != self
.directory_metadata
:
570 list.append(v
["cursor"])
573 def download_to_stream(self
, id, file_stream
):
574 file_stream
.write(BytesIO(self
.tar_info
[id]["stream_content"]).read())
577 def test_file_extract():
578 tar_path
= "tmp/Example.tar.gz"
579 folder_path
= "tmp/example_tar"
582 subprocess
.call(["rm", "-rf", "./tmp"])
583 subprocess
.call(["mkdir", "-p", "{}/directory".format(folder_path
)])
584 subprocess
.call(["mkdir", "-p", "{}/symlinks".format(folder_path
)])
585 p
= Path("{}/directory/file".format(folder_path
))
586 p
.write_text("Example test")
587 os
.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path
))
588 os
.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path
))
589 if os
.path
.exists(tar_path
):
591 subprocess
.call(["tar", "-czvf", tar_path
, folder_path
])
594 tar
= tarfile
.open(tar_path
, "r")
597 fs
.file_extract(tar_object
=tar
, path
=".")
600 subprocess
.call(["rm", "-rf", "./tmp"])
603 def test_upload_local_fs():
606 subprocess
.call(["rm", "-rf", path
])
612 assert os
.path
.isdir("{}example_tar".format(path
))
613 assert os
.path
.isdir("{}example_tar/directory".format(path
))
614 assert os
.path
.isdir("{}example_tar/symlinks".format(path
))
615 assert os
.path
.isfile("{}example_tar/directory/file".format(path
))
616 assert os
.path
.islink("{}example_tar/symlinks/file_link".format(path
))
617 assert os
.path
.islink("{}example_tar/symlinks/directory_link".format(path
))
619 subprocess
.call(["rm", "-rf", path
])
622 def test_upload_mongo_fs():
625 subprocess
.call(["rm", "-rf", path
])
630 fs
.fs
.find
.return_value
= {}
632 file_content
= "Test file content"
634 # Create local dir and upload content to fakefs
636 os
.mkdir("{}example_local".format(path
))
637 os
.mkdir("{}example_local/directory".format(path
))
638 with
open("{}example_local/directory/test_file".format(path
), "w+") as test_file
:
639 test_file
.write(file_content
)
640 fs
.reverse_sync("example_local")
642 assert fs
.fs
.upload_from_stream
.call_count
== 2
644 # first call to upload_from_stream, dir_name
645 dir_name
= "example_local/directory"
646 call_args_0
= fs
.fs
.upload_from_stream
.call_args_list
[0]
647 assert call_args_0
[0][0] == dir_name
648 assert call_args_0
[1].get("metadata").get("type") == "dir"
650 # second call to upload_from_stream, dir_name
651 file_name
= "example_local/directory/test_file"
652 call_args_1
= fs
.fs
.upload_from_stream
.call_args_list
[1]
653 assert call_args_1
[0][0] == file_name
654 assert call_args_1
[1].get("metadata").get("type") == "file"
657 subprocess
.call(["rm", "-rf", path
])