osm_common/tests/test_fsmongo.py

   1 # Copyright 2019 Canonical
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 #
  15 # For those usages not covered by the Apache License, Version 2.0 please
  16 # contact: eduardo.sousa@canonical.com
  17 ##
  18
  19 import logging
  20 import pytest
  21 import tempfile
  22 import tarfile
  23 import os
  24 import subprocess
  25
  26 from pymongo import MongoClient
  27 from gridfs import GridFSBucket
  28
  29 from io import BytesIO
  30
  31 from osm_common.fsbase import FsException
  32 from osm_common.fsmongo import FsMongo
  33 from pathlib import Path
  34
  35 __author__ = "Eduardo Sousa <eduardo.sousa@canonical.com>"
  36
  37
  38 def valid_path():
  39     return tempfile.gettempdir() + '/'
  40
  41
  42 def invalid_path():
  43     return '/#tweeter/'
  44
  45
  46 @pytest.fixture(scope="function", params=[True, False])
  47 def fs_mongo(request, monkeypatch):
  48     def mock_mongoclient_constructor(a, b, c):
  49         pass
  50
  51     def mock_mongoclient_getitem(a, b):
  52         pass
  53
  54     def mock_gridfs_constructor(a, b):
  55         pass
  56
  57     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
  58     monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem)
  59     monkeypatch.setattr(GridFSBucket, '__init__', mock_gridfs_constructor)
  60     fs = FsMongo(lock=request.param)
  61     fs.fs_connect({
  62         'path': valid_path(),
  63         'host': 'mongo',
  64         'port': 27017,
  65         'collection': 'files'})
  66     return fs
  67
  68
  69 def generic_fs_exception_message(message):
  70     return "storage exception {}".format(message)
  71
  72
  73 def fs_connect_exception_message(path):
  74     return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path)
  75
  76
  77 def file_open_file_not_found_exception(storage):
  78     f = storage if isinstance(storage, str) else '/'.join(storage)
  79     return "storage exception File {} does not exist".format(f)
  80
  81
  82 def file_open_io_exception(storage):
  83     f = storage if isinstance(storage, str) else '/'.join(storage)
  84     return "storage exception File {} cannot be opened".format(f)
  85
  86
  87 def dir_ls_not_a_directory_exception(storage):
  88     f = storage if isinstance(storage, str) else '/'.join(storage)
  89     return "storage exception File {} does not exist".format(f)
  90
  91
  92 def dir_ls_io_exception(storage):
  93     f = storage if isinstance(storage, str) else '/'.join(storage)
  94     return "storage exception File {} cannot be opened".format(f)
  95
  96
  97 def file_delete_exception_message(storage):
  98     return "storage exception File {} does not exist".format(storage)
  99
 100
 101 def test_constructor_without_logger():
 102     fs = FsMongo()
 103     assert fs.logger == logging.getLogger('fs')
 104     assert fs.path is None
 105     assert fs.client is None
 106     assert fs.fs is None
 107
 108
 109 def test_constructor_with_logger():
 110     logger_name = 'fs_mongo'
 111     fs = FsMongo(logger_name=logger_name)
 112     assert fs.logger == logging.getLogger(logger_name)
 113     assert fs.path is None
 114     assert fs.client is None
 115     assert fs.fs is None
 116
 117
 118 def test_get_params(fs_mongo, monkeypatch):
 119     def mock_gridfs_find(self, search_query, **kwargs):
 120         return []
 121
 122     monkeypatch.setattr(GridFSBucket, 'find', mock_gridfs_find)
 123     params = fs_mongo.get_params()
 124     assert len(params) == 2
 125     assert "fs" in params
 126     assert "path" in params
 127     assert params["fs"] == "mongo"
 128     assert params["path"] == valid_path()
 129
 130
 131 @pytest.mark.parametrize("config, exp_logger, exp_path", [
 132     (
 133         {
 134             'logger_name': 'fs_mongo',
 135             'path': valid_path(),
 136             'uri': 'mongo:27017',
 137             'collection': 'files'
 138         },
 139         'fs_mongo', valid_path()
 140     ),
 141     (
 142         {
 143             'logger_name': 'fs_mongo',
 144             'path': valid_path(),
 145             'host': 'mongo',
 146             'port': 27017,
 147             'collection': 'files'
 148         },
 149         'fs_mongo', valid_path()
 150     ),
 151     (
 152         {
 153             'logger_name': 'fs_mongo',
 154             'path': valid_path()[:-1],
 155             'uri': 'mongo:27017',
 156             'collection': 'files'
 157         },
 158         'fs_mongo', valid_path()
 159     ),
 160     (
 161         {
 162             'logger_name': 'fs_mongo',
 163             'path': valid_path()[:-1],
 164             'host': 'mongo',
 165             'port': 27017,
 166             'collection': 'files'
 167         },
 168         'fs_mongo', valid_path()
 169     ),
 170     (
 171         {
 172             'path': valid_path(),
 173             'uri': 'mongo:27017',
 174             'collection': 'files'
 175         },
 176         'fs', valid_path()
 177     ),
 178     (
 179         {
 180             'path': valid_path(),
 181             'host': 'mongo',
 182             'port': 27017,
 183             'collection': 'files'
 184         },
 185         'fs', valid_path()
 186     ),
 187     (
 188         {
 189             'path': valid_path()[:-1],
 190             'uri': 'mongo:27017',
 191             'collection': 'files'
 192         },
 193         'fs', valid_path()
 194     ),
 195     (
 196         {
 197             'path': valid_path()[:-1],
 198             'host': 'mongo',
 199             'port': 27017,
 200             'collection': 'files'
 201         },
 202         'fs', valid_path()
 203     )])
 204 def test_fs_connect_with_valid_config(config, exp_logger, exp_path):
 205     fs = FsMongo()
 206     fs.fs_connect(config)
 207     assert fs.logger == logging.getLogger(exp_logger)
 208     assert fs.path == exp_path
 209     assert type(fs.client) == MongoClient
 210     assert type(fs.fs) == GridFSBucket
 211
 212
 213 @pytest.mark.parametrize("config, exp_exception_message", [
 214     (
 215         {
 216             'logger_name': 'fs_mongo',
 217             'path': invalid_path(),
 218             'uri': 'mongo:27017',
 219             'collection': 'files'
 220         },
 221         fs_connect_exception_message(invalid_path())
 222     ),
 223     (
 224         {
 225             'logger_name': 'fs_mongo',
 226             'path': invalid_path(),
 227             'host': 'mongo',
 228             'port': 27017,
 229             'collection': 'files'
 230         },
 231         fs_connect_exception_message(invalid_path())
 232     ),
 233     (
 234         {
 235             'logger_name': 'fs_mongo',
 236             'path': invalid_path()[:-1],
 237             'uri': 'mongo:27017',
 238             'collection': 'files'
 239         },
 240         fs_connect_exception_message(invalid_path()[:-1])
 241     ),
 242     (
 243         {
 244             'logger_name': 'fs_mongo',
 245             'path': invalid_path()[:-1],
 246             'host': 'mongo',
 247             'port': 27017,
 248             'collection': 'files'
 249         },
 250         fs_connect_exception_message(invalid_path()[:-1])
 251     ),
 252     (
 253         {
 254             'path': invalid_path(),
 255             'uri': 'mongo:27017',
 256             'collection': 'files'
 257         },
 258         fs_connect_exception_message(invalid_path())
 259     ),
 260     (
 261         {
 262             'path': invalid_path(),
 263             'host': 'mongo',
 264             'port': 27017,
 265             'collection': 'files'
 266         },
 267         fs_connect_exception_message(invalid_path())
 268     ),
 269     (
 270         {
 271             'path': invalid_path()[:-1],
 272             'uri': 'mongo:27017',
 273             'collection': 'files'
 274         },
 275         fs_connect_exception_message(invalid_path()[:-1])
 276     ),
 277     (
 278         {
 279             'path': invalid_path()[:-1],
 280             'host': 'mongo',
 281             'port': 27017,
 282             'collection': 'files'
 283         },
 284         fs_connect_exception_message(invalid_path()[:-1])
 285     ),
 286     (
 287         {
 288             'path': '/',
 289             'host': 'mongo',
 290             'port': 27017,
 291             'collection': 'files'
 292         },
 293         generic_fs_exception_message(
 294             "Invalid configuration param at '[storage]': path '/' is not writable"
 295         )
 296     )])
 297 def test_fs_connect_with_invalid_path(config, exp_exception_message):
 298     fs = FsMongo()
 299     with pytest.raises(FsException) as excinfo:
 300         fs.fs_connect(config)
 301     assert str(excinfo.value) == exp_exception_message
 302
 303
 304 @pytest.mark.parametrize("config, exp_exception_message", [
 305     (
 306         {
 307             'logger_name': 'fs_mongo',
 308             'uri': 'mongo:27017',
 309             'collection': 'files'
 310         },
 311         "Missing parameter \"path\""
 312     ),
 313     (
 314         {
 315             'logger_name': 'fs_mongo',
 316             'host': 'mongo',
 317             'port': 27017,
 318             'collection': 'files'
 319         },
 320         "Missing parameter \"path\""
 321     ),
 322     (
 323         {
 324             'logger_name': 'fs_mongo',
 325             'path': valid_path(),
 326             'collection': 'files'
 327         },
 328         "Missing parameters: \"uri\" or \"host\" + \"port\""
 329     ),
 330     (
 331         {
 332             'logger_name': 'fs_mongo',
 333             'path': valid_path(),
 334             'port': 27017,
 335             'collection': 'files'
 336         },
 337         "Missing parameters: \"uri\" or \"host\" + \"port\""
 338     ),
 339     (
 340         {
 341             'logger_name': 'fs_mongo',
 342             'path': valid_path(),
 343             'host': 'mongo',
 344             'collection': 'files'
 345         },
 346         "Missing parameters: \"uri\" or \"host\" + \"port\""
 347     ),
 348     (
 349         {
 350             'logger_name': 'fs_mongo',
 351             'path': valid_path(),
 352             'uri': 'mongo:27017'
 353         },
 354         "Missing parameter \"collection\""
 355     ),
 356     (
 357         {
 358             'logger_name': 'fs_mongo',
 359             'path': valid_path(),
 360             'host': 'mongo',
 361             'port': 27017,
 362         },
 363         "Missing parameter \"collection\""
 364     )])
 365 def test_fs_connect_with_missing_parameters(config, exp_exception_message):
 366     fs = FsMongo()
 367     with pytest.raises(FsException) as excinfo:
 368         fs.fs_connect(config)
 369     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 370
 371
 372 @pytest.mark.parametrize("config, exp_exception_message", [
 373     (
 374         {
 375             'logger_name': 'fs_mongo',
 376             'path': valid_path(),
 377             'uri': 'mongo:27017',
 378             'collection': 'files'
 379         },
 380         "MongoClient crashed"
 381     ),
 382     (
 383         {
 384             'logger_name': 'fs_mongo',
 385             'path': valid_path(),
 386             'host': 'mongo',
 387             'port': 27017,
 388             'collection': 'files'
 389         },
 390         "MongoClient crashed"
 391     )])
 392 def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monkeypatch):
 393     def generate_exception(a, b, c=None):
 394         raise Exception(exp_exception_message)
 395
 396     monkeypatch.setattr(MongoClient, '__init__', generate_exception)
 397
 398     fs = FsMongo()
 399     with pytest.raises(FsException) as excinfo:
 400         fs.fs_connect(config)
 401     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 402
 403
 404 @pytest.mark.parametrize("config, exp_exception_message", [
 405     (
 406         {
 407             'logger_name': 'fs_mongo',
 408             'path': valid_path(),
 409             'uri': 'mongo:27017',
 410             'collection': 'files'
 411         },
 412         "Collection unavailable"
 413     ),
 414     (
 415         {
 416             'logger_name': 'fs_mongo',
 417             'path': valid_path(),
 418             'host': 'mongo',
 419             'port': 27017,
 420             'collection': 'files'
 421         },
 422         "Collection unavailable"
 423     )])
 424 def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, monkeypatch):
 425     def mock_mongoclient_constructor(a, b, c=None):
 426         pass
 427
 428     def generate_exception(a, b):
 429         raise Exception(exp_exception_message)
 430
 431     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
 432     monkeypatch.setattr(MongoClient, '__getitem__', generate_exception)
 433
 434     fs = FsMongo()
 435     with pytest.raises(FsException) as excinfo:
 436         fs.fs_connect(config)
 437     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 438
 439
 440 @pytest.mark.parametrize("config, exp_exception_message", [
 441     (
 442         {
 443             'logger_name': 'fs_mongo',
 444             'path': valid_path(),
 445             'uri': 'mongo:27017',
 446             'collection': 'files'
 447         },
 448         "GridFsBucket crashed"
 449     ),
 450     (
 451         {
 452             'logger_name': 'fs_mongo',
 453             'path': valid_path(),
 454             'host': 'mongo',
 455             'port': 27017,
 456             'collection': 'files'
 457         },
 458         "GridFsBucket crashed"
 459     )])
 460 def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, monkeypatch):
 461     def mock_mongoclient_constructor(a, b, c=None):
 462         pass
 463
 464     def mock_mongoclient_getitem(a, b):
 465         pass
 466
 467     def generate_exception(a, b):
 468         raise Exception(exp_exception_message)
 469
 470     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
 471     monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem)
 472     monkeypatch.setattr(GridFSBucket, '__init__', generate_exception)
 473
 474     fs = FsMongo()
 475     with pytest.raises(FsException) as excinfo:
 476         fs.fs_connect(config)
 477     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 478
 479
 480 def test_fs_disconnect(fs_mongo):
 481     fs_mongo.fs_disconnect()
 482
 483
 484 # Example.tar.gz
 485 # example_tar/
 486 # ├── directory
 487 # │   └── file
 488 # └── symlinks
 489 #     ├── directory_link -> ../directory/
 490 #     └── file_link -> ../directory/file
 491 class FakeCursor:
 492     def __init__(self, id, filename, metadata):
 493         self._id = id
 494         self.filename = filename
 495         self.metadata = metadata
 496
 497
 498 class FakeFS:
 499     directory_metadata = {'type': 'dir', 'permissions': 509}
 500     file_metadata = {'type': 'file', 'permissions': 436}
 501     symlink_metadata = {'type': 'sym', 'permissions': 511}
 502
 503     tar_info = {
 504         1: {
 505             "cursor": FakeCursor(1, 'example_tar', directory_metadata),
 506             "metadata": directory_metadata,
 507             "stream_content": b'',
 508             "stream_content_bad": b"Something",
 509             "path": './tmp/example_tar',
 510         },
 511         2: {
 512             "cursor": FakeCursor(2, 'example_tar/directory', directory_metadata),
 513             "metadata": directory_metadata,
 514             "stream_content": b'',
 515             "stream_content_bad": b"Something",
 516             "path": './tmp/example_tar/directory',
 517         },
 518         3: {
 519             "cursor": FakeCursor(3, 'example_tar/symlinks', directory_metadata),
 520             "metadata": directory_metadata,
 521             "stream_content": b'',
 522             "stream_content_bad": b"Something",
 523             "path": './tmp/example_tar/symlinks',
 524         },
 525         4: {
 526             "cursor": FakeCursor(4, 'example_tar/directory/file', file_metadata),
 527             "metadata": file_metadata,
 528             "stream_content": b"Example test",
 529             "stream_content_bad": b"Example test2",
 530             "path": './tmp/example_tar/directory/file',
 531         },
 532         5: {
 533             "cursor": FakeCursor(5, 'example_tar/symlinks/file_link', symlink_metadata),
 534             "metadata": symlink_metadata,
 535             "stream_content": b"../directory/file",
 536             "stream_content_bad": b"",
 537             "path": './tmp/example_tar/symlinks/file_link',
 538         },
 539         6: {
 540             "cursor": FakeCursor(6, 'example_tar/symlinks/directory_link', symlink_metadata),
 541             "metadata": symlink_metadata,
 542             "stream_content": b"../directory/",
 543             "stream_content_bad": b"",
 544             "path": './tmp/example_tar/symlinks/directory_link',
 545         }
 546     }
 547
 548     def upload_from_stream(self, f, stream, metadata=None):
 549         found = False
 550         for i, v in self.tar_info.items():
 551             if f == v["path"]:
 552                 assert metadata["type"] == v["metadata"]["type"]
 553                 assert stream.read() == BytesIO(v["stream_content"]).read()
 554                 stream.seek(0)
 555                 assert stream.read() != BytesIO(v["stream_content_bad"]).read()
 556                 found = True
 557                 continue
 558         assert found
 559
 560     def find(self, type, no_cursor_timeout=True):
 561         list = []
 562         for i, v in self.tar_info.items():
 563             if type["metadata.type"] == "dir":
 564                 if v["metadata"] == self.directory_metadata:
 565                     list.append(v["cursor"])
 566             else:
 567                 if v["metadata"] != self.directory_metadata:
 568                     list.append(v["cursor"])
 569         return list
 570
 571     def download_to_stream(self, id, file_stream):
 572         file_stream.write(BytesIO(self.tar_info[id]["stream_content"]).read())
 573
 574
 575 def test_file_extract():
 576     tar_path = "tmp/Example.tar.gz"
 577     folder_path = "tmp/example_tar"
 578
 579     # Generate package
 580     subprocess.call(["rm", "-rf", "./tmp"])
 581     subprocess.call(["mkdir", "-p", "{}/directory".format(folder_path)])
 582     subprocess.call(["mkdir", "-p", "{}/symlinks".format(folder_path)])
 583     p = Path("{}/directory/file".format(folder_path))
 584     p.write_text("Example test")
 585     os.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path))
 586     os.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path))
 587     if os.path.exists(tar_path):
 588         os.remove(tar_path)
 589     subprocess.call(["tar", "-czvf", tar_path, folder_path])
 590
 591     try:
 592         tar = tarfile.open(tar_path, "r")
 593         fs = FsMongo()
 594         fs.fs = FakeFS()
 595         fs.file_extract(tar_object=tar, path=".")
 596     finally:
 597         os.remove(tar_path)
 598         subprocess.call(["rm", "-rf", "./tmp"])
 599
 600
 601 def test_upload_local_fs():
 602     path = "./tmp/"
 603
 604     subprocess.call(["rm", "-rf", path])
 605     try:
 606         fs = FsMongo()
 607         fs.path = path
 608         fs.fs = FakeFS()
 609         fs.sync()
 610         assert os.path.isdir("{}example_tar".format(path))
 611         assert os.path.isdir("{}example_tar/directory".format(path))
 612         assert os.path.isdir("{}example_tar/symlinks".format(path))
 613         assert os.path.isfile("{}example_tar/directory/file".format(path))
 614         assert os.path.islink("{}example_tar/symlinks/file_link".format(path))
 615         assert os.path.islink("{}example_tar/symlinks/directory_link".format(path))
 616     finally:
 617         subprocess.call(["rm", "-rf", path])