osm_common/tests/test_fsmongo.py

   1 # Copyright 2019 Canonical
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 #
  15 # For those usages not covered by the Apache License, Version 2.0 please
  16 # contact: eduardo.sousa@canonical.com
  17 ##
  18
  19 import logging
  20 import pytest
  21 import tempfile
  22 import tarfile
  23 import os
  24 import subprocess
  25
  26 from pymongo import MongoClient
  27 from gridfs import GridFSBucket
  28
  29 from io import BytesIO
  30
  31 from unittest.mock import Mock
  32
  33 from osm_common.fsbase import FsException
  34 from osm_common.fsmongo import FsMongo
  35 from pathlib import Path
  36
  37 __author__ = "Eduardo Sousa <eduardo.sousa@canonical.com>"
  38
  39
  40 def valid_path():
  41     return tempfile.gettempdir() + "/"
  42
  43
  44 def invalid_path():
  45     return "/#tweeter/"
  46
  47
  48 @pytest.fixture(scope="function", params=[True, False])
  49 def fs_mongo(request, monkeypatch):
  50     def mock_mongoclient_constructor(a, b, c):
  51         pass
  52
  53     def mock_mongoclient_getitem(a, b):
  54         pass
  55
  56     def mock_gridfs_constructor(a, b):
  57         pass
  58
  59     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
  60     monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem)
  61     monkeypatch.setattr(GridFSBucket, "__init__", mock_gridfs_constructor)
  62     fs = FsMongo(lock=request.param)
  63     fs.fs_connect(
  64         {"path": valid_path(), "host": "mongo", "port": 27017, "collection": "files"}
  65     )
  66     return fs
  67
  68
  69 def generic_fs_exception_message(message):
  70     return "storage exception {}".format(message)
  71
  72
  73 def fs_connect_exception_message(path):
  74     return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(
  75         path
  76     )
  77
  78
  79 def file_open_file_not_found_exception(storage):
  80     f = storage if isinstance(storage, str) else "/".join(storage)
  81     return "storage exception File {} does not exist".format(f)
  82
  83
  84 def file_open_io_exception(storage):
  85     f = storage if isinstance(storage, str) else "/".join(storage)
  86     return "storage exception File {} cannot be opened".format(f)
  87
  88
  89 def dir_ls_not_a_directory_exception(storage):
  90     f = storage if isinstance(storage, str) else "/".join(storage)
  91     return "storage exception File {} does not exist".format(f)
  92
  93
  94 def dir_ls_io_exception(storage):
  95     f = storage if isinstance(storage, str) else "/".join(storage)
  96     return "storage exception File {} cannot be opened".format(f)
  97
  98
  99 def file_delete_exception_message(storage):
 100     return "storage exception File {} does not exist".format(storage)
 101
 102
 103 def test_constructor_without_logger():
 104     fs = FsMongo()
 105     assert fs.logger == logging.getLogger("fs")
 106     assert fs.path is None
 107     assert fs.client is None
 108     assert fs.fs is None
 109
 110
 111 def test_constructor_with_logger():
 112     logger_name = "fs_mongo"
 113     fs = FsMongo(logger_name=logger_name)
 114     assert fs.logger == logging.getLogger(logger_name)
 115     assert fs.path is None
 116     assert fs.client is None
 117     assert fs.fs is None
 118
 119
 120 def test_get_params(fs_mongo, monkeypatch):
 121     def mock_gridfs_find(self, search_query, **kwargs):
 122         return []
 123
 124     monkeypatch.setattr(GridFSBucket, "find", mock_gridfs_find)
 125     params = fs_mongo.get_params()
 126     assert len(params) == 2
 127     assert "fs" in params
 128     assert "path" in params
 129     assert params["fs"] == "mongo"
 130     assert params["path"] == valid_path()
 131
 132
 133 @pytest.mark.parametrize(
 134     "config, exp_logger, exp_path",
 135     [
 136         (
 137             {
 138                 "logger_name": "fs_mongo",
 139                 "path": valid_path(),
 140                 "uri": "mongo:27017",
 141                 "collection": "files",
 142             },
 143             "fs_mongo",
 144             valid_path(),
 145         ),
 146         (
 147             {
 148                 "logger_name": "fs_mongo",
 149                 "path": valid_path(),
 150                 "host": "mongo",
 151                 "port": 27017,
 152                 "collection": "files",
 153             },
 154             "fs_mongo",
 155             valid_path(),
 156         ),
 157         (
 158             {
 159                 "logger_name": "fs_mongo",
 160                 "path": valid_path()[:-1],
 161                 "uri": "mongo:27017",
 162                 "collection": "files",
 163             },
 164             "fs_mongo",
 165             valid_path(),
 166         ),
 167         (
 168             {
 169                 "logger_name": "fs_mongo",
 170                 "path": valid_path()[:-1],
 171                 "host": "mongo",
 172                 "port": 27017,
 173                 "collection": "files",
 174             },
 175             "fs_mongo",
 176             valid_path(),
 177         ),
 178         (
 179             {"path": valid_path(), "uri": "mongo:27017", "collection": "files"},
 180             "fs",
 181             valid_path(),
 182         ),
 183         (
 184             {
 185                 "path": valid_path(),
 186                 "host": "mongo",
 187                 "port": 27017,
 188                 "collection": "files",
 189             },
 190             "fs",
 191             valid_path(),
 192         ),
 193         (
 194             {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
 195             "fs",
 196             valid_path(),
 197         ),
 198         (
 199             {
 200                 "path": valid_path()[:-1],
 201                 "host": "mongo",
 202                 "port": 27017,
 203                 "collection": "files",
 204             },
 205             "fs",
 206             valid_path(),
 207         ),
 208     ],
 209 )
 210 def test_fs_connect_with_valid_config(config, exp_logger, exp_path):
 211     fs = FsMongo()
 212     fs.fs_connect(config)
 213     assert fs.logger == logging.getLogger(exp_logger)
 214     assert fs.path == exp_path
 215     assert type(fs.client) == MongoClient
 216     assert type(fs.fs) == GridFSBucket
 217
 218
 219 @pytest.mark.parametrize(
 220     "config, exp_exception_message",
 221     [
 222         (
 223             {
 224                 "logger_name": "fs_mongo",
 225                 "path": invalid_path(),
 226                 "uri": "mongo:27017",
 227                 "collection": "files",
 228             },
 229             fs_connect_exception_message(invalid_path()),
 230         ),
 231         (
 232             {
 233                 "logger_name": "fs_mongo",
 234                 "path": invalid_path(),
 235                 "host": "mongo",
 236                 "port": 27017,
 237                 "collection": "files",
 238             },
 239             fs_connect_exception_message(invalid_path()),
 240         ),
 241         (
 242             {
 243                 "logger_name": "fs_mongo",
 244                 "path": invalid_path()[:-1],
 245                 "uri": "mongo:27017",
 246                 "collection": "files",
 247             },
 248             fs_connect_exception_message(invalid_path()[:-1]),
 249         ),
 250         (
 251             {
 252                 "logger_name": "fs_mongo",
 253                 "path": invalid_path()[:-1],
 254                 "host": "mongo",
 255                 "port": 27017,
 256                 "collection": "files",
 257             },
 258             fs_connect_exception_message(invalid_path()[:-1]),
 259         ),
 260         (
 261             {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"},
 262             fs_connect_exception_message(invalid_path()),
 263         ),
 264         (
 265             {
 266                 "path": invalid_path(),
 267                 "host": "mongo",
 268                 "port": 27017,
 269                 "collection": "files",
 270             },
 271             fs_connect_exception_message(invalid_path()),
 272         ),
 273         (
 274             {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
 275             fs_connect_exception_message(invalid_path()[:-1]),
 276         ),
 277         (
 278             {
 279                 "path": invalid_path()[:-1],
 280                 "host": "mongo",
 281                 "port": 27017,
 282                 "collection": "files",
 283             },
 284             fs_connect_exception_message(invalid_path()[:-1]),
 285         ),
 286         (
 287             {"path": "/", "host": "mongo", "port": 27017, "collection": "files"},
 288             generic_fs_exception_message(
 289                 "Invalid configuration param at '[storage]': path '/' is not writable"
 290             ),
 291         ),
 292     ],
 293 )
 294 def test_fs_connect_with_invalid_path(config, exp_exception_message):
 295     fs = FsMongo()
 296     with pytest.raises(FsException) as excinfo:
 297         fs.fs_connect(config)
 298     assert str(excinfo.value) == exp_exception_message
 299
 300
 301 @pytest.mark.parametrize(
 302     "config, exp_exception_message",
 303     [
 304         (
 305             {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"},
 306             'Missing parameter "path"',
 307         ),
 308         (
 309             {
 310                 "logger_name": "fs_mongo",
 311                 "host": "mongo",
 312                 "port": 27017,
 313                 "collection": "files",
 314             },
 315             'Missing parameter "path"',
 316         ),
 317         (
 318             {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"},
 319             'Missing parameters: "uri" or "host" + "port"',
 320         ),
 321         (
 322             {
 323                 "logger_name": "fs_mongo",
 324                 "path": valid_path(),
 325                 "port": 27017,
 326                 "collection": "files",
 327             },
 328             'Missing parameters: "uri" or "host" + "port"',
 329         ),
 330         (
 331             {
 332                 "logger_name": "fs_mongo",
 333                 "path": valid_path(),
 334                 "host": "mongo",
 335                 "collection": "files",
 336             },
 337             'Missing parameters: "uri" or "host" + "port"',
 338         ),
 339         (
 340             {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"},
 341             'Missing parameter "collection"',
 342         ),
 343         (
 344             {
 345                 "logger_name": "fs_mongo",
 346                 "path": valid_path(),
 347                 "host": "mongo",
 348                 "port": 27017,
 349             },
 350             'Missing parameter "collection"',
 351         ),
 352     ],
 353 )
 354 def test_fs_connect_with_missing_parameters(config, exp_exception_message):
 355     fs = FsMongo()
 356     with pytest.raises(FsException) as excinfo:
 357         fs.fs_connect(config)
 358     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 359
 360
 361 @pytest.mark.parametrize(
 362     "config, exp_exception_message",
 363     [
 364         (
 365             {
 366                 "logger_name": "fs_mongo",
 367                 "path": valid_path(),
 368                 "uri": "mongo:27017",
 369                 "collection": "files",
 370             },
 371             "MongoClient crashed",
 372         ),
 373         (
 374             {
 375                 "logger_name": "fs_mongo",
 376                 "path": valid_path(),
 377                 "host": "mongo",
 378                 "port": 27017,
 379                 "collection": "files",
 380             },
 381             "MongoClient crashed",
 382         ),
 383     ],
 384 )
 385 def test_fs_connect_with_invalid_mongoclient(
 386     config, exp_exception_message, monkeypatch
 387 ):
 388     def generate_exception(a, b, c=None):
 389         raise Exception(exp_exception_message)
 390
 391     monkeypatch.setattr(MongoClient, "__init__", generate_exception)
 392
 393     fs = FsMongo()
 394     with pytest.raises(FsException) as excinfo:
 395         fs.fs_connect(config)
 396     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 397
 398
 399 @pytest.mark.parametrize(
 400     "config, exp_exception_message",
 401     [
 402         (
 403             {
 404                 "logger_name": "fs_mongo",
 405                 "path": valid_path(),
 406                 "uri": "mongo:27017",
 407                 "collection": "files",
 408             },
 409             "Collection unavailable",
 410         ),
 411         (
 412             {
 413                 "logger_name": "fs_mongo",
 414                 "path": valid_path(),
 415                 "host": "mongo",
 416                 "port": 27017,
 417                 "collection": "files",
 418             },
 419             "Collection unavailable",
 420         ),
 421     ],
 422 )
 423 def test_fs_connect_with_invalid_mongo_collection(
 424     config, exp_exception_message, monkeypatch
 425 ):
 426     def mock_mongoclient_constructor(a, b, c=None):
 427         pass
 428
 429     def generate_exception(a, b):
 430         raise Exception(exp_exception_message)
 431
 432     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
 433     monkeypatch.setattr(MongoClient, "__getitem__", generate_exception)
 434
 435     fs = FsMongo()
 436     with pytest.raises(FsException) as excinfo:
 437         fs.fs_connect(config)
 438     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 439
 440
 441 @pytest.mark.parametrize(
 442     "config, exp_exception_message",
 443     [
 444         (
 445             {
 446                 "logger_name": "fs_mongo",
 447                 "path": valid_path(),
 448                 "uri": "mongo:27017",
 449                 "collection": "files",
 450             },
 451             "GridFsBucket crashed",
 452         ),
 453         (
 454             {
 455                 "logger_name": "fs_mongo",
 456                 "path": valid_path(),
 457                 "host": "mongo",
 458                 "port": 27017,
 459                 "collection": "files",
 460             },
 461             "GridFsBucket crashed",
 462         ),
 463     ],
 464 )
 465 def test_fs_connect_with_invalid_gridfsbucket(
 466     config, exp_exception_message, monkeypatch
 467 ):
 468     def mock_mongoclient_constructor(a, b, c=None):
 469         pass
 470
 471     def mock_mongoclient_getitem(a, b):
 472         pass
 473
 474     def generate_exception(a, b):
 475         raise Exception(exp_exception_message)
 476
 477     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
 478     monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem)
 479     monkeypatch.setattr(GridFSBucket, "__init__", generate_exception)
 480
 481     fs = FsMongo()
 482     with pytest.raises(FsException) as excinfo:
 483         fs.fs_connect(config)
 484     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 485
 486
 487 def test_fs_disconnect(fs_mongo):
 488     fs_mongo.fs_disconnect()
 489
 490
 491 # Example.tar.gz
 492 # example_tar/
 493 # ├── directory
 494 # │   └── file
 495 # └── symlinks
 496 #     ├── directory_link -> ../directory/
 497 #     └── file_link -> ../directory/file
 498 class FakeCursor:
 499     def __init__(self, id, filename, metadata):
 500         self._id = id
 501         self.filename = filename
 502         self.metadata = metadata
 503
 504
 505 class FakeFS:
 506     directory_metadata = {"type": "dir", "permissions": 509}
 507     file_metadata = {"type": "file", "permissions": 436}
 508     symlink_metadata = {"type": "sym", "permissions": 511}
 509
 510     tar_info = {
 511         1: {
 512             "cursor": FakeCursor(1, "example_tar", directory_metadata),
 513             "metadata": directory_metadata,
 514             "stream_content": b"",
 515             "stream_content_bad": b"Something",
 516             "path": "./tmp/example_tar",
 517         },
 518         2: {
 519             "cursor": FakeCursor(2, "example_tar/directory", directory_metadata),
 520             "metadata": directory_metadata,
 521             "stream_content": b"",
 522             "stream_content_bad": b"Something",
 523             "path": "./tmp/example_tar/directory",
 524         },
 525         3: {
 526             "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata),
 527             "metadata": directory_metadata,
 528             "stream_content": b"",
 529             "stream_content_bad": b"Something",
 530             "path": "./tmp/example_tar/symlinks",
 531         },
 532         4: {
 533             "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata),
 534             "metadata": file_metadata,
 535             "stream_content": b"Example test",
 536             "stream_content_bad": b"Example test2",
 537             "path": "./tmp/example_tar/directory/file",
 538         },
 539         5: {
 540             "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata),
 541             "metadata": symlink_metadata,
 542             "stream_content": b"../directory/file",
 543             "stream_content_bad": b"",
 544             "path": "./tmp/example_tar/symlinks/file_link",
 545         },
 546         6: {
 547             "cursor": FakeCursor(
 548                 6, "example_tar/symlinks/directory_link", symlink_metadata
 549             ),
 550             "metadata": symlink_metadata,
 551             "stream_content": b"../directory/",
 552             "stream_content_bad": b"",
 553             "path": "./tmp/example_tar/symlinks/directory_link",
 554         },
 555     }
 556
 557     def upload_from_stream(self, f, stream, metadata=None):
 558         found = False
 559         for i, v in self.tar_info.items():
 560             if f == v["path"]:
 561                 assert metadata["type"] == v["metadata"]["type"]
 562                 assert stream.read() == BytesIO(v["stream_content"]).read()
 563                 stream.seek(0)
 564                 assert stream.read() != BytesIO(v["stream_content_bad"]).read()
 565                 found = True
 566                 continue
 567         assert found
 568
 569     def find(self, type, no_cursor_timeout=True, sort=None):
 570         list = []
 571         for i, v in self.tar_info.items():
 572             if type["metadata.type"] == "dir":
 573                 if v["metadata"] == self.directory_metadata:
 574                     list.append(v["cursor"])
 575             else:
 576                 if v["metadata"] != self.directory_metadata:
 577                     list.append(v["cursor"])
 578         return list
 579
 580     def download_to_stream(self, id, file_stream):
 581         file_stream.write(BytesIO(self.tar_info[id]["stream_content"]).read())
 582
 583
 584 def test_file_extract():
 585     tar_path = "tmp/Example.tar.gz"
 586     folder_path = "tmp/example_tar"
 587
 588     # Generate package
 589     subprocess.call(["rm", "-rf", "./tmp"])
 590     subprocess.call(["mkdir", "-p", "{}/directory".format(folder_path)])
 591     subprocess.call(["mkdir", "-p", "{}/symlinks".format(folder_path)])
 592     p = Path("{}/directory/file".format(folder_path))
 593     p.write_text("Example test")
 594     os.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path))
 595     os.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path))
 596     if os.path.exists(tar_path):
 597         os.remove(tar_path)
 598     subprocess.call(["tar", "-czvf", tar_path, folder_path])
 599
 600     try:
 601         tar = tarfile.open(tar_path, "r")
 602         fs = FsMongo()
 603         fs.fs = FakeFS()
 604         fs.file_extract(compressed_object=tar, path=".")
 605     finally:
 606         os.remove(tar_path)
 607         subprocess.call(["rm", "-rf", "./tmp"])
 608
 609
 610 def test_upload_local_fs():
 611     path = "./tmp/"
 612
 613     subprocess.call(["rm", "-rf", path])
 614     try:
 615         fs = FsMongo()
 616         fs.path = path
 617         fs.fs = FakeFS()
 618         fs.sync()
 619         assert os.path.isdir("{}example_tar".format(path))
 620         assert os.path.isdir("{}example_tar/directory".format(path))
 621         assert os.path.isdir("{}example_tar/symlinks".format(path))
 622         assert os.path.isfile("{}example_tar/directory/file".format(path))
 623         assert os.path.islink("{}example_tar/symlinks/file_link".format(path))
 624         assert os.path.islink("{}example_tar/symlinks/directory_link".format(path))
 625     finally:
 626         subprocess.call(["rm", "-rf", path])
 627
 628
 629 def test_upload_mongo_fs():
 630     path = "./tmp/"
 631
 632     subprocess.call(["rm", "-rf", path])
 633     try:
 634         fs = FsMongo()
 635         fs.path = path
 636         fs.fs = Mock()
 637         fs.fs.find.return_value = {}
 638
 639         file_content = "Test file content"
 640
 641         # Create local dir and upload content to fakefs
 642         os.mkdir(path)
 643         os.mkdir("{}example_local".format(path))
 644         os.mkdir("{}example_local/directory".format(path))
 645         with open(
 646             "{}example_local/directory/test_file".format(path), "w+"
 647         ) as test_file:
 648             test_file.write(file_content)
 649         fs.reverse_sync("example_local")
 650
 651         assert fs.fs.upload_from_stream.call_count == 2
 652
 653         # first call to upload_from_stream, dir_name
 654         dir_name = "example_local/directory"
 655         call_args_0 = fs.fs.upload_from_stream.call_args_list[0]
 656         assert call_args_0[0][0] == dir_name
 657         assert call_args_0[1].get("metadata").get("type") == "dir"
 658
 659         # second call to upload_from_stream, dir_name
 660         file_name = "example_local/directory/test_file"
 661         call_args_1 = fs.fs.upload_from_stream.call_args_list[1]
 662         assert call_args_1[0][0] == file_name
 663         assert call_args_1[1].get("metadata").get("type") == "file"
 664
 665     finally:
 666         subprocess.call(["rm", "-rf", path])
 667         pass