osm_common/tests/test_fsmongo.py

   1 # Copyright 2019 Canonical
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 #
  15 # For those usages not covered by the Apache License, Version 2.0 please
  16 # contact: eduardo.sousa@canonical.com
  17 ##
  18
  19 from io import BytesIO
  20 import logging
  21 import os
  22 from pathlib import Path
  23 import subprocess
  24 import tarfile
  25 import tempfile
  26 from unittest.mock import Mock
  27
  28 from gridfs import GridFSBucket
  29 from osm_common.fsbase import FsException
  30 from osm_common.fsmongo import FsMongo
  31 from pymongo import MongoClient
  32 import pytest
  33
  34 __author__ = "Eduardo Sousa <eduardo.sousa@canonical.com>"
  35
  36
  37 def valid_path():
  38     return tempfile.gettempdir() + "/"
  39
  40
  41 def invalid_path():
  42     return "/#tweeter/"
  43
  44
  45 @pytest.fixture(scope="function", params=[True, False])
  46 def fs_mongo(request, monkeypatch):
  47     def mock_mongoclient_constructor(a, b, c):
  48         pass
  49
  50     def mock_mongoclient_getitem(a, b):
  51         pass
  52
  53     def mock_gridfs_constructor(a, b):
  54         pass
  55
  56     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
  57     monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem)
  58     monkeypatch.setattr(GridFSBucket, "__init__", mock_gridfs_constructor)
  59     fs = FsMongo(lock=request.param)
  60     fs.fs_connect(
  61         {"path": valid_path(), "host": "mongo", "port": 27017, "collection": "files"}
  62     )
  63     return fs
  64
  65
  66 def generic_fs_exception_message(message):
  67     return "storage exception {}".format(message)
  68
  69
  70 def fs_connect_exception_message(path):
  71     return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(
  72         path
  73     )
  74
  75
  76 def file_open_file_not_found_exception(storage):
  77     f = storage if isinstance(storage, str) else "/".join(storage)
  78     return "storage exception File {} does not exist".format(f)
  79
  80
  81 def file_open_io_exception(storage):
  82     f = storage if isinstance(storage, str) else "/".join(storage)
  83     return "storage exception File {} cannot be opened".format(f)
  84
  85
  86 def dir_ls_not_a_directory_exception(storage):
  87     f = storage if isinstance(storage, str) else "/".join(storage)
  88     return "storage exception File {} does not exist".format(f)
  89
  90
  91 def dir_ls_io_exception(storage):
  92     f = storage if isinstance(storage, str) else "/".join(storage)
  93     return "storage exception File {} cannot be opened".format(f)
  94
  95
  96 def file_delete_exception_message(storage):
  97     return "storage exception File {} does not exist".format(storage)
  98
  99
 100 def test_constructor_without_logger():
 101     fs = FsMongo()
 102     assert fs.logger == logging.getLogger("fs")
 103     assert fs.path is None
 104     assert fs.client is None
 105     assert fs.fs is None
 106
 107
 108 def test_constructor_with_logger():
 109     logger_name = "fs_mongo"
 110     fs = FsMongo(logger_name=logger_name)
 111     assert fs.logger == logging.getLogger(logger_name)
 112     assert fs.path is None
 113     assert fs.client is None
 114     assert fs.fs is None
 115
 116
 117 def test_get_params(fs_mongo, monkeypatch):
 118     def mock_gridfs_find(self, search_query, **kwargs):
 119         return []
 120
 121     monkeypatch.setattr(GridFSBucket, "find", mock_gridfs_find)
 122     params = fs_mongo.get_params()
 123     assert len(params) == 2
 124     assert "fs" in params
 125     assert "path" in params
 126     assert params["fs"] == "mongo"
 127     assert params["path"] == valid_path()
 128
 129
 130 @pytest.mark.parametrize(
 131     "config, exp_logger, exp_path",
 132     [
 133         (
 134             {
 135                 "logger_name": "fs_mongo",
 136                 "path": valid_path(),
 137                 "uri": "mongo:27017",
 138                 "collection": "files",
 139             },
 140             "fs_mongo",
 141             valid_path(),
 142         ),
 143         (
 144             {
 145                 "logger_name": "fs_mongo",
 146                 "path": valid_path(),
 147                 "host": "mongo",
 148                 "port": 27017,
 149                 "collection": "files",
 150             },
 151             "fs_mongo",
 152             valid_path(),
 153         ),
 154         (
 155             {
 156                 "logger_name": "fs_mongo",
 157                 "path": valid_path()[:-1],
 158                 "uri": "mongo:27017",
 159                 "collection": "files",
 160             },
 161             "fs_mongo",
 162             valid_path(),
 163         ),
 164         (
 165             {
 166                 "logger_name": "fs_mongo",
 167                 "path": valid_path()[:-1],
 168                 "host": "mongo",
 169                 "port": 27017,
 170                 "collection": "files",
 171             },
 172             "fs_mongo",
 173             valid_path(),
 174         ),
 175         (
 176             {"path": valid_path(), "uri": "mongo:27017", "collection": "files"},
 177             "fs",
 178             valid_path(),
 179         ),
 180         (
 181             {
 182                 "path": valid_path(),
 183                 "host": "mongo",
 184                 "port": 27017,
 185                 "collection": "files",
 186             },
 187             "fs",
 188             valid_path(),
 189         ),
 190         (
 191             {"path": valid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
 192             "fs",
 193             valid_path(),
 194         ),
 195         (
 196             {
 197                 "path": valid_path()[:-1],
 198                 "host": "mongo",
 199                 "port": 27017,
 200                 "collection": "files",
 201             },
 202             "fs",
 203             valid_path(),
 204         ),
 205     ],
 206 )
 207 def test_fs_connect_with_valid_config(config, exp_logger, exp_path):
 208     fs = FsMongo()
 209     fs.fs_connect(config)
 210     assert fs.logger == logging.getLogger(exp_logger)
 211     assert fs.path == exp_path
 212     assert type(fs.client) == MongoClient
 213     assert type(fs.fs) == GridFSBucket
 214
 215
 216 @pytest.mark.parametrize(
 217     "config, exp_exception_message",
 218     [
 219         (
 220             {
 221                 "logger_name": "fs_mongo",
 222                 "path": invalid_path(),
 223                 "uri": "mongo:27017",
 224                 "collection": "files",
 225             },
 226             fs_connect_exception_message(invalid_path()),
 227         ),
 228         (
 229             {
 230                 "logger_name": "fs_mongo",
 231                 "path": invalid_path(),
 232                 "host": "mongo",
 233                 "port": 27017,
 234                 "collection": "files",
 235             },
 236             fs_connect_exception_message(invalid_path()),
 237         ),
 238         (
 239             {
 240                 "logger_name": "fs_mongo",
 241                 "path": invalid_path()[:-1],
 242                 "uri": "mongo:27017",
 243                 "collection": "files",
 244             },
 245             fs_connect_exception_message(invalid_path()[:-1]),
 246         ),
 247         (
 248             {
 249                 "logger_name": "fs_mongo",
 250                 "path": invalid_path()[:-1],
 251                 "host": "mongo",
 252                 "port": 27017,
 253                 "collection": "files",
 254             },
 255             fs_connect_exception_message(invalid_path()[:-1]),
 256         ),
 257         (
 258             {"path": invalid_path(), "uri": "mongo:27017", "collection": "files"},
 259             fs_connect_exception_message(invalid_path()),
 260         ),
 261         (
 262             {
 263                 "path": invalid_path(),
 264                 "host": "mongo",
 265                 "port": 27017,
 266                 "collection": "files",
 267             },
 268             fs_connect_exception_message(invalid_path()),
 269         ),
 270         (
 271             {"path": invalid_path()[:-1], "uri": "mongo:27017", "collection": "files"},
 272             fs_connect_exception_message(invalid_path()[:-1]),
 273         ),
 274         (
 275             {
 276                 "path": invalid_path()[:-1],
 277                 "host": "mongo",
 278                 "port": 27017,
 279                 "collection": "files",
 280             },
 281             fs_connect_exception_message(invalid_path()[:-1]),
 282         ),
 283         (
 284             {"path": "/", "host": "mongo", "port": 27017, "collection": "files"},
 285             generic_fs_exception_message(
 286                 "Invalid configuration param at '[storage]': path '/' is not writable"
 287             ),
 288         ),
 289     ],
 290 )
 291 def test_fs_connect_with_invalid_path(config, exp_exception_message):
 292     fs = FsMongo()
 293     with pytest.raises(FsException) as excinfo:
 294         fs.fs_connect(config)
 295     assert str(excinfo.value) == exp_exception_message
 296
 297
 298 @pytest.mark.parametrize(
 299     "config, exp_exception_message",
 300     [
 301         (
 302             {"logger_name": "fs_mongo", "uri": "mongo:27017", "collection": "files"},
 303             'Missing parameter "path"',
 304         ),
 305         (
 306             {
 307                 "logger_name": "fs_mongo",
 308                 "host": "mongo",
 309                 "port": 27017,
 310                 "collection": "files",
 311             },
 312             'Missing parameter "path"',
 313         ),
 314         (
 315             {"logger_name": "fs_mongo", "path": valid_path(), "collection": "files"},
 316             'Missing parameters: "uri" or "host" + "port"',
 317         ),
 318         (
 319             {
 320                 "logger_name": "fs_mongo",
 321                 "path": valid_path(),
 322                 "port": 27017,
 323                 "collection": "files",
 324             },
 325             'Missing parameters: "uri" or "host" + "port"',
 326         ),
 327         (
 328             {
 329                 "logger_name": "fs_mongo",
 330                 "path": valid_path(),
 331                 "host": "mongo",
 332                 "collection": "files",
 333             },
 334             'Missing parameters: "uri" or "host" + "port"',
 335         ),
 336         (
 337             {"logger_name": "fs_mongo", "path": valid_path(), "uri": "mongo:27017"},
 338             'Missing parameter "collection"',
 339         ),
 340         (
 341             {
 342                 "logger_name": "fs_mongo",
 343                 "path": valid_path(),
 344                 "host": "mongo",
 345                 "port": 27017,
 346             },
 347             'Missing parameter "collection"',
 348         ),
 349     ],
 350 )
 351 def test_fs_connect_with_missing_parameters(config, exp_exception_message):
 352     fs = FsMongo()
 353     with pytest.raises(FsException) as excinfo:
 354         fs.fs_connect(config)
 355     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 356
 357
 358 @pytest.mark.parametrize(
 359     "config, exp_exception_message",
 360     [
 361         (
 362             {
 363                 "logger_name": "fs_mongo",
 364                 "path": valid_path(),
 365                 "uri": "mongo:27017",
 366                 "collection": "files",
 367             },
 368             "MongoClient crashed",
 369         ),
 370         (
 371             {
 372                 "logger_name": "fs_mongo",
 373                 "path": valid_path(),
 374                 "host": "mongo",
 375                 "port": 27017,
 376                 "collection": "files",
 377             },
 378             "MongoClient crashed",
 379         ),
 380     ],
 381 )
 382 def test_fs_connect_with_invalid_mongoclient(
 383     config, exp_exception_message, monkeypatch
 384 ):
 385     def generate_exception(a, b, c=None):
 386         raise Exception(exp_exception_message)
 387
 388     monkeypatch.setattr(MongoClient, "__init__", generate_exception)
 389
 390     fs = FsMongo()
 391     with pytest.raises(FsException) as excinfo:
 392         fs.fs_connect(config)
 393     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 394
 395
 396 @pytest.mark.parametrize(
 397     "config, exp_exception_message",
 398     [
 399         (
 400             {
 401                 "logger_name": "fs_mongo",
 402                 "path": valid_path(),
 403                 "uri": "mongo:27017",
 404                 "collection": "files",
 405             },
 406             "Collection unavailable",
 407         ),
 408         (
 409             {
 410                 "logger_name": "fs_mongo",
 411                 "path": valid_path(),
 412                 "host": "mongo",
 413                 "port": 27017,
 414                 "collection": "files",
 415             },
 416             "Collection unavailable",
 417         ),
 418     ],
 419 )
 420 def test_fs_connect_with_invalid_mongo_collection(
 421     config, exp_exception_message, monkeypatch
 422 ):
 423     def mock_mongoclient_constructor(a, b, c=None):
 424         pass
 425
 426     def generate_exception(a, b):
 427         raise Exception(exp_exception_message)
 428
 429     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
 430     monkeypatch.setattr(MongoClient, "__getitem__", generate_exception)
 431
 432     fs = FsMongo()
 433     with pytest.raises(FsException) as excinfo:
 434         fs.fs_connect(config)
 435     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 436
 437
 438 @pytest.mark.parametrize(
 439     "config, exp_exception_message",
 440     [
 441         (
 442             {
 443                 "logger_name": "fs_mongo",
 444                 "path": valid_path(),
 445                 "uri": "mongo:27017",
 446                 "collection": "files",
 447             },
 448             "GridFsBucket crashed",
 449         ),
 450         (
 451             {
 452                 "logger_name": "fs_mongo",
 453                 "path": valid_path(),
 454                 "host": "mongo",
 455                 "port": 27017,
 456                 "collection": "files",
 457             },
 458             "GridFsBucket crashed",
 459         ),
 460     ],
 461 )
 462 def test_fs_connect_with_invalid_gridfsbucket(
 463     config, exp_exception_message, monkeypatch
 464 ):
 465     def mock_mongoclient_constructor(a, b, c=None):
 466         pass
 467
 468     def mock_mongoclient_getitem(a, b):
 469         pass
 470
 471     def generate_exception(a, b):
 472         raise Exception(exp_exception_message)
 473
 474     monkeypatch.setattr(MongoClient, "__init__", mock_mongoclient_constructor)
 475     monkeypatch.setattr(MongoClient, "__getitem__", mock_mongoclient_getitem)
 476     monkeypatch.setattr(GridFSBucket, "__init__", generate_exception)
 477
 478     fs = FsMongo()
 479     with pytest.raises(FsException) as excinfo:
 480         fs.fs_connect(config)
 481     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 482
 483
 484 def test_fs_disconnect(fs_mongo):
 485     fs_mongo.fs_disconnect()
 486
 487
 488 # Example.tar.gz
 489 # example_tar/
 490 # ├── directory
 491 # │   └── file
 492 # └── symlinks
 493 #     ├── directory_link -> ../directory/
 494 #     └── file_link -> ../directory/file
 495 class FakeCursor:
 496     def __init__(self, id, filename, metadata):
 497         self._id = id
 498         self.filename = filename
 499         self.metadata = metadata
 500
 501
 502 class FakeFS:
 503     directory_metadata = {"type": "dir", "permissions": 509}
 504     file_metadata = {"type": "file", "permissions": 436}
 505     symlink_metadata = {"type": "sym", "permissions": 511}
 506
 507     tar_info = {
 508         1: {
 509             "cursor": FakeCursor(1, "example_tar", directory_metadata),
 510             "metadata": directory_metadata,
 511             "stream_content": b"",
 512             "stream_content_bad": b"Something",
 513             "path": "./tmp/example_tar",
 514         },
 515         2: {
 516             "cursor": FakeCursor(2, "example_tar/directory", directory_metadata),
 517             "metadata": directory_metadata,
 518             "stream_content": b"",
 519             "stream_content_bad": b"Something",
 520             "path": "./tmp/example_tar/directory",
 521         },
 522         3: {
 523             "cursor": FakeCursor(3, "example_tar/symlinks", directory_metadata),
 524             "metadata": directory_metadata,
 525             "stream_content": b"",
 526             "stream_content_bad": b"Something",
 527             "path": "./tmp/example_tar/symlinks",
 528         },
 529         4: {
 530             "cursor": FakeCursor(4, "example_tar/directory/file", file_metadata),
 531             "metadata": file_metadata,
 532             "stream_content": b"Example test",
 533             "stream_content_bad": b"Example test2",
 534             "path": "./tmp/example_tar/directory/file",
 535         },
 536         5: {
 537             "cursor": FakeCursor(5, "example_tar/symlinks/file_link", symlink_metadata),
 538             "metadata": symlink_metadata,
 539             "stream_content": b"../directory/file",
 540             "stream_content_bad": b"",
 541             "path": "./tmp/example_tar/symlinks/file_link",
 542         },
 543         6: {
 544             "cursor": FakeCursor(
 545                 6, "example_tar/symlinks/directory_link", symlink_metadata
 546             ),
 547             "metadata": symlink_metadata,
 548             "stream_content": b"../directory/",
 549             "stream_content_bad": b"",
 550             "path": "./tmp/example_tar/symlinks/directory_link",
 551         },
 552     }
 553
 554     def upload_from_stream(self, f, stream, metadata=None):
 555         found = False
 556         for i, v in self.tar_info.items():
 557             if f == v["path"]:
 558                 assert metadata["type"] == v["metadata"]["type"]
 559                 assert stream.read() == BytesIO(v["stream_content"]).read()
 560                 stream.seek(0)
 561                 assert stream.read() != BytesIO(v["stream_content_bad"]).read()
 562                 found = True
 563                 continue
 564         assert found
 565
 566     def find(self, type, no_cursor_timeout=True, sort=None):
 567         list = []
 568         for i, v in self.tar_info.items():
 569             if type["metadata.type"] == "dir":
 570                 if v["metadata"] == self.directory_metadata:
 571                     list.append(v["cursor"])
 572             else:
 573                 if v["metadata"] != self.directory_metadata:
 574                     list.append(v["cursor"])
 575         return list
 576
 577     def download_to_stream(self, id, file_stream):
 578         file_stream.write(BytesIO(self.tar_info[id]["stream_content"]).read())
 579
 580
 581 def test_file_extract():
 582     tar_path = "tmp/Example.tar.gz"
 583     folder_path = "tmp/example_tar"
 584
 585     # Generate package
 586     subprocess.call(["rm", "-rf", "./tmp"])
 587     subprocess.call(["mkdir", "-p", "{}/directory".format(folder_path)])
 588     subprocess.call(["mkdir", "-p", "{}/symlinks".format(folder_path)])
 589     p = Path("{}/directory/file".format(folder_path))
 590     p.write_text("Example test")
 591     os.symlink("../directory/file", "{}/symlinks/file_link".format(folder_path))
 592     os.symlink("../directory/", "{}/symlinks/directory_link".format(folder_path))
 593     if os.path.exists(tar_path):
 594         os.remove(tar_path)
 595     subprocess.call(["tar", "-czvf", tar_path, folder_path])
 596
 597     try:
 598         tar = tarfile.open(tar_path, "r")
 599         fs = FsMongo()
 600         fs.fs = FakeFS()
 601         fs.file_extract(compressed_object=tar, path=".")
 602     finally:
 603         os.remove(tar_path)
 604         subprocess.call(["rm", "-rf", "./tmp"])
 605
 606
 607 def test_upload_local_fs():
 608     path = "./tmp/"
 609
 610     subprocess.call(["rm", "-rf", path])
 611     try:
 612         fs = FsMongo()
 613         fs.path = path
 614         fs.fs = FakeFS()
 615         fs.sync()
 616         assert os.path.isdir("{}example_tar".format(path))
 617         assert os.path.isdir("{}example_tar/directory".format(path))
 618         assert os.path.isdir("{}example_tar/symlinks".format(path))
 619         assert os.path.isfile("{}example_tar/directory/file".format(path))
 620         assert os.path.islink("{}example_tar/symlinks/file_link".format(path))
 621         assert os.path.islink("{}example_tar/symlinks/directory_link".format(path))
 622     finally:
 623         subprocess.call(["rm", "-rf", path])
 624
 625
 626 def test_upload_mongo_fs():
 627     path = "./tmp/"
 628
 629     subprocess.call(["rm", "-rf", path])
 630     try:
 631         fs = FsMongo()
 632         fs.path = path
 633         fs.fs = Mock()
 634         fs.fs.find.return_value = {}
 635
 636         file_content = "Test file content"
 637
 638         # Create local dir and upload content to fakefs
 639         os.mkdir(path)
 640         os.mkdir("{}example_local".format(path))
 641         os.mkdir("{}example_local/directory".format(path))
 642         with open(
 643             "{}example_local/directory/test_file".format(path), "w+"
 644         ) as test_file:
 645             test_file.write(file_content)
 646         fs.reverse_sync("example_local")
 647
 648         assert fs.fs.upload_from_stream.call_count == 2
 649
 650         # first call to upload_from_stream, dir_name
 651         dir_name = "example_local/directory"
 652         call_args_0 = fs.fs.upload_from_stream.call_args_list[0]
 653         assert call_args_0[0][0] == dir_name
 654         assert call_args_0[1].get("metadata").get("type") == "dir"
 655
 656         # second call to upload_from_stream, dir_name
 657         file_name = "example_local/directory/test_file"
 658         call_args_1 = fs.fs.upload_from_stream.call_args_list[1]
 659         assert call_args_1[0][0] == file_name
 660         assert call_args_1[1].get("metadata").get("type") == "file"
 661
 662     finally:
 663         subprocess.call(["rm", "-rf", path])
 664         pass