osm_common/tests/test_fsmongo.py

   1 # Copyright 2019 Canonical
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 #
  15 # For those usages not covered by the Apache License, Version 2.0 please
  16 # contact: eduardo.sousa@canonical.com
  17 ##
  18
  19 import logging
  20 import pytest
  21 import tempfile
  22
  23 from pymongo import MongoClient
  24 from gridfs import GridFSBucket
  25
  26 from osm_common.fsbase import FsException
  27 from osm_common.fsmongo import FsMongo
  28
  29 __author__ = "Eduardo Sousa <eduardo.sousa@canonical.com>"
  30
  31
  32 def valid_path():
  33     return tempfile.gettempdir() + '/'
  34
  35
  36 def invalid_path():
  37     return '/#tweeter/'
  38
  39
  40 @pytest.fixture(scope="function", params=[True, False])
  41 def fs_mongo(request, monkeypatch):
  42     def mock_mongoclient_constructor(a, b, c):
  43         pass
  44
  45     def mock_mongoclient_getitem(a, b):
  46         pass
  47
  48     def mock_gridfs_constructor(a, b):
  49         pass
  50
  51     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
  52     monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem)
  53     monkeypatch.setattr(GridFSBucket, '__init__', mock_gridfs_constructor)
  54     fs = FsMongo(lock=request.param)
  55     fs.fs_connect({
  56         'path': valid_path(),
  57         'host': 'mongo',
  58         'port': 27017,
  59         'collection': 'files'})
  60     return fs
  61
  62
  63 def generic_fs_exception_message(message):
  64     return "storage exception {}".format(message)
  65
  66
  67 def fs_connect_exception_message(path):
  68     return "storage exception Invalid configuration param at '[storage]': path '{}' does not exist".format(path)
  69
  70
  71 def file_open_file_not_found_exception(storage):
  72     f = storage if isinstance(storage, str) else '/'.join(storage)
  73     return "storage exception File {} does not exist".format(f)
  74
  75
  76 def file_open_io_exception(storage):
  77     f = storage if isinstance(storage, str) else '/'.join(storage)
  78     return "storage exception File {} cannot be opened".format(f)
  79
  80
  81 def dir_ls_not_a_directory_exception(storage):
  82     f = storage if isinstance(storage, str) else '/'.join(storage)
  83     return "storage exception File {} does not exist".format(f)
  84
  85
  86 def dir_ls_io_exception(storage):
  87     f = storage if isinstance(storage, str) else '/'.join(storage)
  88     return "storage exception File {} cannot be opened".format(f)
  89
  90
  91 def file_delete_exception_message(storage):
  92     return "storage exception File {} does not exist".format(storage)
  93
  94
  95 def test_constructor_without_logger():
  96     fs = FsMongo()
  97     assert fs.logger == logging.getLogger('fs')
  98     assert fs.path is None
  99     assert fs.client is None
 100     assert fs.fs is None
 101
 102
 103 def test_constructor_with_logger():
 104     logger_name = 'fs_mongo'
 105     fs = FsMongo(logger_name=logger_name)
 106     assert fs.logger == logging.getLogger(logger_name)
 107     assert fs.path is None
 108     assert fs.client is None
 109     assert fs.fs is None
 110
 111
 112 def test_get_params(fs_mongo, monkeypatch):
 113     def mock_gridfs_find(self, search_query, **kwargs):
 114         return []
 115
 116     monkeypatch.setattr(GridFSBucket, 'find', mock_gridfs_find)
 117     params = fs_mongo.get_params()
 118     assert len(params) == 2
 119     assert "fs" in params
 120     assert "path" in params
 121     assert params["fs"] == "mongo"
 122     assert params["path"] == valid_path()
 123
 124
 125 @pytest.mark.parametrize("config, exp_logger, exp_path", [
 126     (
 127         {
 128             'logger_name': 'fs_mongo',
 129             'path': valid_path(),
 130             'uri': 'mongo:27017',
 131             'collection': 'files'
 132         },
 133         'fs_mongo', valid_path()
 134     ),
 135     (
 136         {
 137             'logger_name': 'fs_mongo',
 138             'path': valid_path(),
 139             'host': 'mongo',
 140             'port': 27017,
 141             'collection': 'files'
 142         },
 143         'fs_mongo', valid_path()
 144     ),
 145     (
 146         {
 147             'logger_name': 'fs_mongo',
 148             'path': valid_path()[:-1],
 149             'uri': 'mongo:27017',
 150             'collection': 'files'
 151         },
 152         'fs_mongo', valid_path()
 153     ),
 154     (
 155         {
 156             'logger_name': 'fs_mongo',
 157             'path': valid_path()[:-1],
 158             'host': 'mongo',
 159             'port': 27017,
 160             'collection': 'files'
 161         },
 162         'fs_mongo', valid_path()
 163     ),
 164     (
 165         {
 166             'path': valid_path(),
 167             'uri': 'mongo:27017',
 168             'collection': 'files'
 169         },
 170         'fs', valid_path()
 171     ),
 172     (
 173         {
 174             'path': valid_path(),
 175             'host': 'mongo',
 176             'port': 27017,
 177             'collection': 'files'
 178         },
 179         'fs', valid_path()
 180     ),
 181     (
 182         {
 183             'path': valid_path()[:-1],
 184             'uri': 'mongo:27017',
 185             'collection': 'files'
 186         },
 187         'fs', valid_path()
 188     ),
 189     (
 190         {
 191             'path': valid_path()[:-1],
 192             'host': 'mongo',
 193             'port': 27017,
 194             'collection': 'files'
 195         },
 196         'fs', valid_path()
 197     )])
 198 def test_fs_connect_with_valid_config(config, exp_logger, exp_path):
 199     fs = FsMongo()
 200     fs.fs_connect(config)
 201     assert fs.logger == logging.getLogger(exp_logger)
 202     assert fs.path == exp_path
 203     assert type(fs.client) == MongoClient
 204     assert type(fs.fs) == GridFSBucket
 205
 206
 207 @pytest.mark.parametrize("config, exp_exception_message", [
 208     (
 209         {
 210             'logger_name': 'fs_mongo',
 211             'path': invalid_path(),
 212             'uri': 'mongo:27017',
 213             'collection': 'files'
 214         },
 215         fs_connect_exception_message(invalid_path())
 216     ),
 217     (
 218         {
 219             'logger_name': 'fs_mongo',
 220             'path': invalid_path(),
 221             'host': 'mongo',
 222             'port': 27017,
 223             'collection': 'files'
 224         },
 225         fs_connect_exception_message(invalid_path())
 226     ),
 227     (
 228         {
 229             'logger_name': 'fs_mongo',
 230             'path': invalid_path()[:-1],
 231             'uri': 'mongo:27017',
 232             'collection': 'files'
 233         },
 234         fs_connect_exception_message(invalid_path()[:-1])
 235     ),
 236     (
 237         {
 238             'logger_name': 'fs_mongo',
 239             'path': invalid_path()[:-1],
 240             'host': 'mongo',
 241             'port': 27017,
 242             'collection': 'files'
 243         },
 244         fs_connect_exception_message(invalid_path()[:-1])
 245     ),
 246     (
 247         {
 248             'path': invalid_path(),
 249             'uri': 'mongo:27017',
 250             'collection': 'files'
 251         },
 252         fs_connect_exception_message(invalid_path())
 253     ),
 254     (
 255         {
 256             'path': invalid_path(),
 257             'host': 'mongo',
 258             'port': 27017,
 259             'collection': 'files'
 260         },
 261         fs_connect_exception_message(invalid_path())
 262     ),
 263     (
 264         {
 265             'path': invalid_path()[:-1],
 266             'uri': 'mongo:27017',
 267             'collection': 'files'
 268         },
 269         fs_connect_exception_message(invalid_path()[:-1])
 270     ),
 271     (
 272         {
 273             'path': invalid_path()[:-1],
 274             'host': 'mongo',
 275             'port': 27017,
 276             'collection': 'files'
 277         },
 278         fs_connect_exception_message(invalid_path()[:-1])
 279     ),
 280     (
 281         {
 282             'path': '/',
 283             'host': 'mongo',
 284             'port': 27017,
 285             'collection': 'files'
 286         },
 287         generic_fs_exception_message(
 288             "Invalid configuration param at '[storage]': path '/' is not writable"
 289         )
 290     )])
 291 def test_fs_connect_with_invalid_path(config, exp_exception_message):
 292     fs = FsMongo()
 293     with pytest.raises(FsException) as excinfo:
 294         fs.fs_connect(config)
 295     assert str(excinfo.value) == exp_exception_message
 296
 297
 298 @pytest.mark.parametrize("config, exp_exception_message", [
 299     (
 300         {
 301             'logger_name': 'fs_mongo',
 302             'uri': 'mongo:27017',
 303             'collection': 'files'
 304         },
 305         "Missing parameter \"path\""
 306     ),
 307     (
 308         {
 309             'logger_name': 'fs_mongo',
 310             'host': 'mongo',
 311             'port': 27017,
 312             'collection': 'files'
 313         },
 314         "Missing parameter \"path\""
 315     ),
 316     (
 317         {
 318             'logger_name': 'fs_mongo',
 319             'path': valid_path(),
 320             'collection': 'files'
 321         },
 322         "Missing parameters: \"uri\" or \"host\" + \"port\""
 323     ),
 324     (
 325         {
 326             'logger_name': 'fs_mongo',
 327             'path': valid_path(),
 328             'port': 27017,
 329             'collection': 'files'
 330         },
 331         "Missing parameters: \"uri\" or \"host\" + \"port\""
 332     ),
 333     (
 334         {
 335             'logger_name': 'fs_mongo',
 336             'path': valid_path(),
 337             'host': 'mongo',
 338             'collection': 'files'
 339         },
 340         "Missing parameters: \"uri\" or \"host\" + \"port\""
 341     ),
 342     (
 343         {
 344             'logger_name': 'fs_mongo',
 345             'path': valid_path(),
 346             'uri': 'mongo:27017'
 347         },
 348         "Missing parameter \"collection\""
 349     ),
 350     (
 351         {
 352             'logger_name': 'fs_mongo',
 353             'path': valid_path(),
 354             'host': 'mongo',
 355             'port': 27017,
 356         },
 357         "Missing parameter \"collection\""
 358     )])
 359 def test_fs_connect_with_missing_parameters(config, exp_exception_message):
 360     fs = FsMongo()
 361     with pytest.raises(FsException) as excinfo:
 362         fs.fs_connect(config)
 363     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 364
 365
 366 @pytest.mark.parametrize("config, exp_exception_message", [
 367     (
 368         {
 369             'logger_name': 'fs_mongo',
 370             'path': valid_path(),
 371             'uri': 'mongo:27017',
 372             'collection': 'files'
 373         },
 374         "MongoClient crashed"
 375     ),
 376     (
 377         {
 378             'logger_name': 'fs_mongo',
 379             'path': valid_path(),
 380             'host': 'mongo',
 381             'port': 27017,
 382             'collection': 'files'
 383         },
 384         "MongoClient crashed"
 385     )])
 386 def test_fs_connect_with_invalid_mongoclient(config, exp_exception_message, monkeypatch):
 387     def generate_exception(a, b, c=None):
 388         raise Exception(exp_exception_message)
 389
 390     monkeypatch.setattr(MongoClient, '__init__', generate_exception)
 391
 392     fs = FsMongo()
 393     with pytest.raises(FsException) as excinfo:
 394         fs.fs_connect(config)
 395     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 396
 397
 398 @pytest.mark.parametrize("config, exp_exception_message", [
 399     (
 400         {
 401             'logger_name': 'fs_mongo',
 402             'path': valid_path(),
 403             'uri': 'mongo:27017',
 404             'collection': 'files'
 405         },
 406         "Collection unavailable"
 407     ),
 408     (
 409         {
 410             'logger_name': 'fs_mongo',
 411             'path': valid_path(),
 412             'host': 'mongo',
 413             'port': 27017,
 414             'collection': 'files'
 415         },
 416         "Collection unavailable"
 417     )])
 418 def test_fs_connect_with_invalid_mongo_collection(config, exp_exception_message, monkeypatch):
 419     def mock_mongoclient_constructor(a, b, c=None):
 420         pass
 421
 422     def generate_exception(a, b):
 423         raise Exception(exp_exception_message)
 424
 425     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
 426     monkeypatch.setattr(MongoClient, '__getitem__', generate_exception)
 427
 428     fs = FsMongo()
 429     with pytest.raises(FsException) as excinfo:
 430         fs.fs_connect(config)
 431     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 432
 433
 434 @pytest.mark.parametrize("config, exp_exception_message", [
 435     (
 436         {
 437             'logger_name': 'fs_mongo',
 438             'path': valid_path(),
 439             'uri': 'mongo:27017',
 440             'collection': 'files'
 441         },
 442         "GridFsBucket crashed"
 443     ),
 444     (
 445         {
 446             'logger_name': 'fs_mongo',
 447             'path': valid_path(),
 448             'host': 'mongo',
 449             'port': 27017,
 450             'collection': 'files'
 451         },
 452         "GridFsBucket crashed"
 453     )])
 454 def test_fs_connect_with_invalid_gridfsbucket(config, exp_exception_message, monkeypatch):
 455     def mock_mongoclient_constructor(a, b, c=None):
 456         pass
 457
 458     def mock_mongoclient_getitem(a, b):
 459         pass
 460
 461     def generate_exception(a, b):
 462         raise Exception(exp_exception_message)
 463
 464     monkeypatch.setattr(MongoClient, '__init__', mock_mongoclient_constructor)
 465     monkeypatch.setattr(MongoClient, '__getitem__', mock_mongoclient_getitem)
 466     monkeypatch.setattr(GridFSBucket, '__init__', generate_exception)
 467
 468     fs = FsMongo()
 469     with pytest.raises(FsException) as excinfo:
 470         fs.fs_connect(config)
 471     assert str(excinfo.value) == generic_fs_exception_message(exp_exception_message)
 472
 473
 474 def test_fs_disconnect(fs_mongo):
 475     fs_mongo.fs_disconnect()