Exits mon-evaluator process when encountered by a database exception

Database errors could mean database node failure. In this cases, the process
should exit so it could be automatically restarted (when handled by Docker Swarm, K8s, etc)
so it can obtain a new DB connection to an alive node.

Change-Id: I2575062d4bd23b3902b7c2ed14553d5201933002
Signed-off-by: Benjamin Diaz <bdiaz@whitestack.com>

diff --git a/osm_mon/cmd/mon_collector.py b/osm_mon/cmd/mon_collector.py
index 1c2aaa1..6389d49 100644 (file)
--- a/osm_mon/cmd/mon_collector.py
+++ b/osm_mon/cmd/mon_collector.py
@@ -46,7 +46,6 @@ def main():
     log.info("Starting MON Collector...")
     log.debug("Config: %s", vars(cfg))
     collector = Collector()
-    collector.init_backends()
     collector.collect_forever()
 
 

diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py
index 016014b..3143882 100644 (file)
--- a/osm_mon/collector/collector.py
+++ b/osm_mon/collector/collector.py
@@ -24,6 +24,8 @@ import logging
 import multiprocessing
 import time
 
+import peewee
+
 from osm_mon.collector.backends.prometheus import PrometheusBackend
 from osm_mon.collector.collectors.juju import VCACollector
 from osm_mon.collector.collectors.openstack import OpenstackCollector
@@ -50,10 +52,7 @@ class Collector:
         self.database_manager = DatabaseManager()
         self.database_manager.create_tables()
         self.queue = multiprocessing.Queue()
-
-    def init_backends(self):
-        for backend in METRIC_BACKENDS:
-            self.plugins.append(backend())
+        self._init_backends()
 
     def collect_forever(self):
         log.debug('collect_forever')
@@ -62,6 +61,9 @@ class Collector:
             try:
                 self.collect_metrics()
                 time.sleep(cfg.OSMMON_COLLECTOR_INTERVAL)
+            except peewee.PeeweeException:
+                log.exception("Database error consuming message: ")
+                raise
             except Exception:
                 log.exception("Error collecting metrics")
 
@@ -106,3 +108,7 @@ class Collector:
             metrics.append(self.queue.get())
         for plugin in self.plugins:
             plugin.handle(metrics)
+
+    def _init_backends(self):
+        for backend in METRIC_BACKENDS:
+            self.plugins.append(backend())

diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py
index 8f49a66..9dc8c48 100644 (file)
--- a/osm_mon/evaluator/evaluator.py
+++ b/osm_mon/evaluator/evaluator.py
@@ -25,6 +25,7 @@ import logging
 import multiprocessing
 import time
 
+import peewee
 import requests
 from osm_common.dbbase import DbException
 
@@ -87,6 +88,9 @@ class Evaluator:
             try:
                 self.evaluate()
                 time.sleep(cfg.OSMMON_EVALUATOR_INTERVAL)
+            except peewee.PeeweeException:
+                log.exception("Database error evaluating alarms: ")
+                raise
             except Exception:
                 log.exception("Error evaluating alarms")
 

diff --git a/osm_mon/tests/collector/test_collector.py b/osm_mon/tests/collector/test_collector.py
index f558e23..9159f35 100644 (file)
--- a/osm_mon/tests/collector/test_collector.py
+++ b/osm_mon/tests/collector/test_collector.py
@@ -20,35 +20,28 @@
 # For those usages not covered by the Apache License, Version 2.0 please
 # contact: bdiaz@whitestack.com or glavado@whitestack.com
 ##
+import os
 import unittest
 from unittest import mock
 
-from peewee import SqliteDatabase
-
 from osm_mon.collector.collector import Collector
 from osm_mon.collector.collectors.openstack import OpenstackCollector
-from osm_mon.core import database
-from osm_mon.core.database import DatabaseManager, VimCredentials, Alarm
-
-test_db = SqliteDatabase(':memory:')
-
-MODELS = [VimCredentials, Alarm]
+from osm_mon.core.database import DatabaseManager, db
 
 
 class CollectorTest(unittest.TestCase):
     def setUp(self):
         super().setUp()
-        database.db = test_db
-        test_db.bind(MODELS)
-        test_db.connect()
-        test_db.drop_tables(MODELS)
-        test_db.create_tables(MODELS)
+        os.environ["DATABASE"] = "sqlite:///:memory:"
+        db_manager = DatabaseManager()
+        db_manager.create_tables()
 
     def tearDown(self):
         super().tearDown()
-        test_db.close()
+        db.close()
 
     @mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock())
+    @mock.patch.object(Collector, "_init_backends", mock.Mock())
     @mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None)
     @mock.patch.object(OpenstackCollector, "collect")
     @mock.patch.object(DatabaseManager, "get_vim_type")
@@ -59,6 +52,7 @@ class CollectorTest(unittest.TestCase):
         collect.assert_called_once_with({})
 
     @mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock())
+    @mock.patch.object(Collector, "_init_backends", mock.Mock())
     @mock.patch.object(OpenstackCollector, "collect")
     @mock.patch.object(DatabaseManager, "get_vim_type")
     def test_init_vim_collector_and_collect_unknown(self, _get_vim_type, openstack_collect):