Exits mon-evaluator process when encountered by a database exception
Database errors could mean database node failure. In this cases, the process
should exit so it could be automatically restarted (when handled by Docker Swarm, K8s, etc)
so it can obtain a new DB connection to an alive node.
Change-Id: I2575062d4bd23b3902b7c2ed14553d5201933002
Signed-off-by: Benjamin Diaz <bdiaz@whitestack.com>
diff --git a/osm_mon/cmd/mon_collector.py b/osm_mon/cmd/mon_collector.py
index 1c2aaa1..6389d49 100644
--- a/osm_mon/cmd/mon_collector.py
+++ b/osm_mon/cmd/mon_collector.py
@@ -46,7 +46,6 @@
log.info("Starting MON Collector...")
log.debug("Config: %s", vars(cfg))
collector = Collector()
- collector.init_backends()
collector.collect_forever()
diff --git a/osm_mon/collector/collector.py b/osm_mon/collector/collector.py
index 016014b..3143882 100644
--- a/osm_mon/collector/collector.py
+++ b/osm_mon/collector/collector.py
@@ -24,6 +24,8 @@
import multiprocessing
import time
+import peewee
+
from osm_mon.collector.backends.prometheus import PrometheusBackend
from osm_mon.collector.collectors.juju import VCACollector
from osm_mon.collector.collectors.openstack import OpenstackCollector
@@ -50,10 +52,7 @@
self.database_manager = DatabaseManager()
self.database_manager.create_tables()
self.queue = multiprocessing.Queue()
-
- def init_backends(self):
- for backend in METRIC_BACKENDS:
- self.plugins.append(backend())
+ self._init_backends()
def collect_forever(self):
log.debug('collect_forever')
@@ -62,6 +61,9 @@
try:
self.collect_metrics()
time.sleep(cfg.OSMMON_COLLECTOR_INTERVAL)
+ except peewee.PeeweeException:
+ log.exception("Database error consuming message: ")
+ raise
except Exception:
log.exception("Error collecting metrics")
@@ -106,3 +108,7 @@
metrics.append(self.queue.get())
for plugin in self.plugins:
plugin.handle(metrics)
+
+ def _init_backends(self):
+ for backend in METRIC_BACKENDS:
+ self.plugins.append(backend())
diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py
index 8f49a66..9dc8c48 100644
--- a/osm_mon/evaluator/evaluator.py
+++ b/osm_mon/evaluator/evaluator.py
@@ -25,6 +25,7 @@
import multiprocessing
import time
+import peewee
import requests
from osm_common.dbbase import DbException
@@ -87,6 +88,9 @@
try:
self.evaluate()
time.sleep(cfg.OSMMON_EVALUATOR_INTERVAL)
+ except peewee.PeeweeException:
+ log.exception("Database error evaluating alarms: ")
+ raise
except Exception:
log.exception("Error evaluating alarms")
diff --git a/osm_mon/tests/collector/test_collector.py b/osm_mon/tests/collector/test_collector.py
index f558e23..9159f35 100644
--- a/osm_mon/tests/collector/test_collector.py
+++ b/osm_mon/tests/collector/test_collector.py
@@ -20,35 +20,28 @@
# For those usages not covered by the Apache License, Version 2.0 please
# contact: bdiaz@whitestack.com or glavado@whitestack.com
##
+import os
import unittest
from unittest import mock
-from peewee import SqliteDatabase
-
from osm_mon.collector.collector import Collector
from osm_mon.collector.collectors.openstack import OpenstackCollector
-from osm_mon.core import database
-from osm_mon.core.database import DatabaseManager, VimCredentials, Alarm
-
-test_db = SqliteDatabase(':memory:')
-
-MODELS = [VimCredentials, Alarm]
+from osm_mon.core.database import DatabaseManager, db
class CollectorTest(unittest.TestCase):
def setUp(self):
super().setUp()
- database.db = test_db
- test_db.bind(MODELS)
- test_db.connect()
- test_db.drop_tables(MODELS)
- test_db.create_tables(MODELS)
+ os.environ["DATABASE"] = "sqlite:///:memory:"
+ db_manager = DatabaseManager()
+ db_manager.create_tables()
def tearDown(self):
super().tearDown()
- test_db.close()
+ db.close()
@mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock())
+ @mock.patch.object(Collector, "_init_backends", mock.Mock())
@mock.patch.object(OpenstackCollector, "__init__", lambda *args, **kwargs: None)
@mock.patch.object(OpenstackCollector, "collect")
@mock.patch.object(DatabaseManager, "get_vim_type")
@@ -59,6 +52,7 @@
collect.assert_called_once_with({})
@mock.patch("osm_mon.collector.collector.CommonDbClient", mock.Mock())
+ @mock.patch.object(Collector, "_init_backends", mock.Mock())
@mock.patch.object(OpenstackCollector, "collect")
@mock.patch.object(DatabaseManager, "get_vim_type")
def test_init_vim_collector_and_collect_unknown(self, _get_vim_type, openstack_collect):