osm_mon/evaluator/service.py

   1 # -*- coding: utf-8 -*-
   2
   3 # Copyright 2018 Whitestack, LLC
   4 # *************************************************************
   5
   6 # This file is part of OSM Monitoring module
   7 # All Rights Reserved to Whitestack, LLC
   8
   9 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  10 # not use this file except in compliance with the License. You may obtain
  11 # a copy of the License at
  12
  13 #         http://www.apache.org/licenses/LICENSE-2.0
  14
  15 # Unless required by applicable law or agreed to in writing, software
  16 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  17 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  18 # License for the specific language governing permissions and limitations
  19 # under the License.
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  22 ##
  23 import logging
  24 import multiprocessing
  25 from enum import Enum
  26 from typing import Tuple, List
  27
  28 from osm_mon.core import database
  29 from osm_mon.core.common_db import CommonDbClient
  30 from osm_mon.core.config import Config
  31 from osm_mon.core.database import Alarm, AlarmRepository
  32 from osm_mon.evaluator.backends.prometheus import PrometheusBackend
  33
  34 log = logging.getLogger(__name__)
  35
  36 BACKENDS = {
  37     'prometheus': PrometheusBackend
  38 }
  39
  40
  41 class AlarmStatus(Enum):
  42     ALARM = 'alarm'
  43     OK = 'ok'
  44     INSUFFICIENT = 'insufficient-data'
  45
  46
  47 class EvaluatorService:
  48
  49     def __init__(self, config: Config):
  50         self.conf = config
  51         self.common_db = CommonDbClient(self.conf)
  52         self.queue = multiprocessing.Queue()
  53
  54     def _get_metric_value(self,
  55                           metric_name: str,
  56                           tags: dict):
  57         return BACKENDS[self.conf.get('evaluator', 'backend')](self.conf).get_metric_value(metric_name, tags)
  58
  59     def _evaluate_metric(self,
  60                          alarm: Alarm, tags: dict):
  61         log.debug("_evaluate_metric")
  62         metric_value = self._get_metric_value(alarm.metric, tags)
  63         if metric_value is None:
  64             log.warning("No metric result for alarm %s", alarm.id)
  65             self.queue.put((alarm, AlarmStatus.INSUFFICIENT))
  66         else:
  67             if alarm.operation.upper() == 'GT':
  68                 if metric_value > alarm.threshold:
  69                     self.queue.put((alarm, AlarmStatus.ALARM))
  70                 else:
  71                     self.queue.put((alarm, AlarmStatus.OK))
  72             elif alarm.operation.upper() == 'LT':
  73                 if metric_value < alarm.threshold:
  74                     self.queue.put((alarm, AlarmStatus.ALARM))
  75                 else:
  76                     self.queue.put((alarm, AlarmStatus.OK))
  77
  78     def evaluate_alarms(self) -> List[Tuple[Alarm, AlarmStatus]]:
  79         log.debug('evaluate_alarms')
  80         processes = []
  81         database.db.connect()
  82         try:
  83             with database.db.atomic():
  84                 for alarm in AlarmRepository.list():
  85                     # Tags need to be passed inside a dict to avoid database locking issues related to process forking
  86                     tags = {}
  87                     for tag in alarm.tags:
  88                         tags[tag.name] = tag.value
  89                     p = multiprocessing.Process(target=self._evaluate_metric,
  90                                                 args=(alarm, tags))
  91                     processes.append(p)
  92                     p.start()
  93
  94                 for process in processes:
  95                     process.join(timeout=10)
  96                 alarms_tuples = []
  97                 log.info("Appending alarms to queue")
  98                 while not self.queue.empty():
  99                     alarms_tuples.append(self.queue.get())
 100                 return alarms_tuples
 101         finally:
 102             database.db.close()