X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_mon%2Fevaluator%2Fevaluator.py;h=732f8ac7699df08e2921f48fd5916ece8a87b93d;hb=refs%2Fchanges%2F67%2F10967%2F1;hp=d8589bb5665985936c2b046233c00e59300c9651;hpb=c811d6839dac356b54a7cbb39f7687587e9947de;p=osm%2FMON.git diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py index d8589bb..732f8ac 100644 --- a/osm_mon/evaluator/evaluator.py +++ b/osm_mon/evaluator/evaluator.py @@ -22,8 +22,8 @@ ## import asyncio import logging +import multiprocessing import time -import concurrent.futures from osm_mon.core.config import Config from osm_mon.core.message_bus_client import MessageBusClient @@ -35,7 +35,6 @@ log = logging.getLogger(__name__) class Evaluator: - def __init__(self, config: Config, loop=None): self.conf = config if not loop: @@ -45,64 +44,39 @@ class Evaluator: self.msg_bus = MessageBusClient(config) def evaluate_forever(self): - log.debug('evaluate_forever') + log.debug("evaluate_forever") while True: try: self.evaluate() - time.sleep(int(self.conf.get('evaluator', 'interval'))) + time.sleep(int(self.conf.get("evaluator", "interval"))) except Exception: log.exception("Error evaluating alarms") def evaluate(self): - log.debug('evaluate') + log.debug("evaluate") + log.info("Starting alarm evaluation") alarms_tuples = self.service.evaluate_alarms() - # Starting evaluate executor pool with pool size process_pool_size. Default process_pool_size is 20 - with concurrent.futures.ProcessPoolExecutor(self.conf.get('evaluator', 'process_pool_size')) as executor: - log.info('Started evaluate process pool with pool size %s' % (self.conf.get('evaluator', - 'process_pool_size'))) - evaluate_futures = [] - for alarm, status in alarms_tuples: - evaluate_futures.append(executor.submit(Evaluator._notify_alarm, self.conf, alarm, status)) - - try: - # Wait for future calls to complete till process_timeout. Default is 50 seconds - for evaluate_future in concurrent.futures.as_completed(evaluate_futures, - self.conf.get('evaluator', 'process_timeout')): - result = evaluate_future.result(timeout=int(self.conf.get('evaluator', - 'process_timeout'))) - log.debug('result = %s' % (result)) - except concurrent.futures.TimeoutError as e: - # Some processes have not completed due to timeout error - log.info('Some processes have not finished due to TimeoutError exception') - log.debug('concurrent.futures.TimeoutError exception %s' % (e)) - Evaluator._stop_process_pool(executor) - - @staticmethod - def _stop_process_pool(executor): - log.debug("_stop_process_pool") - log.info('Stopping all processes in the process pool') - try: - for pid, process in executor._processes.items(): - if process.is_alive(): - process.terminate() - except Exception as e: - log.info("Exception during process termination") - log.debug("Exception %s" % (e)) - executor.shutdown() - return - - @staticmethod - def _notify_alarm(conf: Config, alarm: Alarm, status: AlarmStatus): + processes = [] + for alarm, status in alarms_tuples: + p = multiprocessing.Process(target=self.notify_alarm, args=(alarm, status)) + p.start() + processes.append(p) + for process in processes: + process.join(timeout=10) + log.info("Alarm evaluation is complete") + + def notify_alarm(self, alarm: Alarm, status: AlarmStatus): log.debug("_notify_alarm") - resp_message = Evaluator._build_alarm_response(alarm, status) - msg_bus = MessageBusClient(conf) - loop = asyncio.get_event_loop() + resp_message = self._build_alarm_response(alarm, status) log.info("Sent alarm notification: %s", resp_message) - loop.run_until_complete(msg_bus.aiowrite('alarm_response', 'notify_alarm', resp_message)) + self.loop.run_until_complete( + self.msg_bus.aiowrite("alarm_response", "notify_alarm", resp_message) + ) + evaluator_service = EvaluatorService(self.conf) + evaluator_service.update_alarm_status(status.value, alarm.uuid) return - @staticmethod - def _build_alarm_response(alarm: Alarm, status: AlarmStatus): + def _build_alarm_response(self, alarm: Alarm, status: AlarmStatus): log.debug("_build_alarm_response") response = ResponseBuilder() tags = {} @@ -110,7 +84,7 @@ class Evaluator: tags[name] = value now = time.strftime("%d-%m-%Y") + " " + time.strftime("%X") return response.generate_response( - 'notify_alarm', + "notify_alarm", alarm_id=alarm.uuid, metric_name=alarm.metric, operation=alarm.operation, @@ -118,4 +92,5 @@ class Evaluator: sev=alarm.severity, status=status.value, date=now, - tags=tags) + tags=tags, + )