X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_policy_module%2Falarming%2Fservice.py;h=d1d1ea567a335b2c6644437423f737f78c7167a0;hb=3fbf2fbf60e3aed3ec7ca0134839f19bc0d733d4;hp=b916724df940fa8793e9e5825b521a964aecdb88;hpb=acac7559b430f370dd06d09a40a3a387e440f922;p=osm%2FPOL.git diff --git a/osm_policy_module/alarming/service.py b/osm_policy_module/alarming/service.py index b916724..d1d1ea5 100644 --- a/osm_policy_module/alarming/service.py +++ b/osm_policy_module/alarming/service.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# pylint: disable=no-member # Copyright 2018 Whitestack, LLC # ************************************************************* @@ -26,20 +27,24 @@ import json import logging import requests +from requests.exceptions import ConnectionError, RequestException from osm_policy_module.common.common_db_client import CommonDbClient from osm_policy_module.common.lcm_client import LcmClient from osm_policy_module.common.mon_client import MonClient from osm_policy_module.core import database from osm_policy_module.core.config import Config -from osm_policy_module.core.database import VnfAlarm, VnfAlarmRepository, AlarmActionRepository +from osm_policy_module.core.database import ( + VnfAlarm, + VnfAlarmRepository, + AlarmActionRepository, +) from osm_policy_module.core.exceptions import VdurNotFound log = logging.getLogger(__name__) class AlarmingService: - def __init__(self, config: Config, loop=None): self.conf = config if not loop: @@ -58,57 +63,84 @@ class AlarmingService: vnfrs = self.db_client.get_vnfrs(nsr_id) for vnfr in vnfrs: log.debug("Processing vnfr: %s", vnfr) - vnfd = self.db_client.get_vnfd(vnfr['vnfd-id']) - for vdur in vnfr['vdur']: + vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"]) + for vdur in vnfr["vdur"]: vdu = next( filter( - lambda vdu: vdu['id'] == vdur['vdu-id-ref'], - vnfd['vdu'] + lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"] ) ) - if 'alarm' in vdu: - alarm_descriptors = vdu['alarm'] + if "alarm" in vdu: + alarm_descriptors = vdu["alarm"] for alarm_descriptor in alarm_descriptors: try: VnfAlarmRepository.get( - VnfAlarm.alarm_id == alarm_descriptor['alarm-id'], - VnfAlarm.vnf_member_index == vnfr['member-vnf-index-ref'], - VnfAlarm.vdu_name == vdur['name'], - VnfAlarm.nsr_id == nsr_id + VnfAlarm.alarm_id + == alarm_descriptor["alarm-id"], + VnfAlarm.vnf_member_index + == vnfr["member-vnf-index-ref"], + VnfAlarm.vdu_name == vdur["name"], + VnfAlarm.nsr_id == nsr_id, + ) + log.debug( + "vdu %s already has an alarm configured with same id %s", + vdur["name"], + alarm_descriptor["alarm-id"], ) - log.debug("vdu %s already has an alarm configured with same id %s", vdur['name'], - alarm_descriptor['alarm-id']) continue except VnfAlarm.DoesNotExist: pass vnf_monitoring_param = next( filter( - lambda param: param['id'] == alarm_descriptor['vnf-monitoring-param-ref'], - vnfd['monitoring-param']) + lambda param: param["id"] + == alarm_descriptor["vnf-monitoring-param-ref"], + vdu.get("monitoring-parameter", []), + ), + {}, + ) + metric_name = self._get_metric_name( + vnf_monitoring_param ) + alarm_action = dict() + for action_type in ["ok", "insufficient-data", "alarm"]: + if "actions" in alarm_descriptor and action_type in alarm_descriptor["actions"]: + for url in alarm_descriptor["actions"][action_type]: + if "webhook" in alarm_action: + alarm_action["webhook"].append(url["url"]) + else: + alarm_action["webhook"] = [url["url"]] alarm_uuid = await self.mon_client.create_alarm( - metric_name=alarm_descriptor['vnf-monitoring-param-ref'], + metric_name=metric_name, ns_id=nsr_id, - vdu_name=vdur['name'], - vnf_member_index=vnfr['member-vnf-index-ref'], - threshold=alarm_descriptor['value'], - operation=alarm_descriptor['operation'], - statistic=vnf_monitoring_param['aggregation-type'] + vdu_name=vdur["name"], + vnf_member_index=vnfr["member-vnf-index-ref"], + threshold=alarm_descriptor["value"], + operation=alarm_descriptor["operation"], + action=str(alarm_action), ) alarm = VnfAlarmRepository.create( - alarm_id=alarm_descriptor['alarm-id'], + alarm_id=alarm_descriptor["alarm-id"], alarm_uuid=alarm_uuid, nsr_id=nsr_id, - vnf_member_index=int(vnfr['member-vnf-index-ref']), - vdu_name=vdur['name'] + vnf_member_index=vnfr["member-vnf-index-ref"], + vdu_name=vdur["name"], + last_action='insufficient-data', + id_suffix=0, + ok_ack=False, + alarm_ack=False ) - for action_type in ['ok', 'insufficient-data', 'alarm']: - if 'actions' in alarm_descriptor and action_type in alarm_descriptor['actions']: - for url in alarm_descriptor['actions'][action_type]: + for action_type in ["ok", "insufficient-data", "alarm"]: + if ( + "actions" in alarm_descriptor + and action_type in alarm_descriptor["actions"] + ): + for url in alarm_descriptor["actions"][ + action_type + ]: AlarmActionRepository.create( type=action_type, - url=url['url'], - alarm=alarm + url=url["url"], + alarm=alarm, ) alarms_created.append(alarm) @@ -118,24 +150,31 @@ class AlarmingService: log.debug("Cleaning alarm resources in MON") for alarm in alarms_created: try: - await self.mon_client.delete_alarm(alarm.nsr_id, - alarm.vnf_member_index, - alarm.vdu_name, - alarm.alarm_uuid) + await self.mon_client.delete_alarm( + alarm.nsr_id, + alarm.vnf_member_index, + alarm.vdu_name, + alarm.alarm_uuid, + ) except ValueError: - log.exception("Error deleting alarm in MON %s", alarm.alarm_uuid) + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) raise e finally: database.db.close() async def delete_orphaned_alarms(self, nsr_id): + # TODO: Review as it seems this code is never called log.info("Deleting orphaned vnf alarms for network service %s", nsr_id) database.db.connect() try: with database.db.atomic(): for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id): try: - self.db_client.get_vdur(nsr_id, alarm.vnf_member_index, alarm.vdu_name) + self.db_client.get_vdur( + nsr_id, alarm.vnf_member_index, alarm.vdu_name + ) except VdurNotFound: log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid) try: @@ -143,9 +182,12 @@ class AlarmingService: alarm.nsr_id, alarm.vnf_member_index, alarm.vdu_name, - alarm.alarm_uuid) + alarm.alarm_uuid, + ) except ValueError: - log.exception("Error deleting alarm in MON %s", alarm.alarm_uuid) + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) alarm.delete_instance() except Exception as e: log.exception("Error deleting orphaned alarms:") @@ -165,9 +207,12 @@ class AlarmingService: alarm.nsr_id, alarm.vnf_member_index, alarm.vdu_name, - alarm.alarm_uuid) + alarm.alarm_uuid, + ) except ValueError: - log.exception("Error deleting alarm in MON %s", alarm.alarm_uuid) + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) alarm.delete_instance() except Exception as e: @@ -177,17 +222,90 @@ class AlarmingService: database.db.close() async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict): + alert_timeout = int(self.conf.get('alert', 'timeout')) database.db.connect() try: with database.db.atomic(): alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid) - log.debug("Handling vnf alarm %s with status %s", alarm.alarm_id, status) + log.debug( + "Handling vnf alarm %s with status %s", alarm.alarm_id, status + ) for action in alarm.actions: + """ + Compares the current status with the last_action status. + If both the status are 'alarm', it avoid sending repetitive alarm notification. + If both the status are 'ok', it avoid sending repetitive ok notification. + """ if action.type == status: - log.info("Executing request to url %s for vnf alarm %s with status %s", action.url, - alarm.alarm_id, status) - requests.post(url=action.url, json=json.dumps(payload)) + if bool(self.conf.get('alert', 'enhanced_alarms')): + if ((status != "ok" or (status == "ok" and alarm.ok_ack is False)) and + (status != "alarm" or (status == "alarm" and alarm.alarm_ack is False))): + log.info( + "Executing request to url %s for vnf alarm %s with status %s", + action.url, + alarm.alarm_id, + status + ) + try: + if status == "alarm" and alarm.last_action == "ok": + alarm.id_suffix += 1 + alarm.ok_ack = False + if status == "ok" and alarm.last_action == "alarm": + alarm.alarm_ack = False + alarm.last_action = status + alarm.save() + except Exception as e: + log.exception(e) + + payload["notify_details"]["alarm_number"] = alarm.id_suffix + headers = {"content-type": "application/json"} + try: + resp = requests.post(url=action.url, data=json.dumps(payload), + headers=headers, verify=False, timeout=alert_timeout) + log.info("Response %s", resp) + if resp.status_code == 200: + if status == "ok": + alarm.ok_ack = True + alarm.save() + if status == "alarm": + alarm.alarm_ack = True + alarm.save() + if status == "insufficient-data": + alarm.alarm_ack = False + alarm.ok_ack = False + alarm.save() + except RequestException as e: + log.info("Error: RequestException while connecting to url %s", action.url) + log.debug("RequestException %s", e) + + except ConnectionError: + log.exception("Error connecting to url %s", action.url) + else: + log.info( + "Executing request to url %s for vnf alarm %s with status %s", + action.url, + alarm.alarm_id, + status + ) + try: + requests.post(url=action.url, json=json.dumps(payload), timeout=alert_timeout) + except RequestException as e: + log.info("Error: RequestException while connecting to url %s", action.url) + log.debug("RequestException %s", e) + except ConnectionError: + log.exception("Error connecting to url %s", action.url) + except VnfAlarm.DoesNotExist: - log.debug("There is no alarming action configured for alarm %s.", alarm_uuid) + log.debug( + "There is no alarming action configured for alarm %s.", alarm_uuid + ) finally: database.db.close() + + def _get_metric_name(self, vnf_monitoring_param: dict): + if "performance-metric" in vnf_monitoring_param: + return vnf_monitoring_param["performance-metric"] + raise ValueError( + "No metric name found for vnf_monitoring_param %s" + % vnf_monitoring_param["id"] + )