X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_policy_module%2Falarming%2Fservice.py;h=0e355d886beb3aae596d3866f0eeca431eda3ffa;hb=refs%2Fchanges%2F55%2F11955%2F4;hp=590b06a4211b70380de42aec2a67438ac7466d77;hpb=889c53c007d28ee5355d218a11d6612ce2b09d21;p=osm%2FPOL.git diff --git a/osm_policy_module/alarming/service.py b/osm_policy_module/alarming/service.py index 590b06a..0e355d8 100644 --- a/osm_policy_module/alarming/service.py +++ b/osm_policy_module/alarming/service.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# pylint: disable=no-member # Copyright 2018 Whitestack, LLC # ************************************************************* @@ -24,22 +25,28 @@ import asyncio import json import logging +import operator +import functools import requests +from requests.exceptions import ConnectionError, RequestException from osm_policy_module.common.common_db_client import CommonDbClient from osm_policy_module.common.lcm_client import LcmClient from osm_policy_module.common.mon_client import MonClient from osm_policy_module.core import database from osm_policy_module.core.config import Config -from osm_policy_module.core.database import VnfAlarm, VnfAlarmRepository, AlarmActionRepository +from osm_policy_module.core.database import ( + VnfAlarm, + VnfAlarmRepository, + AlarmActionRepository, +) from osm_policy_module.core.exceptions import VdurNotFound log = logging.getLogger(__name__) class AlarmingService: - def __init__(self, config: Config, loop=None): self.conf = config if not loop: @@ -49,90 +56,133 @@ class AlarmingService: self.mon_client = MonClient(config, loop=self.loop) self.lcm_client = LcmClient(config, loop=self.loop) - async def configure_vnf_alarms(self, nsr_id: str): + async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None): log.info("Configuring vnf alarms for network service %s", nsr_id) alarms_created = [] - database.db.connect(reuse_if_open=True) - with database.db.atomic() as tx: - try: - vnfrs = self.db_client.get_vnfrs(nsr_id) + database.db.connect() + try: + with database.db.atomic(): + if vnf_member_index is None: + vnfrs = self.db_client.get_vnfrs(nsr_id) + else: + vnfrs = [] + vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index) + vnfrs.append(vnfr) + # vnfrs = self.db_client.get_vnfrs(nsr_id) for vnfr in vnfrs: log.debug("Processing vnfr: %s", vnfr) - vnfd = self.db_client.get_vnfd(vnfr['vnfd-id']) - for vdur in vnfr['vdur']: + vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"]) + for vdur in vnfr["vdur"]: vdu = next( filter( - lambda vdu: vdu['id'] == vdur['vdu-id-ref'], - vnfd['vdu'] + lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"] ) ) - if 'alarm' in vdu: - alarm_descriptors = vdu['alarm'] + if "alarm" in vdu: + alarm_descriptors = vdu["alarm"] for alarm_descriptor in alarm_descriptors: try: VnfAlarmRepository.get( - VnfAlarm.alarm_id == alarm_descriptor['alarm-id'], - VnfAlarm.vnf_member_index == vnfr['member-vnf-index-ref'], - VnfAlarm.vdu_name == vdur['name'], - VnfAlarm.nsr_id == nsr_id + VnfAlarm.alarm_id + == alarm_descriptor["alarm-id"], + VnfAlarm.vnf_member_index + == vnfr["member-vnf-index-ref"], + VnfAlarm.vdu_name == vdur["name"], + VnfAlarm.nsr_id == nsr_id, + ) + log.debug( + "vdu %s already has an alarm configured with same id %s", + vdur["name"], + alarm_descriptor["alarm-id"], ) - log.debug("vdu %s already has an alarm configured with same id %s", vdur['name'], - alarm_descriptor['alarm-id']) continue except VnfAlarm.DoesNotExist: pass vnf_monitoring_param = next( filter( - lambda param: param['id'] == alarm_descriptor['vnf-monitoring-param-ref'], - vnfd['monitoring-param']) + lambda param: param["id"] + == alarm_descriptor["vnf-monitoring-param-ref"], + vdu.get("monitoring-parameter", []), + ), + {}, ) + metric_name = self._get_metric_name( + vnf_monitoring_param + ) + alarm_action = dict() + for action_type in ["ok", "insufficient-data", "alarm"]: + if "actions" in alarm_descriptor and action_type in alarm_descriptor["actions"]: + for url in alarm_descriptor["actions"][action_type]: + if "webhook" in alarm_action: + alarm_action["webhook"].append(url["url"]) + else: + alarm_action["webhook"] = [url["url"]] alarm_uuid = await self.mon_client.create_alarm( - metric_name=alarm_descriptor['vnf-monitoring-param-ref'], + metric_name=metric_name, ns_id=nsr_id, - vdu_name=vdur['name'], - vnf_member_index=vnfr['member-vnf-index-ref'], - threshold=alarm_descriptor['value'], - operation=alarm_descriptor['operation'], - statistic=vnf_monitoring_param['aggregation-type'] + vdu_name=vdur["name"], + vnf_member_index=vnfr["member-vnf-index-ref"], + threshold=alarm_descriptor["value"], + operation=alarm_descriptor["operation"], + action=str(alarm_action), ) alarm = VnfAlarmRepository.create( - alarm_id=alarm_descriptor['alarm-id'], + alarm_id=alarm_descriptor["alarm-id"], alarm_uuid=alarm_uuid, nsr_id=nsr_id, - vnf_member_index=int(vnfr['member-vnf-index-ref']), - vdu_name=vdur['name'] + vnf_member_index=vnfr["member-vnf-index-ref"], + vdu_name=vdur["name"], + last_action='insufficient-data', + id_suffix=0, + ok_ack=False, + alarm_ack=False ) - for action_type in ['ok', 'insufficient-data', 'alarm']: - if 'actions' in alarm_descriptor and action_type in alarm_descriptor['actions']: - for url in alarm_descriptor['actions'][action_type]: + for action_type in ["ok", "insufficient-data", "alarm"]: + if ( + "actions" in alarm_descriptor + and action_type in alarm_descriptor["actions"] + ): + for url in alarm_descriptor["actions"][ + action_type + ]: AlarmActionRepository.create( type=action_type, - url=url['url'], - alarm=alarm + url=url["url"], + alarm=alarm, ) alarms_created.append(alarm) - except Exception as e: - log.exception("Error configuring VNF alarms:") - tx.rollback() - if len(alarms_created) > 0: - log.debug("Cleaning alarm resources in MON") - for alarm in alarms_created: - await self.mon_client.delete_alarm(alarm.nsr_id, - alarm.vnf_member_index, - alarm.vdu_name, - alarm.alarm_uuid) - raise e - database.db.close() + except Exception as e: + log.exception("Error configuring VNF alarms:") + if len(alarms_created) > 0: + log.debug("Cleaning alarm resources in MON") + for alarm in alarms_created: + try: + await self.mon_client.delete_alarm( + alarm.nsr_id, + alarm.vnf_member_index, + alarm.vdu_name, + alarm.alarm_uuid, + ) + except ValueError: + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) + raise e + finally: + database.db.close() async def delete_orphaned_alarms(self, nsr_id): + # TODO: Review as it seems this code is never called log.info("Deleting orphaned vnf alarms for network service %s", nsr_id) database.db.connect() - with database.db.atomic() as tx: - try: + try: + with database.db.atomic(): for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id): try: - self.db_client.get_vdur(nsr_id, alarm.vnf_member_index, alarm.vdu_name) + self.db_client.get_vdur( + nsr_id, alarm.vnf_member_index, alarm.vdu_name + ) except VdurNotFound: log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid) try: @@ -140,52 +190,136 @@ class AlarmingService: alarm.nsr_id, alarm.vnf_member_index, alarm.vdu_name, - alarm.alarm_uuid) + alarm.alarm_uuid, + ) except ValueError: - log.exception("Error deleting alarm in MON %s", alarm.alarm_uuid) + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) alarm.delete_instance() + except Exception as e: + log.exception("Error deleting orphaned alarms:") + raise e + finally: + database.db.close() - except Exception as e: - log.exception("Error deleting orphaned alarms:") - tx.rollback() - raise e - database.db.close() - - async def delete_vnf_alarms(self, nsr_id): + async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None): log.info("Deleting vnf alarms for network service %s", nsr_id) database.db.connect() - with database.db.atomic() as tx: - try: - for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id): + try: + with database.db.atomic(): + if vnf_member_index is None: + alarm_conditions = VnfAlarm.nsr_id == nsr_id + else: + query_list = [VnfAlarm.nsr_id == nsr_id, + VnfAlarm.vnf_member_index == vnf_member_index] + alarm_conditions = functools.reduce(operator.and_, query_list) + for alarm in VnfAlarmRepository.list(alarm_conditions): log.debug("Deleting vnf alarm %s", alarm.alarm_uuid) try: await self.mon_client.delete_alarm( alarm.nsr_id, alarm.vnf_member_index, alarm.vdu_name, - alarm.alarm_uuid) + alarm.alarm_uuid, + ) except ValueError: - log.exception("Error deleting alarm in MON %s", alarm.alarm_uuid) + log.exception( + "Error deleting alarm in MON %s", alarm.alarm_uuid + ) alarm.delete_instance() - except Exception as e: - log.exception("Error deleting orphaned alarms:") - tx.rollback() - raise e - database.db.close() + except Exception as e: + log.exception("Error deleting vnf alarms:") + raise e + finally: + database.db.close() async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict): + alert_timeout = int(self.conf.get('alert', 'timeout')) database.db.connect() try: with database.db.atomic(): alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid) - log.debug("Handling vnf alarm %s with status %s", alarm.alarm_id, status) + log.debug( + "Handling vnf alarm %s with status %s", alarm.alarm_id, status + ) for action in alarm.actions: + """ + Compares the current status with the last_action status. + If both the status are 'alarm', it avoid sending repetitive alarm notification. + If both the status are 'ok', it avoid sending repetitive ok notification. + """ if action.type == status: - log.info("Executing request to url %s for vnf alarm %s with status %s", action.url, - alarm.alarm_id, status) - requests.post(url=action.url, json=json.dumps(payload)) + if bool(self.conf.get('alert', 'enhanced_alarms')): + if ((status != "ok" or (status == "ok" and alarm.ok_ack is False)) and + (status != "alarm" or (status == "alarm" and alarm.alarm_ack is False))): + log.info( + "Executing request to url %s for vnf alarm %s with status %s", + action.url, + alarm.alarm_id, + status + ) + try: + if status == "alarm" and alarm.last_action == "ok": + alarm.id_suffix += 1 + alarm.ok_ack = False + if status == "ok" and alarm.last_action == "alarm": + alarm.alarm_ack = False + alarm.last_action = status + alarm.save() + except Exception as e: + log.exception(e) + + payload["notify_details"]["alarm_number"] = alarm.id_suffix + headers = {"content-type": "application/json"} + try: + resp = requests.post(url=action.url, data=json.dumps(payload), + headers=headers, verify=False, timeout=alert_timeout) + log.info("Response %s", resp) + if resp.status_code == 200: + if status == "ok": + alarm.ok_ack = True + alarm.save() + if status == "alarm": + alarm.alarm_ack = True + alarm.save() + if status == "insufficient-data": + alarm.alarm_ack = False + alarm.ok_ack = False + alarm.save() + except RequestException as e: + log.info("Error: RequestException while connecting to url %s", action.url) + log.debug("RequestException %s", e) + + except ConnectionError: + log.exception("Error connecting to url %s", action.url) + else: + log.info( + "Executing request to url %s for vnf alarm %s with status %s", + action.url, + alarm.alarm_id, + status + ) + try: + requests.post(url=action.url, json=json.dumps(payload), timeout=alert_timeout) + except RequestException as e: + log.info("Error: RequestException while connecting to url %s", action.url) + log.debug("RequestException %s", e) + except ConnectionError: + log.exception("Error connecting to url %s", action.url) + except VnfAlarm.DoesNotExist: - log.debug("There is no alarming action configured for alarm %s.", alarm_uuid) + log.debug( + "There is no alarming action configured for alarm %s.", alarm_uuid + ) finally: database.db.close() + + def _get_metric_name(self, vnf_monitoring_param: dict): + if "performance-metric" in vnf_monitoring_param: + return vnf_monitoring_param["performance-metric"] + raise ValueError( + "No metric name found for vnf_monitoring_param %s" + % vnf_monitoring_param["id"] + )