| # -*- coding: utf-8 -*- |
| # pylint: disable=no-member |
| |
| # Copyright 2018 Whitestack, LLC |
| # ************************************************************* |
| |
| # This file is part of OSM Monitoring module |
| # All Rights Reserved to Whitestack, LLC |
| |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| # For those usages not covered by the Apache License, Version 2.0 please |
| # contact: bdiaz@whitestack.com or glavado@whitestack.com |
| ## |
| import json |
| import logging |
| import operator |
| import functools |
| |
| import requests |
| from requests.exceptions import ConnectionError, RequestException |
| |
| from osm_policy_module.common.common_db_client import CommonDbClient |
| from osm_policy_module.common.lcm_client import LcmClient |
| from osm_policy_module.common.mon_client import MonClient |
| from osm_policy_module.core import database |
| from osm_policy_module.core.config import Config |
| from osm_policy_module.core.database import ( |
| VnfAlarm, |
| VnfAlarmRepository, |
| AlarmActionRepository, |
| ) |
| from osm_policy_module.core.exceptions import VdurNotFound |
| |
| log = logging.getLogger(__name__) |
| |
| |
| class AlarmingService: |
| def __init__(self, config: Config): |
| self.conf = config |
| self.db_client = CommonDbClient(config) |
| self.mon_client = MonClient(config) |
| self.lcm_client = LcmClient(config) |
| |
| async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None): |
| log.info("Configuring vnf alarms for network service %s", nsr_id) |
| alarms_created = [] |
| database.db.connect() |
| try: |
| with database.db.atomic(): |
| if vnf_member_index is None: |
| vnfrs = self.db_client.get_vnfrs(nsr_id) |
| else: |
| vnfrs = [] |
| vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index) |
| vnfrs.append(vnfr) |
| # vnfrs = self.db_client.get_vnfrs(nsr_id) |
| for vnfr in vnfrs: |
| log.debug("Processing vnfr: %s", vnfr) |
| vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"]) |
| for vdur in vnfr["vdur"]: |
| vdu = next( |
| filter( |
| lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"] |
| ) |
| ) |
| if "alarm" in vdu: |
| alarm_descriptors = vdu["alarm"] |
| for alarm_descriptor in alarm_descriptors: |
| try: |
| VnfAlarmRepository.get( |
| VnfAlarm.alarm_id |
| == alarm_descriptor["alarm-id"], |
| VnfAlarm.vnf_member_index |
| == vnfr["member-vnf-index-ref"], |
| VnfAlarm.vdu_name == vdur["name"], |
| VnfAlarm.nsr_id == nsr_id, |
| ) |
| log.debug( |
| "vdu %s already has an alarm configured with same id %s", |
| vdur["name"], |
| alarm_descriptor["alarm-id"], |
| ) |
| continue |
| except VnfAlarm.DoesNotExist: |
| pass |
| vnf_monitoring_param = next( |
| filter( |
| lambda param: param["id"] |
| == alarm_descriptor["vnf-monitoring-param-ref"], |
| vdu.get("monitoring-parameter", []), |
| ), |
| {}, |
| ) |
| metric_name = self._get_metric_name( |
| vnf_monitoring_param |
| ) |
| alarm_action = dict() |
| for action_type in ["ok", "insufficient-data", "alarm"]: |
| if ( |
| "actions" in alarm_descriptor |
| and action_type in alarm_descriptor["actions"] |
| ): |
| for url in alarm_descriptor["actions"][ |
| action_type |
| ]: |
| if "webhook" in alarm_action: |
| alarm_action["webhook"].append( |
| url["url"] |
| ) |
| else: |
| alarm_action["webhook"] = [url["url"]] |
| alarm_uuid = await self.mon_client.create_alarm( |
| metric_name=metric_name, |
| ns_id=nsr_id, |
| vdu_name=vdur["name"], |
| vnf_member_index=vnfr["member-vnf-index-ref"], |
| threshold=alarm_descriptor["value"], |
| operation=alarm_descriptor["operation"], |
| action=str(alarm_action), |
| ) |
| alarm = VnfAlarmRepository.create( |
| alarm_id=alarm_descriptor["alarm-id"], |
| alarm_uuid=alarm_uuid, |
| nsr_id=nsr_id, |
| vnf_member_index=vnfr["member-vnf-index-ref"], |
| vdu_name=vdur["name"], |
| last_action="insufficient-data", |
| id_suffix=0, |
| ok_ack=False, |
| alarm_ack=False, |
| ) |
| for action_type in ["ok", "insufficient-data", "alarm"]: |
| if ( |
| "actions" in alarm_descriptor |
| and action_type in alarm_descriptor["actions"] |
| ): |
| for url in alarm_descriptor["actions"][ |
| action_type |
| ]: |
| AlarmActionRepository.create( |
| type=action_type, |
| url=url["url"], |
| alarm=alarm, |
| ) |
| alarms_created.append(alarm) |
| |
| except Exception as e: |
| log.exception("Error configuring VNF alarms:") |
| if len(alarms_created) > 0: |
| log.debug("Cleaning alarm resources in MON") |
| for alarm in alarms_created: |
| try: |
| await self.mon_client.delete_alarm( |
| alarm.nsr_id, |
| alarm.vnf_member_index, |
| alarm.vdu_name, |
| alarm.alarm_uuid, |
| ) |
| except ValueError: |
| log.exception( |
| "Error deleting alarm in MON %s", alarm.alarm_uuid |
| ) |
| raise e |
| finally: |
| database.db.close() |
| |
| async def delete_orphaned_alarms(self, nsr_id): |
| # TODO: Review as it seems this code is never called |
| log.info("Deleting orphaned vnf alarms for network service %s", nsr_id) |
| database.db.connect() |
| try: |
| with database.db.atomic(): |
| for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id): |
| try: |
| self.db_client.get_vdur( |
| nsr_id, alarm.vnf_member_index, alarm.vdu_name |
| ) |
| except VdurNotFound: |
| log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid) |
| try: |
| await self.mon_client.delete_alarm( |
| alarm.nsr_id, |
| alarm.vnf_member_index, |
| alarm.vdu_name, |
| alarm.alarm_uuid, |
| ) |
| except ValueError: |
| log.exception( |
| "Error deleting alarm in MON %s", alarm.alarm_uuid |
| ) |
| alarm.delete_instance() |
| except Exception as e: |
| log.exception("Error deleting orphaned alarms:") |
| raise e |
| finally: |
| database.db.close() |
| |
| async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None): |
| log.info("Deleting vnf alarms for network service %s", nsr_id) |
| database.db.connect() |
| try: |
| with database.db.atomic(): |
| if vnf_member_index is None: |
| alarm_conditions = VnfAlarm.nsr_id == nsr_id |
| else: |
| query_list = [ |
| VnfAlarm.nsr_id == nsr_id, |
| VnfAlarm.vnf_member_index == vnf_member_index, |
| ] |
| alarm_conditions = functools.reduce(operator.and_, query_list) |
| for alarm in VnfAlarmRepository.list(alarm_conditions): |
| log.debug("Deleting vnf alarm %s", alarm.alarm_uuid) |
| try: |
| await self.mon_client.delete_alarm( |
| alarm.nsr_id, |
| alarm.vnf_member_index, |
| alarm.vdu_name, |
| alarm.alarm_uuid, |
| ) |
| except ValueError: |
| log.exception( |
| "Error deleting alarm in MON %s", alarm.alarm_uuid |
| ) |
| alarm.delete_instance() |
| |
| except Exception as e: |
| log.exception("Error deleting vnf alarms:") |
| raise e |
| finally: |
| database.db.close() |
| |
| async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict): |
| alert_timeout = int(self.conf.get("alert", "timeout")) |
| database.db.connect() |
| try: |
| with database.db.atomic(): |
| alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid) |
| log.debug( |
| "Handling vnf alarm %s with status %s", alarm.alarm_id, status |
| ) |
| for action in alarm.actions: |
| """ |
| Compares the current status with the last_action status. |
| If both the status are 'alarm', it avoid sending repetitive alarm notification. |
| If both the status are 'ok', it avoid sending repetitive ok notification. |
| """ |
| if action.type == status: |
| if bool(self.conf.get("alert", "enhanced_alarms")): |
| if ( |
| status != "ok" |
| or (status == "ok" and alarm.ok_ack is False) |
| ) and ( |
| status != "alarm" |
| or (status == "alarm" and alarm.alarm_ack is False) |
| ): |
| log.info( |
| "Executing request to url %s for vnf alarm %s with status %s", |
| action.url, |
| alarm.alarm_id, |
| status, |
| ) |
| try: |
| if status == "alarm" and alarm.last_action == "ok": |
| alarm.id_suffix += 1 |
| alarm.ok_ack = False |
| if status == "ok" and alarm.last_action == "alarm": |
| alarm.alarm_ack = False |
| alarm.last_action = status |
| alarm.save() |
| except Exception as e: |
| log.exception(e) |
| |
| payload["notify_details"][ |
| "alarm_number" |
| ] = alarm.id_suffix |
| headers = {"content-type": "application/json"} |
| try: |
| resp = requests.post( |
| url=action.url, |
| data=json.dumps(payload), |
| headers=headers, |
| timeout=alert_timeout, |
| ) |
| log.info("Response %s", resp) |
| if resp.status_code == 200: |
| if status == "ok": |
| alarm.ok_ack = True |
| alarm.save() |
| if status == "alarm": |
| alarm.alarm_ack = True |
| alarm.save() |
| if status == "insufficient-data": |
| alarm.alarm_ack = False |
| alarm.ok_ack = False |
| alarm.save() |
| except ConnectionError: |
| log.exception( |
| "Error connecting to url %s", action.url |
| ) |
| except RequestException as e: |
| log.info( |
| "Error: RequestException while connecting to url %s", |
| action.url, |
| ) |
| log.debug("RequestException %s", e) |
| |
| else: |
| log.info( |
| "Executing request to url %s for vnf alarm %s with status %s", |
| action.url, |
| alarm.alarm_id, |
| status, |
| ) |
| try: |
| requests.post( |
| url=action.url, |
| json=json.dumps(payload), |
| timeout=alert_timeout, |
| ) |
| except ConnectionError: |
| log.exception("Error connecting to url %s", action.url) |
| except RequestException as e: |
| log.info( |
| "Error: RequestException while connecting to url %s", |
| action.url, |
| ) |
| log.debug("RequestException %s", e) |
| |
| except VnfAlarm.DoesNotExist: |
| log.debug( |
| "There is no alarming action configured for alarm %s.", alarm_uuid |
| ) |
| finally: |
| database.db.close() |
| |
| def _get_metric_name(self, vnf_monitoring_param: dict): |
| if "performance-metric" in vnf_monitoring_param: |
| return vnf_monitoring_param["performance-metric"] |
| raise ValueError( |
| "No metric name found for vnf_monitoring_param %s" |
| % vnf_monitoring_param["id"] |
| ) |