osm_policy_module/alarming/service.py

# -*- coding: utf-8 -*-
# pylint: disable=no-member

# Copyright 2018 Whitestack, LLC
# *************************************************************

# This file is part of OSM Monitoring module
# All Rights Reserved to Whitestack, LLC

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at

#         http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# For those usages not covered by the Apache License, Version 2.0 please
# contact: bdiaz@whitestack.com or glavado@whitestack.com
##
import asyncio
import json
import logging
import operator
import functools

import requests
from requests.exceptions import ConnectionError, RequestException

from osm_policy_module.common.common_db_client import CommonDbClient
from osm_policy_module.common.lcm_client import LcmClient
from osm_policy_module.common.mon_client import MonClient
from osm_policy_module.core import database
from osm_policy_module.core.config import Config
from osm_policy_module.core.database import (
    VnfAlarm,
    VnfAlarmRepository,
    AlarmActionRepository,
)
from osm_policy_module.core.exceptions import VdurNotFound

log = logging.getLogger(__name__)


class AlarmingService:
    def __init__(self, config: Config, loop=None):
        self.conf = config
        if not loop:
            loop = asyncio.get_event_loop()
        self.loop = loop
        self.db_client = CommonDbClient(config)
        self.mon_client = MonClient(config, loop=self.loop)
        self.lcm_client = LcmClient(config, loop=self.loop)

    async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
        log.info("Configuring vnf alarms for network service %s", nsr_id)
        alarms_created = []
        database.db.connect()
        try:
            with database.db.atomic():
                if vnf_member_index is None:
                    vnfrs = self.db_client.get_vnfrs(nsr_id)
                else:
                    vnfrs = []
                    vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
                    vnfrs.append(vnfr)
                # vnfrs = self.db_client.get_vnfrs(nsr_id)
                for vnfr in vnfrs:
                    log.debug("Processing vnfr: %s", vnfr)
                    vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
                    for vdur in vnfr["vdur"]:
                        vdu = next(
                            filter(
                                lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
                            )
                        )
                        if "alarm" in vdu:
                            alarm_descriptors = vdu["alarm"]
                            for alarm_descriptor in alarm_descriptors:
                                try:
                                    VnfAlarmRepository.get(
                                        VnfAlarm.alarm_id
                                        == alarm_descriptor["alarm-id"],
                                        VnfAlarm.vnf_member_index
                                        == vnfr["member-vnf-index-ref"],
                                        VnfAlarm.vdu_name == vdur["name"],
                                        VnfAlarm.nsr_id == nsr_id,
                                    )
                                    log.debug(
                                        "vdu %s already has an alarm configured with same id %s",
                                        vdur["name"],
                                        alarm_descriptor["alarm-id"],
                                    )
                                    continue
                                except VnfAlarm.DoesNotExist:
                                    pass
                                vnf_monitoring_param = next(
                                    filter(
                                        lambda param: param["id"]
                                        == alarm_descriptor["vnf-monitoring-param-ref"],
                                        vdu.get("monitoring-parameter", []),
                                    ),
                                    {},
                                )
                                metric_name = self._get_metric_name(
                                    vnf_monitoring_param
                                )
                                alarm_action = dict()
                                for action_type in ["ok", "insufficient-data", "alarm"]:
                                    if (
                                        "actions" in alarm_descriptor
                                        and action_type in alarm_descriptor["actions"]
                                    ):
                                        for url in alarm_descriptor["actions"][
                                            action_type
                                        ]:
                                            if "webhook" in alarm_action:
                                                alarm_action["webhook"].append(
                                                    url["url"]
                                                )
                                            else:
                                                alarm_action["webhook"] = [url["url"]]
                                alarm_uuid = await self.mon_client.create_alarm(
                                    metric_name=metric_name,
                                    ns_id=nsr_id,
                                    vdu_name=vdur["name"],
                                    vnf_member_index=vnfr["member-vnf-index-ref"],
                                    threshold=alarm_descriptor["value"],
                                    operation=alarm_descriptor["operation"],
                                    action=str(alarm_action),
                                )
                                alarm = VnfAlarmRepository.create(
                                    alarm_id=alarm_descriptor["alarm-id"],
                                    alarm_uuid=alarm_uuid,
                                    nsr_id=nsr_id,
                                    vnf_member_index=vnfr["member-vnf-index-ref"],
                                    vdu_name=vdur["name"],
                                    last_action="insufficient-data",
                                    id_suffix=0,
                                    ok_ack=False,
                                    alarm_ack=False,
                                )
                                for action_type in ["ok", "insufficient-data", "alarm"]:
                                    if (
                                        "actions" in alarm_descriptor
                                        and action_type in alarm_descriptor["actions"]
                                    ):
                                        for url in alarm_descriptor["actions"][
                                            action_type
                                        ]:
                                            AlarmActionRepository.create(
                                                type=action_type,
                                                url=url["url"],
                                                alarm=alarm,
                                            )
                                alarms_created.append(alarm)

        except Exception as e:
            log.exception("Error configuring VNF alarms:")
            if len(alarms_created) > 0:
                log.debug("Cleaning alarm resources in MON")
                for alarm in alarms_created:
                    try:
                        await self.mon_client.delete_alarm(
                            alarm.nsr_id,
                            alarm.vnf_member_index,
                            alarm.vdu_name,
                            alarm.alarm_uuid,
                        )
                    except ValueError:
                        log.exception(
                            "Error deleting alarm in MON %s", alarm.alarm_uuid
                        )
            raise e
        finally:
            database.db.close()

    async def delete_orphaned_alarms(self, nsr_id):
        # TODO: Review as it seems this code is never called
        log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
        database.db.connect()
        try:
            with database.db.atomic():
                for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
                    try:
                        self.db_client.get_vdur(
                            nsr_id, alarm.vnf_member_index, alarm.vdu_name
                        )
                    except VdurNotFound:
                        log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
                        try:
                            await self.mon_client.delete_alarm(
                                alarm.nsr_id,
                                alarm.vnf_member_index,
                                alarm.vdu_name,
                                alarm.alarm_uuid,
                            )
                        except ValueError:
                            log.exception(
                                "Error deleting alarm in MON %s", alarm.alarm_uuid
                            )
                        alarm.delete_instance()
        except Exception as e:
            log.exception("Error deleting orphaned alarms:")
            raise e
        finally:
            database.db.close()

    async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
        log.info("Deleting vnf alarms for network service %s", nsr_id)
        database.db.connect()
        try:
            with database.db.atomic():
                if vnf_member_index is None:
                    alarm_conditions = VnfAlarm.nsr_id == nsr_id
                else:
                    query_list = [
                        VnfAlarm.nsr_id == nsr_id,
                        VnfAlarm.vnf_member_index == vnf_member_index,
                    ]
                    alarm_conditions = functools.reduce(operator.and_, query_list)
                for alarm in VnfAlarmRepository.list(alarm_conditions):
                    log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
                    try:
                        await self.mon_client.delete_alarm(
                            alarm.nsr_id,
                            alarm.vnf_member_index,
                            alarm.vdu_name,
                            alarm.alarm_uuid,
                        )
                    except ValueError:
                        log.exception(
                            "Error deleting alarm in MON %s", alarm.alarm_uuid
                        )
                    alarm.delete_instance()

        except Exception as e:
            log.exception("Error deleting vnf alarms:")
            raise e
        finally:
            database.db.close()

    async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
        alert_timeout = int(self.conf.get("alert", "timeout"))
        database.db.connect()
        try:
            with database.db.atomic():
                alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
                log.debug(
                    "Handling vnf alarm %s with status %s", alarm.alarm_id, status
                )
                for action in alarm.actions:
                    """
                    Compares the current status with the last_action status.
                    If both the status are 'alarm', it avoid sending repetitive alarm notification.
                    If both the status are 'ok', it avoid sending repetitive ok notification.
                    """
                    if action.type == status:
                        if bool(self.conf.get("alert", "enhanced_alarms")):
                            if (
                                status != "ok"
                                or (status == "ok" and alarm.ok_ack is False)
                            ) and (
                                status != "alarm"
                                or (status == "alarm" and alarm.alarm_ack is False)
                            ):
                                log.info(
                                    "Executing request to url %s for vnf alarm %s with status %s",
                                    action.url,
                                    alarm.alarm_id,
                                    status,
                                )
                                try:
                                    if status == "alarm" and alarm.last_action == "ok":
                                        alarm.id_suffix += 1
                                        alarm.ok_ack = False
                                    if status == "ok" and alarm.last_action == "alarm":
                                        alarm.alarm_ack = False
                                    alarm.last_action = status
                                    alarm.save()
                                except Exception as e:
                                    log.exception(e)

                                payload["notify_details"][
                                    "alarm_number"
                                ] = alarm.id_suffix
                                headers = {"content-type": "application/json"}
                                try:
                                    resp = requests.post(
                                        url=action.url,
                                        data=json.dumps(payload),
                                        headers=headers,
                                        verify=False,
                                        timeout=alert_timeout,
                                    )
                                    log.info("Response %s", resp)
                                    if resp.status_code == 200:
                                        if status == "ok":
                                            alarm.ok_ack = True
                                            alarm.save()
                                        if status == "alarm":
                                            alarm.alarm_ack = True
                                            alarm.save()
                                        if status == "insufficient-data":
                                            alarm.alarm_ack = False
                                            alarm.ok_ack = False
                                            alarm.save()
                                except ConnectionError:
                                    log.exception(
                                        "Error connecting to url %s", action.url
                                    )
                                except RequestException as e:
                                    log.info(
                                        "Error: RequestException while connecting to url %s",
                                        action.url,
                                    )
                                    log.debug("RequestException %s", e)

                        else:
                            log.info(
                                "Executing request to url %s for vnf alarm %s with status %s",
                                action.url,
                                alarm.alarm_id,
                                status,
                            )
                            try:
                                requests.post(
                                    url=action.url,
                                    json=json.dumps(payload),
                                    timeout=alert_timeout,
                                )
                            except ConnectionError:
                                log.exception("Error connecting to url %s", action.url)
                            except RequestException as e:
                                log.info(
                                    "Error: RequestException while connecting to url %s",
                                    action.url,
                                )
                                log.debug("RequestException %s", e)

        except VnfAlarm.DoesNotExist:
            log.debug(
                "There is no alarming action configured for alarm %s.", alarm_uuid
            )
        finally:
            database.db.close()

    def _get_metric_name(self, vnf_monitoring_param: dict):
        if "performance-metric" in vnf_monitoring_param:
            return vnf_monitoring_param["performance-metric"]
        raise ValueError(
            "No metric name found for vnf_monitoring_param %s"
            % vnf_monitoring_param["id"]
        )