osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import asyncio
  26 import json
  27 import logging
  28 import operator
  29 import functools
  30
  31 import requests
  32 from requests.exceptions import ConnectionError, RequestException
  33
  34 from osm_policy_module.common.common_db_client import CommonDbClient
  35 from osm_policy_module.common.lcm_client import LcmClient
  36 from osm_policy_module.common.mon_client import MonClient
  37 from osm_policy_module.core import database
  38 from osm_policy_module.core.config import Config
  39 from osm_policy_module.core.database import (
  40     VnfAlarm,
  41     VnfAlarmRepository,
  42     AlarmActionRepository,
  43 )
  44 from osm_policy_module.core.exceptions import VdurNotFound
  45
  46 log = logging.getLogger(__name__)
  47
  48
  49 class AlarmingService:
  50     def __init__(self, config: Config, loop=None):
  51         self.conf = config
  52         if not loop:
  53             loop = asyncio.get_event_loop()
  54         self.loop = loop
  55         self.db_client = CommonDbClient(config)
  56         self.mon_client = MonClient(config, loop=self.loop)
  57         self.lcm_client = LcmClient(config, loop=self.loop)
  58
  59     async def configure_vnf_alarms(self, nsr_id: str):
  60         log.info("Configuring vnf alarms for network service %s", nsr_id)
  61         alarms_created = []
  62         database.db.connect()
  63         try:
  64             with database.db.atomic():
  65                 vnfrs = self.db_client.get_vnfrs(nsr_id)
  66                 for vnfr in vnfrs:
  67                     log.debug("Processing vnfr: %s", vnfr)
  68                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  69                     for vdur in vnfr["vdur"]:
  70                         vdu = next(
  71                             filter(
  72                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  73                             )
  74                         )
  75                         if "alarm" in vdu:
  76                             alarm_descriptors = vdu["alarm"]
  77                             for alarm_descriptor in alarm_descriptors:
  78                                 try:
  79                                     VnfAlarmRepository.get(
  80                                         VnfAlarm.alarm_id
  81                                         == alarm_descriptor["alarm-id"],
  82                                         VnfAlarm.vnf_member_index
  83                                         == vnfr["member-vnf-index-ref"],
  84                                         VnfAlarm.vdu_name == vdur["name"],
  85                                         VnfAlarm.nsr_id == nsr_id,
  86                                     )
  87                                     log.debug(
  88                                         "vdu %s already has an alarm configured with same id %s",
  89                                         vdur["name"],
  90                                         alarm_descriptor["alarm-id"],
  91                                     )
  92                                     continue
  93                                 except VnfAlarm.DoesNotExist:
  94                                     pass
  95                                 vnf_monitoring_param = next(
  96                                     filter(
  97                                         lambda param: param["id"]
  98                                         == alarm_descriptor["vnf-monitoring-param-ref"],
  99                                         vdu.get("monitoring-parameter", []),
 100                                     ),
 101                                     {},
 102                                 )
 103                                 metric_name = self._get_metric_name(
 104                                     vnf_monitoring_param
 105                                 )
 106                                 alarm_action = dict()
 107                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 108                                     if "actions" in alarm_descriptor and action_type in alarm_descriptor["actions"]:
 109                                         for url in alarm_descriptor["actions"][action_type]:
 110                                             if "webhook" in alarm_action:
 111                                                 alarm_action["webhook"].append(url["url"])
 112                                             else:
 113                                                 alarm_action["webhook"] = [url["url"]]
 114                                 alarm_uuid = await self.mon_client.create_alarm(
 115                                     metric_name=metric_name,
 116                                     ns_id=nsr_id,
 117                                     vdu_name=vdur["name"],
 118                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 119                                     threshold=alarm_descriptor["value"],
 120                                     operation=alarm_descriptor["operation"],
 121                                     action=str(alarm_action),
 122                                 )
 123                                 alarm = VnfAlarmRepository.create(
 124                                     alarm_id=alarm_descriptor["alarm-id"],
 125                                     alarm_uuid=alarm_uuid,
 126                                     nsr_id=nsr_id,
 127                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 128                                     vdu_name=vdur["name"],
 129                                     last_action='insufficient-data',
 130                                     id_suffix=0,
 131                                     ok_ack=False,
 132                                     alarm_ack=False
 133                                 )
 134                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 135                                     if (
 136                                         "actions" in alarm_descriptor
 137                                         and action_type in alarm_descriptor["actions"]
 138                                     ):
 139                                         for url in alarm_descriptor["actions"][
 140                                             action_type
 141                                         ]:
 142                                             AlarmActionRepository.create(
 143                                                 type=action_type,
 144                                                 url=url["url"],
 145                                                 alarm=alarm,
 146                                             )
 147                                 alarms_created.append(alarm)
 148
 149         except Exception as e:
 150             log.exception("Error configuring VNF alarms:")
 151             if len(alarms_created) > 0:
 152                 log.debug("Cleaning alarm resources in MON")
 153                 for alarm in alarms_created:
 154                     try:
 155                         await self.mon_client.delete_alarm(
 156                             alarm.nsr_id,
 157                             alarm.vnf_member_index,
 158                             alarm.vdu_name,
 159                             alarm.alarm_uuid,
 160                         )
 161                     except ValueError:
 162                         log.exception(
 163                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 164                         )
 165             raise e
 166         finally:
 167             database.db.close()
 168
 169     async def delete_orphaned_alarms(self, nsr_id):
 170         # TODO: Review as it seems this code is never called
 171         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 172         database.db.connect()
 173         try:
 174             with database.db.atomic():
 175                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 176                     try:
 177                         self.db_client.get_vdur(
 178                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 179                         )
 180                     except VdurNotFound:
 181                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 182                         try:
 183                             await self.mon_client.delete_alarm(
 184                                 alarm.nsr_id,
 185                                 alarm.vnf_member_index,
 186                                 alarm.vdu_name,
 187                                 alarm.alarm_uuid,
 188                             )
 189                         except ValueError:
 190                             log.exception(
 191                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 192                             )
 193                         alarm.delete_instance()
 194         except Exception as e:
 195             log.exception("Error deleting orphaned alarms:")
 196             raise e
 197         finally:
 198             database.db.close()
 199
 200     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 201         log.info("Deleting vnf alarms for network service %s", nsr_id)
 202         database.db.connect()
 203         try:
 204             with database.db.atomic():
 205                 if vnf_member_index is None:
 206                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 207                 else:
 208                     query_list = [VnfAlarm.nsr_id == nsr_id,
 209                                   VnfAlarm.vnf_member_index == vnf_member_index]
 210                     alarm_conditions = functools.reduce(operator.and_, query_list)
 211                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 212                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 213                     try:
 214                         await self.mon_client.delete_alarm(
 215                             alarm.nsr_id,
 216                             alarm.vnf_member_index,
 217                             alarm.vdu_name,
 218                             alarm.alarm_uuid,
 219                         )
 220                     except ValueError:
 221                         log.exception(
 222                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 223                         )
 224                     alarm.delete_instance()
 225
 226         except Exception as e:
 227             log.exception("Error deleting vnf alarms:")
 228             raise e
 229         finally:
 230             database.db.close()
 231
 232     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 233         alert_timeout = int(self.conf.get('alert', 'timeout'))
 234         database.db.connect()
 235         try:
 236             with database.db.atomic():
 237                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 238                 log.debug(
 239                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 240                 )
 241                 for action in alarm.actions:
 242                     """
 243                     Compares the current status with the last_action status.
 244                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 245                     If both the status are 'ok', it avoid sending repetitive ok notification.
 246                     """
 247                     if action.type == status:
 248                         if bool(self.conf.get('alert', 'enhanced_alarms')):
 249                             if ((status != "ok" or (status == "ok" and alarm.ok_ack is False)) and
 250                                (status != "alarm" or (status == "alarm" and alarm.alarm_ack is False))):
 251                                 log.info(
 252                                     "Executing request to url %s for vnf alarm %s with status %s",
 253                                     action.url,
 254                                     alarm.alarm_id,
 255                                     status
 256                                 )
 257                                 try:
 258                                     if status == "alarm" and alarm.last_action == "ok":
 259                                         alarm.id_suffix += 1
 260                                         alarm.ok_ack = False
 261                                     if status == "ok" and alarm.last_action == "alarm":
 262                                         alarm.alarm_ack = False
 263                                     alarm.last_action = status
 264                                     alarm.save()
 265                                 except Exception as e:
 266                                     log.exception(e)
 267
 268                                 payload["notify_details"]["alarm_number"] = alarm.id_suffix
 269                                 headers = {"content-type": "application/json"}
 270                                 try:
 271                                     resp = requests.post(url=action.url, data=json.dumps(payload),
 272                                                          headers=headers, verify=False, timeout=alert_timeout)
 273                                     log.info("Response %s", resp)
 274                                     if resp.status_code == 200:
 275                                         if status == "ok":
 276                                             alarm.ok_ack = True
 277                                             alarm.save()
 278                                         if status == "alarm":
 279                                             alarm.alarm_ack = True
 280                                             alarm.save()
 281                                         if status == "insufficient-data":
 282                                             alarm.alarm_ack = False
 283                                             alarm.ok_ack = False
 284                                             alarm.save()
 285                                 except RequestException as e:
 286                                     log.info("Error: RequestException while connecting to url %s", action.url)
 287                                     log.debug("RequestException %s", e)
 288
 289                                 except ConnectionError:
 290                                     log.exception("Error connecting to url %s", action.url)
 291                         else:
 292                             log.info(
 293                                 "Executing request to url %s for vnf alarm %s with status %s",
 294                                 action.url,
 295                                 alarm.alarm_id,
 296                                 status
 297                             )
 298                             try:
 299                                 requests.post(url=action.url, json=json.dumps(payload), timeout=alert_timeout)
 300                             except RequestException as e:
 301                                 log.info("Error: RequestException while connecting to url %s", action.url)
 302                                 log.debug("RequestException %s", e)
 303                             except ConnectionError:
 304                                 log.exception("Error connecting to url %s", action.url)
 305
 306         except VnfAlarm.DoesNotExist:
 307             log.debug(
 308                 "There is no alarming action configured for alarm %s.", alarm_uuid
 309             )
 310         finally:
 311             database.db.close()
 312
 313     def _get_metric_name(self, vnf_monitoring_param: dict):
 314         if "performance-metric" in vnf_monitoring_param:
 315             return vnf_monitoring_param["performance-metric"]
 316         raise ValueError(
 317             "No metric name found for vnf_monitoring_param %s"
 318             % vnf_monitoring_param["id"]
 319         )