osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import asyncio
  26 import json
  27 import logging
  28 import operator
  29 import functools
  30
  31 import requests
  32 from requests.exceptions import ConnectionError, RequestException
  33
  34 from osm_policy_module.common.common_db_client import CommonDbClient
  35 from osm_policy_module.common.lcm_client import LcmClient
  36 from osm_policy_module.common.mon_client import MonClient
  37 from osm_policy_module.core import database
  38 from osm_policy_module.core.config import Config
  39 from osm_policy_module.core.database import (
  40     VnfAlarm,
  41     VnfAlarmRepository,
  42     AlarmActionRepository,
  43 )
  44 from osm_policy_module.core.exceptions import VdurNotFound
  45
  46 log = logging.getLogger(__name__)
  47
  48
  49 class AlarmingService:
  50     def __init__(self, config: Config, loop=None):
  51         self.conf = config
  52         if not loop:
  53             loop = asyncio.get_event_loop()
  54         self.loop = loop
  55         self.db_client = CommonDbClient(config)
  56         self.mon_client = MonClient(config, loop=self.loop)
  57         self.lcm_client = LcmClient(config, loop=self.loop)
  58
  59     async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
  60         log.info("Configuring vnf alarms for network service %s", nsr_id)
  61         alarms_created = []
  62         database.db.connect()
  63         try:
  64             with database.db.atomic():
  65                 if vnf_member_index is None:
  66                     vnfrs = self.db_client.get_vnfrs(nsr_id)
  67                 else:
  68                     vnfrs = []
  69                     vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
  70                     vnfrs.append(vnfr)
  71                 # vnfrs = self.db_client.get_vnfrs(nsr_id)
  72                 for vnfr in vnfrs:
  73                     log.debug("Processing vnfr: %s", vnfr)
  74                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  75                     for vdur in vnfr["vdur"]:
  76                         vdu = next(
  77                             filter(
  78                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  79                             )
  80                         )
  81                         if "alarm" in vdu:
  82                             alarm_descriptors = vdu["alarm"]
  83                             for alarm_descriptor in alarm_descriptors:
  84                                 try:
  85                                     VnfAlarmRepository.get(
  86                                         VnfAlarm.alarm_id
  87                                         == alarm_descriptor["alarm-id"],
  88                                         VnfAlarm.vnf_member_index
  89                                         == vnfr["member-vnf-index-ref"],
  90                                         VnfAlarm.vdu_name == vdur["name"],
  91                                         VnfAlarm.nsr_id == nsr_id,
  92                                     )
  93                                     log.debug(
  94                                         "vdu %s already has an alarm configured with same id %s",
  95                                         vdur["name"],
  96                                         alarm_descriptor["alarm-id"],
  97                                     )
  98                                     continue
  99                                 except VnfAlarm.DoesNotExist:
 100                                     pass
 101                                 vnf_monitoring_param = next(
 102                                     filter(
 103                                         lambda param: param["id"]
 104                                         == alarm_descriptor["vnf-monitoring-param-ref"],
 105                                         vdu.get("monitoring-parameter", []),
 106                                     ),
 107                                     {},
 108                                 )
 109                                 metric_name = self._get_metric_name(
 110                                     vnf_monitoring_param
 111                                 )
 112                                 alarm_action = dict()
 113                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 114                                     if "actions" in alarm_descriptor and action_type in alarm_descriptor["actions"]:
 115                                         for url in alarm_descriptor["actions"][action_type]:
 116                                             if "webhook" in alarm_action:
 117                                                 alarm_action["webhook"].append(url["url"])
 118                                             else:
 119                                                 alarm_action["webhook"] = [url["url"]]
 120                                 alarm_uuid = await self.mon_client.create_alarm(
 121                                     metric_name=metric_name,
 122                                     ns_id=nsr_id,
 123                                     vdu_name=vdur["name"],
 124                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 125                                     threshold=alarm_descriptor["value"],
 126                                     operation=alarm_descriptor["operation"],
 127                                     action=str(alarm_action),
 128                                 )
 129                                 alarm = VnfAlarmRepository.create(
 130                                     alarm_id=alarm_descriptor["alarm-id"],
 131                                     alarm_uuid=alarm_uuid,
 132                                     nsr_id=nsr_id,
 133                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 134                                     vdu_name=vdur["name"],
 135                                     last_action='insufficient-data',
 136                                     id_suffix=0,
 137                                     ok_ack=False,
 138                                     alarm_ack=False
 139                                 )
 140                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 141                                     if (
 142                                         "actions" in alarm_descriptor
 143                                         and action_type in alarm_descriptor["actions"]
 144                                     ):
 145                                         for url in alarm_descriptor["actions"][
 146                                             action_type
 147                                         ]:
 148                                             AlarmActionRepository.create(
 149                                                 type=action_type,
 150                                                 url=url["url"],
 151                                                 alarm=alarm,
 152                                             )
 153                                 alarms_created.append(alarm)
 154
 155         except Exception as e:
 156             log.exception("Error configuring VNF alarms:")
 157             if len(alarms_created) > 0:
 158                 log.debug("Cleaning alarm resources in MON")
 159                 for alarm in alarms_created:
 160                     try:
 161                         await self.mon_client.delete_alarm(
 162                             alarm.nsr_id,
 163                             alarm.vnf_member_index,
 164                             alarm.vdu_name,
 165                             alarm.alarm_uuid,
 166                         )
 167                     except ValueError:
 168                         log.exception(
 169                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 170                         )
 171             raise e
 172         finally:
 173             database.db.close()
 174
 175     async def delete_orphaned_alarms(self, nsr_id):
 176         # TODO: Review as it seems this code is never called
 177         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 178         database.db.connect()
 179         try:
 180             with database.db.atomic():
 181                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 182                     try:
 183                         self.db_client.get_vdur(
 184                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 185                         )
 186                     except VdurNotFound:
 187                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 188                         try:
 189                             await self.mon_client.delete_alarm(
 190                                 alarm.nsr_id,
 191                                 alarm.vnf_member_index,
 192                                 alarm.vdu_name,
 193                                 alarm.alarm_uuid,
 194                             )
 195                         except ValueError:
 196                             log.exception(
 197                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 198                             )
 199                         alarm.delete_instance()
 200         except Exception as e:
 201             log.exception("Error deleting orphaned alarms:")
 202             raise e
 203         finally:
 204             database.db.close()
 205
 206     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 207         log.info("Deleting vnf alarms for network service %s", nsr_id)
 208         database.db.connect()
 209         try:
 210             with database.db.atomic():
 211                 if vnf_member_index is None:
 212                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 213                 else:
 214                     query_list = [VnfAlarm.nsr_id == nsr_id,
 215                                   VnfAlarm.vnf_member_index == vnf_member_index]
 216                     alarm_conditions = functools.reduce(operator.and_, query_list)
 217                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 218                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 219                     try:
 220                         await self.mon_client.delete_alarm(
 221                             alarm.nsr_id,
 222                             alarm.vnf_member_index,
 223                             alarm.vdu_name,
 224                             alarm.alarm_uuid,
 225                         )
 226                     except ValueError:
 227                         log.exception(
 228                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 229                         )
 230                     alarm.delete_instance()
 231
 232         except Exception as e:
 233             log.exception("Error deleting vnf alarms:")
 234             raise e
 235         finally:
 236             database.db.close()
 237
 238     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 239         alert_timeout = int(self.conf.get('alert', 'timeout'))
 240         database.db.connect()
 241         try:
 242             with database.db.atomic():
 243                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 244                 log.debug(
 245                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 246                 )
 247                 for action in alarm.actions:
 248                     """
 249                     Compares the current status with the last_action status.
 250                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 251                     If both the status are 'ok', it avoid sending repetitive ok notification.
 252                     """
 253                     if action.type == status:
 254                         if bool(self.conf.get('alert', 'enhanced_alarms')):
 255                             if ((status != "ok" or (status == "ok" and alarm.ok_ack is False)) and
 256                                (status != "alarm" or (status == "alarm" and alarm.alarm_ack is False))):
 257                                 log.info(
 258                                     "Executing request to url %s for vnf alarm %s with status %s",
 259                                     action.url,
 260                                     alarm.alarm_id,
 261                                     status
 262                                 )
 263                                 try:
 264                                     if status == "alarm" and alarm.last_action == "ok":
 265                                         alarm.id_suffix += 1
 266                                         alarm.ok_ack = False
 267                                     if status == "ok" and alarm.last_action == "alarm":
 268                                         alarm.alarm_ack = False
 269                                     alarm.last_action = status
 270                                     alarm.save()
 271                                 except Exception as e:
 272                                     log.exception(e)
 273
 274                                 payload["notify_details"]["alarm_number"] = alarm.id_suffix
 275                                 headers = {"content-type": "application/json"}
 276                                 try:
 277                                     resp = requests.post(url=action.url, data=json.dumps(payload),
 278                                                          headers=headers, verify=False, timeout=alert_timeout)
 279                                     log.info("Response %s", resp)
 280                                     if resp.status_code == 200:
 281                                         if status == "ok":
 282                                             alarm.ok_ack = True
 283                                             alarm.save()
 284                                         if status == "alarm":
 285                                             alarm.alarm_ack = True
 286                                             alarm.save()
 287                                         if status == "insufficient-data":
 288                                             alarm.alarm_ack = False
 289                                             alarm.ok_ack = False
 290                                             alarm.save()
 291                                 except RequestException as e:
 292                                     log.info("Error: RequestException while connecting to url %s", action.url)
 293                                     log.debug("RequestException %s", e)
 294
 295                                 except ConnectionError:
 296                                     log.exception("Error connecting to url %s", action.url)
 297                         else:
 298                             log.info(
 299                                 "Executing request to url %s for vnf alarm %s with status %s",
 300                                 action.url,
 301                                 alarm.alarm_id,
 302                                 status
 303                             )
 304                             try:
 305                                 requests.post(url=action.url, json=json.dumps(payload), timeout=alert_timeout)
 306                             except RequestException as e:
 307                                 log.info("Error: RequestException while connecting to url %s", action.url)
 308                                 log.debug("RequestException %s", e)
 309                             except ConnectionError:
 310                                 log.exception("Error connecting to url %s", action.url)
 311
 312         except VnfAlarm.DoesNotExist:
 313             log.debug(
 314                 "There is no alarming action configured for alarm %s.", alarm_uuid
 315             )
 316         finally:
 317             database.db.close()
 318
 319     def _get_metric_name(self, vnf_monitoring_param: dict):
 320         if "performance-metric" in vnf_monitoring_param:
 321             return vnf_monitoring_param["performance-metric"]
 322         raise ValueError(
 323             "No metric name found for vnf_monitoring_param %s"
 324             % vnf_monitoring_param["id"]
 325         )