osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import asyncio
  26 import json
  27 import logging
  28 import operator
  29 import functools
  30
  31 import requests
  32 from requests.exceptions import ConnectionError, RequestException
  33
  34 from osm_policy_module.common.common_db_client import CommonDbClient
  35 from osm_policy_module.common.lcm_client import LcmClient
  36 from osm_policy_module.common.mon_client import MonClient
  37 from osm_policy_module.core import database
  38 from osm_policy_module.core.config import Config
  39 from osm_policy_module.core.database import (
  40     VnfAlarm,
  41     VnfAlarmRepository,
  42     AlarmActionRepository,
  43 )
  44 from osm_policy_module.core.exceptions import VdurNotFound
  45
  46 log = logging.getLogger(__name__)
  47
  48
  49 class AlarmingService:
  50     def __init__(self, config: Config, loop=None):
  51         self.conf = config
  52         if not loop:
  53             loop = asyncio.get_event_loop()
  54         self.loop = loop
  55         self.db_client = CommonDbClient(config)
  56         self.mon_client = MonClient(config, loop=self.loop)
  57         self.lcm_client = LcmClient(config, loop=self.loop)
  58
  59     async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
  60         log.info("Configuring vnf alarms for network service %s", nsr_id)
  61         alarms_created = []
  62         database.db.connect()
  63         try:
  64             with database.db.atomic():
  65                 if vnf_member_index is None:
  66                     vnfrs = self.db_client.get_vnfrs(nsr_id)
  67                 else:
  68                     vnfrs = []
  69                     vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
  70                     vnfrs.append(vnfr)
  71                 # vnfrs = self.db_client.get_vnfrs(nsr_id)
  72                 for vnfr in vnfrs:
  73                     log.debug("Processing vnfr: %s", vnfr)
  74                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  75                     for vdur in vnfr["vdur"]:
  76                         vdu = next(
  77                             filter(
  78                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  79                             )
  80                         )
  81                         if "alarm" in vdu:
  82                             alarm_descriptors = vdu["alarm"]
  83                             for alarm_descriptor in alarm_descriptors:
  84                                 try:
  85                                     VnfAlarmRepository.get(
  86                                         VnfAlarm.alarm_id
  87                                         == alarm_descriptor["alarm-id"],
  88                                         VnfAlarm.vnf_member_index
  89                                         == vnfr["member-vnf-index-ref"],
  90                                         VnfAlarm.vdu_name == vdur["name"],
  91                                         VnfAlarm.nsr_id == nsr_id,
  92                                     )
  93                                     log.debug(
  94                                         "vdu %s already has an alarm configured with same id %s",
  95                                         vdur["name"],
  96                                         alarm_descriptor["alarm-id"],
  97                                     )
  98                                     continue
  99                                 except VnfAlarm.DoesNotExist:
 100                                     pass
 101                                 vnf_monitoring_param = next(
 102                                     filter(
 103                                         lambda param: param["id"]
 104                                         == alarm_descriptor["vnf-monitoring-param-ref"],
 105                                         vdu.get("monitoring-parameter", []),
 106                                     ),
 107                                     {},
 108                                 )
 109                                 metric_name = self._get_metric_name(
 110                                     vnf_monitoring_param
 111                                 )
 112                                 alarm_action = dict()
 113                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 114                                     if (
 115                                         "actions" in alarm_descriptor
 116                                         and action_type in alarm_descriptor["actions"]
 117                                     ):
 118                                         for url in alarm_descriptor["actions"][
 119                                             action_type
 120                                         ]:
 121                                             if "webhook" in alarm_action:
 122                                                 alarm_action["webhook"].append(
 123                                                     url["url"]
 124                                                 )
 125                                             else:
 126                                                 alarm_action["webhook"] = [url["url"]]
 127                                 alarm_uuid = await self.mon_client.create_alarm(
 128                                     metric_name=metric_name,
 129                                     ns_id=nsr_id,
 130                                     vdu_name=vdur["name"],
 131                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 132                                     threshold=alarm_descriptor["value"],
 133                                     operation=alarm_descriptor["operation"],
 134                                     action=str(alarm_action),
 135                                 )
 136                                 alarm = VnfAlarmRepository.create(
 137                                     alarm_id=alarm_descriptor["alarm-id"],
 138                                     alarm_uuid=alarm_uuid,
 139                                     nsr_id=nsr_id,
 140                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 141                                     vdu_name=vdur["name"],
 142                                     last_action="insufficient-data",
 143                                     id_suffix=0,
 144                                     ok_ack=False,
 145                                     alarm_ack=False,
 146                                 )
 147                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 148                                     if (
 149                                         "actions" in alarm_descriptor
 150                                         and action_type in alarm_descriptor["actions"]
 151                                     ):
 152                                         for url in alarm_descriptor["actions"][
 153                                             action_type
 154                                         ]:
 155                                             AlarmActionRepository.create(
 156                                                 type=action_type,
 157                                                 url=url["url"],
 158                                                 alarm=alarm,
 159                                             )
 160                                 alarms_created.append(alarm)
 161
 162         except Exception as e:
 163             log.exception("Error configuring VNF alarms:")
 164             if len(alarms_created) > 0:
 165                 log.debug("Cleaning alarm resources in MON")
 166                 for alarm in alarms_created:
 167                     try:
 168                         await self.mon_client.delete_alarm(
 169                             alarm.nsr_id,
 170                             alarm.vnf_member_index,
 171                             alarm.vdu_name,
 172                             alarm.alarm_uuid,
 173                         )
 174                     except ValueError:
 175                         log.exception(
 176                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 177                         )
 178             raise e
 179         finally:
 180             database.db.close()
 181
 182     async def delete_orphaned_alarms(self, nsr_id):
 183         # TODO: Review as it seems this code is never called
 184         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 185         database.db.connect()
 186         try:
 187             with database.db.atomic():
 188                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 189                     try:
 190                         self.db_client.get_vdur(
 191                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 192                         )
 193                     except VdurNotFound:
 194                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 195                         try:
 196                             await self.mon_client.delete_alarm(
 197                                 alarm.nsr_id,
 198                                 alarm.vnf_member_index,
 199                                 alarm.vdu_name,
 200                                 alarm.alarm_uuid,
 201                             )
 202                         except ValueError:
 203                             log.exception(
 204                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 205                             )
 206                         alarm.delete_instance()
 207         except Exception as e:
 208             log.exception("Error deleting orphaned alarms:")
 209             raise e
 210         finally:
 211             database.db.close()
 212
 213     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 214         log.info("Deleting vnf alarms for network service %s", nsr_id)
 215         database.db.connect()
 216         try:
 217             with database.db.atomic():
 218                 if vnf_member_index is None:
 219                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 220                 else:
 221                     query_list = [
 222                         VnfAlarm.nsr_id == nsr_id,
 223                         VnfAlarm.vnf_member_index == vnf_member_index,
 224                     ]
 225                     alarm_conditions = functools.reduce(operator.and_, query_list)
 226                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 227                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 228                     try:
 229                         await self.mon_client.delete_alarm(
 230                             alarm.nsr_id,
 231                             alarm.vnf_member_index,
 232                             alarm.vdu_name,
 233                             alarm.alarm_uuid,
 234                         )
 235                     except ValueError:
 236                         log.exception(
 237                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 238                         )
 239                     alarm.delete_instance()
 240
 241         except Exception as e:
 242             log.exception("Error deleting vnf alarms:")
 243             raise e
 244         finally:
 245             database.db.close()
 246
 247     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 248         alert_timeout = int(self.conf.get("alert", "timeout"))
 249         database.db.connect()
 250         try:
 251             with database.db.atomic():
 252                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 253                 log.debug(
 254                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 255                 )
 256                 for action in alarm.actions:
 257                     """
 258                     Compares the current status with the last_action status.
 259                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 260                     If both the status are 'ok', it avoid sending repetitive ok notification.
 261                     """
 262                     if action.type == status:
 263                         if bool(self.conf.get("alert", "enhanced_alarms")):
 264                             if (
 265                                 status != "ok"
 266                                 or (status == "ok" and alarm.ok_ack is False)
 267                             ) and (
 268                                 status != "alarm"
 269                                 or (status == "alarm" and alarm.alarm_ack is False)
 270                             ):
 271                                 log.info(
 272                                     "Executing request to url %s for vnf alarm %s with status %s",
 273                                     action.url,
 274                                     alarm.alarm_id,
 275                                     status,
 276                                 )
 277                                 try:
 278                                     if status == "alarm" and alarm.last_action == "ok":
 279                                         alarm.id_suffix += 1
 280                                         alarm.ok_ack = False
 281                                     if status == "ok" and alarm.last_action == "alarm":
 282                                         alarm.alarm_ack = False
 283                                     alarm.last_action = status
 284                                     alarm.save()
 285                                 except Exception as e:
 286                                     log.exception(e)
 287
 288                                 payload["notify_details"][
 289                                     "alarm_number"
 290                                 ] = alarm.id_suffix
 291                                 headers = {"content-type": "application/json"}
 292                                 try:
 293                                     resp = requests.post(
 294                                         url=action.url,
 295                                         data=json.dumps(payload),
 296                                         headers=headers,
 297                                         verify=False,
 298                                         timeout=alert_timeout,
 299                                     )
 300                                     log.info("Response %s", resp)
 301                                     if resp.status_code == 200:
 302                                         if status == "ok":
 303                                             alarm.ok_ack = True
 304                                             alarm.save()
 305                                         if status == "alarm":
 306                                             alarm.alarm_ack = True
 307                                             alarm.save()
 308                                         if status == "insufficient-data":
 309                                             alarm.alarm_ack = False
 310                                             alarm.ok_ack = False
 311                                             alarm.save()
 312                                 except ConnectionError:
 313                                     log.exception(
 314                                         "Error connecting to url %s", action.url
 315                                     )
 316                                 except RequestException as e:
 317                                     log.info(
 318                                         "Error: RequestException while connecting to url %s",
 319                                         action.url,
 320                                     )
 321                                     log.debug("RequestException %s", e)
 322
 323                         else:
 324                             log.info(
 325                                 "Executing request to url %s for vnf alarm %s with status %s",
 326                                 action.url,
 327                                 alarm.alarm_id,
 328                                 status,
 329                             )
 330                             try:
 331                                 requests.post(
 332                                     url=action.url,
 333                                     json=json.dumps(payload),
 334                                     timeout=alert_timeout,
 335                                 )
 336                             except ConnectionError:
 337                                 log.exception("Error connecting to url %s", action.url)
 338                             except RequestException as e:
 339                                 log.info(
 340                                     "Error: RequestException while connecting to url %s",
 341                                     action.url,
 342                                 )
 343                                 log.debug("RequestException %s", e)
 344
 345         except VnfAlarm.DoesNotExist:
 346             log.debug(
 347                 "There is no alarming action configured for alarm %s.", alarm_uuid
 348             )
 349         finally:
 350             database.db.close()
 351
 352     def _get_metric_name(self, vnf_monitoring_param: dict):
 353         if "performance-metric" in vnf_monitoring_param:
 354             return vnf_monitoring_param["performance-metric"]
 355         raise ValueError(
 356             "No metric name found for vnf_monitoring_param %s"
 357             % vnf_monitoring_param["id"]
 358         )