osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import json
  26 import logging
  27 import operator
  28 import functools
  29
  30 import requests
  31 from requests.exceptions import ConnectionError, RequestException
  32
  33 from osm_policy_module.common.common_db_client import CommonDbClient
  34 from osm_policy_module.common.lcm_client import LcmClient
  35 from osm_policy_module.common.mon_client import MonClient
  36 from osm_policy_module.core import database
  37 from osm_policy_module.core.config import Config
  38 from osm_policy_module.core.database import (
  39     VnfAlarm,
  40     VnfAlarmRepository,
  41     AlarmActionRepository,
  42 )
  43 from osm_policy_module.core.exceptions import VdurNotFound
  44
  45 log = logging.getLogger(__name__)
  46
  47
  48 class AlarmingService:
  49     def __init__(self, config: Config):
  50         self.conf = config
  51         self.db_client = CommonDbClient(config)
  52         self.mon_client = MonClient(config)
  53         self.lcm_client = LcmClient(config)
  54
  55     async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
  56         log.info("Configuring vnf alarms for network service %s", nsr_id)
  57         alarms_created = []
  58         database.db.connect()
  59         try:
  60             with database.db.atomic():
  61                 if vnf_member_index is None:
  62                     vnfrs = self.db_client.get_vnfrs(nsr_id)
  63                 else:
  64                     vnfrs = []
  65                     vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
  66                     vnfrs.append(vnfr)
  67                 # vnfrs = self.db_client.get_vnfrs(nsr_id)
  68                 for vnfr in vnfrs:
  69                     log.debug("Processing vnfr: %s", vnfr)
  70                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  71                     for vdur in vnfr["vdur"]:
  72                         vdu = next(
  73                             filter(
  74                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  75                             )
  76                         )
  77                         if "alarm" in vdu:
  78                             alarm_descriptors = vdu["alarm"]
  79                             for alarm_descriptor in alarm_descriptors:
  80                                 try:
  81                                     VnfAlarmRepository.get(
  82                                         VnfAlarm.alarm_id
  83                                         == alarm_descriptor["alarm-id"],
  84                                         VnfAlarm.vnf_member_index
  85                                         == vnfr["member-vnf-index-ref"],
  86                                         VnfAlarm.vdu_name == vdur["name"],
  87                                         VnfAlarm.nsr_id == nsr_id,
  88                                     )
  89                                     log.debug(
  90                                         "vdu %s already has an alarm configured with same id %s",
  91                                         vdur["name"],
  92                                         alarm_descriptor["alarm-id"],
  93                                     )
  94                                     continue
  95                                 except VnfAlarm.DoesNotExist:
  96                                     pass
  97                                 vnf_monitoring_param = next(
  98                                     filter(
  99                                         lambda param: param["id"]
 100                                         == alarm_descriptor["vnf-monitoring-param-ref"],
 101                                         vdu.get("monitoring-parameter", []),
 102                                     ),
 103                                     {},
 104                                 )
 105                                 metric_name = self._get_metric_name(
 106                                     vnf_monitoring_param
 107                                 )
 108                                 alarm_action = dict()
 109                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 110                                     if (
 111                                         "actions" in alarm_descriptor
 112                                         and action_type in alarm_descriptor["actions"]
 113                                     ):
 114                                         for url in alarm_descriptor["actions"][
 115                                             action_type
 116                                         ]:
 117                                             if "webhook" in alarm_action:
 118                                                 alarm_action["webhook"].append(
 119                                                     url["url"]
 120                                                 )
 121                                             else:
 122                                                 alarm_action["webhook"] = [url["url"]]
 123                                 alarm_uuid = await self.mon_client.create_alarm(
 124                                     metric_name=metric_name,
 125                                     ns_id=nsr_id,
 126                                     vdu_name=vdur["name"],
 127                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 128                                     threshold=alarm_descriptor["value"],
 129                                     operation=alarm_descriptor["operation"],
 130                                     action=str(alarm_action),
 131                                     vnfr=vnfr,
 132                                     vnfd=vnfd,
 133                                 )
 134                                 alarm = VnfAlarmRepository.create(
 135                                     alarm_id=alarm_descriptor["alarm-id"],
 136                                     alarm_uuid=alarm_uuid,
 137                                     nsr_id=nsr_id,
 138                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 139                                     vdu_name=vdur["name"],
 140                                     last_action="insufficient-data",
 141                                     id_suffix=0,
 142                                     ok_ack=False,
 143                                     alarm_ack=False,
 144                                 )
 145                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 146                                     if (
 147                                         "actions" in alarm_descriptor
 148                                         and action_type in alarm_descriptor["actions"]
 149                                     ):
 150                                         for url in alarm_descriptor["actions"][
 151                                             action_type
 152                                         ]:
 153                                             AlarmActionRepository.create(
 154                                                 type=action_type,
 155                                                 url=url["url"],
 156                                                 alarm=alarm,
 157                                             )
 158                                 alarms_created.append(alarm)
 159
 160         except Exception as e:
 161             log.exception("Error configuring VNF alarms:")
 162             if len(alarms_created) > 0:
 163                 log.debug("Cleaning alarm resources in MON")
 164                 for alarm in alarms_created:
 165                     try:
 166                         await self.mon_client.delete_alarm(
 167                             alarm.nsr_id,
 168                             alarm.vnf_member_index,
 169                             alarm.vdu_name,
 170                             alarm.alarm_uuid,
 171                         )
 172                     except ValueError:
 173                         log.exception(
 174                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 175                         )
 176             raise e
 177         finally:
 178             database.db.close()
 179
 180     async def delete_orphaned_alarms(self, nsr_id):
 181         # TODO: Review as it seems this code is never called
 182         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 183         database.db.connect()
 184         try:
 185             with database.db.atomic():
 186                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 187                     try:
 188                         self.db_client.get_vdur(
 189                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 190                         )
 191                     except VdurNotFound:
 192                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 193                         try:
 194                             await self.mon_client.delete_alarm(
 195                                 alarm.nsr_id,
 196                                 alarm.vnf_member_index,
 197                                 alarm.vdu_name,
 198                                 alarm.alarm_uuid,
 199                             )
 200                         except ValueError:
 201                             log.exception(
 202                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 203                             )
 204                         alarm.delete_instance()
 205         except Exception as e:
 206             log.exception("Error deleting orphaned alarms:")
 207             raise e
 208         finally:
 209             database.db.close()
 210
 211     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 212         log.info("Deleting vnf alarms for network service %s", nsr_id)
 213         database.db.connect()
 214         try:
 215             with database.db.atomic():
 216                 if vnf_member_index is None:
 217                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 218                 else:
 219                     query_list = [
 220                         VnfAlarm.nsr_id == nsr_id,
 221                         VnfAlarm.vnf_member_index == vnf_member_index,
 222                     ]
 223                     alarm_conditions = functools.reduce(operator.and_, query_list)
 224                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 225                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 226                     try:
 227                         await self.mon_client.delete_alarm(
 228                             alarm.nsr_id,
 229                             alarm.vnf_member_index,
 230                             alarm.vdu_name,
 231                             alarm.alarm_uuid,
 232                         )
 233                     except ValueError:
 234                         log.exception(
 235                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 236                         )
 237                     alarm.delete_instance()
 238
 239         except Exception as e:
 240             log.exception("Error deleting vnf alarms:")
 241             raise e
 242         finally:
 243             database.db.close()
 244
 245     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 246         alert_timeout = int(self.conf.get("alert", "timeout"))
 247         database.db.connect()
 248         try:
 249             with database.db.atomic():
 250                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 251                 log.debug(
 252                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 253                 )
 254                 for action in alarm.actions:
 255                     """
 256                     Compares the current status with the last_action status.
 257                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 258                     If both the status are 'ok', it avoid sending repetitive ok notification.
 259                     """
 260                     if action.type == status:
 261                         if bool(self.conf.get("alert", "enhanced_alarms")):
 262                             if (
 263                                 status != "ok"
 264                                 or (status == "ok" and alarm.ok_ack is False)
 265                             ) and (
 266                                 status != "alarm"
 267                                 or (status == "alarm" and alarm.alarm_ack is False)
 268                             ):
 269                                 log.info(
 270                                     "Executing request to url %s for vnf alarm %s with status %s",
 271                                     action.url,
 272                                     alarm.alarm_id,
 273                                     status,
 274                                 )
 275                                 try:
 276                                     if status == "alarm" and alarm.last_action == "ok":
 277                                         alarm.id_suffix += 1
 278                                         alarm.ok_ack = False
 279                                     if status == "ok" and alarm.last_action == "alarm":
 280                                         alarm.alarm_ack = False
 281                                     alarm.last_action = status
 282                                     alarm.save()
 283                                 except Exception as e:
 284                                     log.exception(e)
 285
 286                                 payload["notify_details"][
 287                                     "alarm_number"
 288                                 ] = alarm.id_suffix
 289                                 headers = {"content-type": "application/json"}
 290                                 try:
 291                                     resp = requests.post(
 292                                         url=action.url,
 293                                         data=json.dumps(payload),
 294                                         headers=headers,
 295                                         timeout=alert_timeout,
 296                                     )
 297                                     log.info("Response %s", resp)
 298                                     if resp.status_code == 200:
 299                                         if status == "ok":
 300                                             alarm.ok_ack = True
 301                                             alarm.save()
 302                                         if status == "alarm":
 303                                             alarm.alarm_ack = True
 304                                             alarm.save()
 305                                         if status == "insufficient-data":
 306                                             alarm.alarm_ack = False
 307                                             alarm.ok_ack = False
 308                                             alarm.save()
 309                                 except ConnectionError:
 310                                     log.exception(
 311                                         "Error connecting to url %s", action.url
 312                                     )
 313                                 except RequestException as e:
 314                                     log.info(
 315                                         "Error: RequestException while connecting to url %s",
 316                                         action.url,
 317                                     )
 318                                     log.debug("RequestException %s", e)
 319
 320                         else:
 321                             log.info(
 322                                 "Executing request to url %s for vnf alarm %s with status %s",
 323                                 action.url,
 324                                 alarm.alarm_id,
 325                                 status,
 326                             )
 327                             try:
 328                                 requests.post(
 329                                     url=action.url,
 330                                     json=json.dumps(payload),
 331                                     timeout=alert_timeout,
 332                                 )
 333                             except ConnectionError:
 334                                 log.exception("Error connecting to url %s", action.url)
 335                             except RequestException as e:
 336                                 log.info(
 337                                     "Error: RequestException while connecting to url %s",
 338                                     action.url,
 339                                 )
 340                                 log.debug("RequestException %s", e)
 341
 342         except VnfAlarm.DoesNotExist:
 343             log.debug(
 344                 "There is no alarming action configured for alarm %s.", alarm_uuid
 345             )
 346         finally:
 347             database.db.close()
 348
 349     def _get_metric_name(self, vnf_monitoring_param: dict):
 350         if "performance-metric" in vnf_monitoring_param:
 351             return vnf_monitoring_param["performance-metric"]
 352         raise ValueError(
 353             "No metric name found for vnf_monitoring_param %s"
 354             % vnf_monitoring_param["id"]
 355         )