osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import json
  26 import logging
  27 import operator
  28 import functools
  29
  30 import requests
  31 from requests.exceptions import ConnectionError, RequestException
  32
  33 from osm_policy_module.common.common_db_client import CommonDbClient
  34 from osm_policy_module.common.lcm_client import LcmClient
  35 from osm_policy_module.common.mon_client import MonClient
  36 from osm_policy_module.core import database
  37 from osm_policy_module.core.config import Config
  38 from osm_policy_module.core.database import (
  39     VnfAlarm,
  40     VnfAlarmRepository,
  41     AlarmActionRepository,
  42 )
  43 from osm_policy_module.core.exceptions import VdurNotFound
  44
  45 log = logging.getLogger(__name__)
  46
  47
  48 class AlarmingService:
  49     def __init__(self, config: Config):
  50         self.conf = config
  51         self.db_client = CommonDbClient(config)
  52         self.mon_client = MonClient(config)
  53         self.lcm_client = LcmClient(config)
  54
  55     async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
  56         log.info("Configuring vnf alarms for network service %s", nsr_id)
  57         alarms_created = []
  58         database.db.connect()
  59         try:
  60             with database.db.atomic():
  61                 if vnf_member_index is None:
  62                     vnfrs = self.db_client.get_vnfrs(nsr_id)
  63                 else:
  64                     vnfrs = []
  65                     vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
  66                     vnfrs.append(vnfr)
  67                 # vnfrs = self.db_client.get_vnfrs(nsr_id)
  68                 for vnfr in vnfrs:
  69                     log.debug("Processing vnfr: %s", vnfr)
  70                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  71                     for vdur in vnfr["vdur"]:
  72                         vdu = next(
  73                             filter(
  74                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  75                             )
  76                         )
  77                         if "alarm" in vdu:
  78                             alarm_descriptors = vdu["alarm"]
  79                             for alarm_descriptor in alarm_descriptors:
  80                                 try:
  81                                     VnfAlarmRepository.get(
  82                                         VnfAlarm.alarm_id
  83                                         == alarm_descriptor["alarm-id"],
  84                                         VnfAlarm.vnf_member_index
  85                                         == vnfr["member-vnf-index-ref"],
  86                                         VnfAlarm.vdu_name == vdur["name"],
  87                                         VnfAlarm.nsr_id == nsr_id,
  88                                     )
  89                                     log.debug(
  90                                         "vdu %s already has an alarm configured with same id %s",
  91                                         vdur["name"],
  92                                         alarm_descriptor["alarm-id"],
  93                                     )
  94                                     continue
  95                                 except VnfAlarm.DoesNotExist:
  96                                     pass
  97                                 vnf_monitoring_param = next(
  98                                     filter(
  99                                         lambda param: param["id"]
 100                                         == alarm_descriptor["vnf-monitoring-param-ref"],
 101                                         vdu.get("monitoring-parameter", []),
 102                                     ),
 103                                     {},
 104                                 )
 105                                 metric_name = self._get_metric_name(
 106                                     vnf_monitoring_param
 107                                 )
 108                                 alarm_action = dict()
 109                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 110                                     if (
 111                                         "actions" in alarm_descriptor
 112                                         and action_type in alarm_descriptor["actions"]
 113                                     ):
 114                                         for url in alarm_descriptor["actions"][
 115                                             action_type
 116                                         ]:
 117                                             if "webhook" in alarm_action:
 118                                                 alarm_action["webhook"].append(
 119                                                     url["url"]
 120                                                 )
 121                                             else:
 122                                                 alarm_action["webhook"] = [url["url"]]
 123                                 alarm_uuid = await self.mon_client.create_alarm(
 124                                     metric_name=metric_name,
 125                                     ns_id=nsr_id,
 126                                     vdu_name=vdur["name"],
 127                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 128                                     threshold=alarm_descriptor["value"],
 129                                     operation=alarm_descriptor["operation"],
 130                                     action=str(alarm_action),
 131                                     vnfr=vnfr,
 132                                     vnfd=vnfd,
 133                                 )
 134                                 alarm = VnfAlarmRepository.create(
 135                                     alarm_id=alarm_descriptor["alarm-id"],
 136                                     alarm_uuid=alarm_uuid,
 137                                     nsr_id=nsr_id,
 138                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 139                                     vdu_name=vdur["name"],
 140                                     last_action="insufficient-data",
 141                                     id_suffix=0,
 142                                     ok_ack=False,
 143                                     alarm_ack=False,
 144                                 )
 145                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 146                                     if (
 147                                         "actions" in alarm_descriptor
 148                                         and action_type in alarm_descriptor["actions"]
 149                                     ):
 150                                         for url in alarm_descriptor["actions"][
 151                                             action_type
 152                                         ]:
 153                                             AlarmActionRepository.create(
 154                                                 type=action_type,
 155                                                 url=url["url"],
 156                                                 alarm=alarm,
 157                                             )
 158                                 alarms_created.append(alarm)
 159
 160         except Exception as e:
 161             log.exception("Error configuring VNF alarms:")
 162             if len(alarms_created) > 0:
 163                 log.debug("Cleaning alarm resources in MON")
 164                 for alarm in alarms_created:
 165                     try:
 166                         await self.mon_client.delete_alarm(
 167                             alarm.nsr_id,
 168                             alarm.vnf_member_index,
 169                             alarm.vdu_name,
 170                             alarm.alarm_uuid,
 171                         )
 172                     except ValueError:
 173                         log.exception(
 174                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 175                         )
 176             raise e
 177         finally:
 178             database.db.close()
 179
 180     async def delete_orphaned_alarms(self, nsr_id):
 181         # TODO: Review as it seems this code is never called
 182         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 183         database.db.connect()
 184         try:
 185             with database.db.atomic():
 186                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 187                     try:
 188                         self.db_client.get_vdur(
 189                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 190                         )
 191                     except VdurNotFound:
 192                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 193                         try:
 194                             await self.mon_client.delete_alarm(
 195                                 alarm.nsr_id,
 196                                 alarm.vnf_member_index,
 197                                 alarm.vdu_name,
 198                                 alarm.alarm_uuid,
 199                             )
 200                         except ValueError:
 201                             log.exception(
 202                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 203                             )
 204                         alarm.delete_instance()
 205         except Exception as e:
 206             log.exception("Error deleting orphaned alarms:")
 207             raise e
 208         finally:
 209             database.db.close()
 210
 211     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 212         log.info("Deleting vnf alarms for network service %s", nsr_id)
 213         database.db.connect()
 214         try:
 215             with database.db.atomic():
 216                 if vnf_member_index is None:
 217                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 218                 else:
 219                     query_list = [
 220                         VnfAlarm.nsr_id == nsr_id,
 221                         VnfAlarm.vnf_member_index == vnf_member_index,
 222                     ]
 223                     alarm_conditions = functools.reduce(operator.and_, query_list)
 224                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 225                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 226                     try:
 227                         await self.mon_client.delete_alarm(
 228                             alarm.nsr_id,
 229                             alarm.vnf_member_index,
 230                             alarm.vdu_name,
 231                             alarm.alarm_uuid,
 232                         )
 233                     except ValueError:
 234                         log.exception(
 235                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 236                         )
 237                     alarm.delete_instance()
 238
 239         except Exception as e:
 240             log.exception("Error deleting vnf alarms:")
 241             raise e
 242         finally:
 243             database.db.close()
 244
 245     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 246         alert_timeout = int(self.conf.get("alert", "timeout"))
 247         database.db.connect()
 248         try:
 249             with database.db.atomic():
 250                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 251                 log.debug(
 252                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 253                 )
 254                 for action in alarm.actions:
 255                     """
 256                     Compares the current status with the last_action status.
 257                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 258                     If both the status are 'ok', it avoid sending repetitive ok notification.
 259                     """
 260                     if action.type == status:
 261                         if bool(self.conf.get("alert", "enhanced_alarms")):
 262                             if (
 263                                 status != "ok"
 264                                 or (status == "ok" and alarm.ok_ack is False)
 265                             ) and (
 266                                 status != "alarm"
 267                                 or (status == "alarm" and alarm.alarm_ack is False)
 268                             ):
 269                                 log.info(
 270                                     "Executing request to url %s for vnf alarm %s with status %s",
 271                                     action.url,
 272                                     alarm.alarm_id,
 273                                     status,
 274                                 )
 275                                 try:
 276                                     if (
 277                                         status == "alarm"
 278                                         and alarm.last_action == "ok"
 279                                         or (
 280                                             status == "alarm"
 281                                             and alarm.last_action == "insufficient-data"
 282                                         )
 283                                     ):
 284                                         alarm.id_suffix += 1
 285                                         alarm.ok_ack = False
 286                                     if status == "ok" and alarm.last_action == "alarm":
 287                                         alarm.alarm_ack = False
 288                                     alarm.last_action = status
 289                                     alarm.save()
 290                                 except Exception as e:
 291                                     log.exception(e)
 292
 293                                 payload["notify_details"][
 294                                     "alarm_number"
 295                                 ] = alarm.id_suffix
 296                                 headers = {"content-type": "application/json"}
 297                                 try:
 298                                     resp = requests.post(
 299                                         url=action.url,
 300                                         data=json.dumps(payload),
 301                                         headers=headers,
 302                                         timeout=alert_timeout,
 303                                     )
 304                                     log.info("Response %s", resp)
 305                                     if resp.status_code == 200:
 306                                         if status == "ok":
 307                                             alarm.ok_ack = True
 308                                             alarm.save()
 309                                         if status == "alarm":
 310                                             alarm.alarm_ack = True
 311                                             alarm.save()
 312                                         if status == "insufficient-data":
 313                                             alarm.alarm_ack = False
 314                                             alarm.ok_ack = False
 315                                             alarm.save()
 316                                 except ConnectionError:
 317                                     log.exception(
 318                                         "Error connecting to url %s", action.url
 319                                     )
 320                                 except RequestException as e:
 321                                     log.info(
 322                                         "Error: RequestException while connecting to url %s",
 323                                         action.url,
 324                                     )
 325                                     log.debug("RequestException %s", e)
 326
 327                         else:
 328                             log.info(
 329                                 "Executing request to url %s for vnf alarm %s with status %s",
 330                                 action.url,
 331                                 alarm.alarm_id,
 332                                 status,
 333                             )
 334                             try:
 335                                 requests.post(
 336                                     url=action.url,
 337                                     json=json.dumps(payload),
 338                                     timeout=alert_timeout,
 339                                 )
 340                             except ConnectionError:
 341                                 log.exception("Error connecting to url %s", action.url)
 342                             except RequestException as e:
 343                                 log.info(
 344                                     "Error: RequestException while connecting to url %s",
 345                                     action.url,
 346                                 )
 347                                 log.debug("RequestException %s", e)
 348
 349         except VnfAlarm.DoesNotExist:
 350             log.debug(
 351                 "There is no alarming action configured for alarm %s.", alarm_uuid
 352             )
 353         finally:
 354             database.db.close()
 355
 356     def _get_metric_name(self, vnf_monitoring_param: dict):
 357         if "performance-metric" in vnf_monitoring_param:
 358             return vnf_monitoring_param["performance-metric"]
 359         raise ValueError(
 360             "No metric name found for vnf_monitoring_param %s"
 361             % vnf_monitoring_param["id"]
 362         )