osm_policy_module/alarming/service.py

   1 # -*- coding: utf-8 -*-
   2 # pylint: disable=no-member
   3
   4 # Copyright 2018 Whitestack, LLC
   5 # *************************************************************
   6
   7 # This file is part of OSM Monitoring module
   8 # All Rights Reserved to Whitestack, LLC
   9
  10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
  11 # not use this file except in compliance with the License. You may obtain
  12 # a copy of the License at
  13
  14 #         http://www.apache.org/licenses/LICENSE-2.0
  15
  16 # Unless required by applicable law or agreed to in writing, software
  17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  19 # License for the specific language governing permissions and limitations
  20 # under the License.
  21
  22 # For those usages not covered by the Apache License, Version 2.0 please
  23 # contact: bdiaz@whitestack.com or glavado@whitestack.com
  24 ##
  25 import json
  26 import logging
  27 import operator
  28 import functools
  29
  30 import requests
  31 from requests.exceptions import ConnectionError, RequestException
  32
  33 from osm_policy_module.common.common_db_client import CommonDbClient
  34 from osm_policy_module.common.lcm_client import LcmClient
  35 from osm_policy_module.common.mon_client import MonClient
  36 from osm_policy_module.core import database
  37 from osm_policy_module.core.config import Config
  38 from osm_policy_module.core.database import (
  39     VnfAlarm,
  40     VnfAlarmRepository,
  41     AlarmActionRepository,
  42 )
  43 from osm_policy_module.core.exceptions import VdurNotFound
  44
  45 log = logging.getLogger(__name__)
  46
  47
  48 class AlarmingService:
  49     def __init__(self, config: Config):
  50         self.conf = config
  51         self.db_client = CommonDbClient(config)
  52         self.mon_client = MonClient(config)
  53         self.lcm_client = LcmClient(config)
  54
  55     async def configure_vnf_alarms(self, nsr_id: str, vnf_member_index=None):
  56         log.info("Configuring vnf alarms for network service %s", nsr_id)
  57         alarms_created = []
  58         database.db.connect()
  59         try:
  60             with database.db.atomic():
  61                 if vnf_member_index is None:
  62                     vnfrs = self.db_client.get_vnfrs(nsr_id)
  63                 else:
  64                     vnfrs = []
  65                     vnfr = self.db_client.get_vnfr(nsr_id, vnf_member_index)
  66                     vnfrs.append(vnfr)
  67                 # vnfrs = self.db_client.get_vnfrs(nsr_id)
  68                 for vnfr in vnfrs:
  69                     log.debug("Processing vnfr: %s", vnfr)
  70                     vnfd = self.db_client.get_vnfd(vnfr["vnfd-id"])
  71                     for vdur in vnfr["vdur"]:
  72                         vdu = next(
  73                             filter(
  74                                 lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]
  75                             )
  76                         )
  77                         if "alarm" in vdu:
  78                             alarm_descriptors = vdu["alarm"]
  79                             for alarm_descriptor in alarm_descriptors:
  80                                 try:
  81                                     VnfAlarmRepository.get(
  82                                         VnfAlarm.alarm_id
  83                                         == alarm_descriptor["alarm-id"],
  84                                         VnfAlarm.vnf_member_index
  85                                         == vnfr["member-vnf-index-ref"],
  86                                         VnfAlarm.vdu_name == vdur["name"],
  87                                         VnfAlarm.nsr_id == nsr_id,
  88                                     )
  89                                     log.debug(
  90                                         "vdu %s already has an alarm configured with same id %s",
  91                                         vdur["name"],
  92                                         alarm_descriptor["alarm-id"],
  93                                     )
  94                                     continue
  95                                 except VnfAlarm.DoesNotExist:
  96                                     pass
  97                                 vnf_monitoring_param = next(
  98                                     filter(
  99                                         lambda param: param["id"]
 100                                         == alarm_descriptor["vnf-monitoring-param-ref"],
 101                                         vdu.get("monitoring-parameter", []),
 102                                     ),
 103                                     {},
 104                                 )
 105                                 metric_name = self._get_metric_name(
 106                                     vnf_monitoring_param
 107                                 )
 108                                 alarm_action = dict()
 109                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 110                                     if (
 111                                         "actions" in alarm_descriptor
 112                                         and action_type in alarm_descriptor["actions"]
 113                                     ):
 114                                         for url in alarm_descriptor["actions"][
 115                                             action_type
 116                                         ]:
 117                                             if "webhook" in alarm_action:
 118                                                 alarm_action["webhook"].append(
 119                                                     url["url"]
 120                                                 )
 121                                             else:
 122                                                 alarm_action["webhook"] = [url["url"]]
 123                                 alarm_uuid = await self.mon_client.create_alarm(
 124                                     metric_name=metric_name,
 125                                     ns_id=nsr_id,
 126                                     vdu_name=vdur["name"],
 127                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 128                                     threshold=alarm_descriptor["value"],
 129                                     operation=alarm_descriptor["operation"],
 130                                     action=str(alarm_action),
 131                                 )
 132                                 alarm = VnfAlarmRepository.create(
 133                                     alarm_id=alarm_descriptor["alarm-id"],
 134                                     alarm_uuid=alarm_uuid,
 135                                     nsr_id=nsr_id,
 136                                     vnf_member_index=vnfr["member-vnf-index-ref"],
 137                                     vdu_name=vdur["name"],
 138                                     last_action="insufficient-data",
 139                                     id_suffix=0,
 140                                     ok_ack=False,
 141                                     alarm_ack=False,
 142                                 )
 143                                 for action_type in ["ok", "insufficient-data", "alarm"]:
 144                                     if (
 145                                         "actions" in alarm_descriptor
 146                                         and action_type in alarm_descriptor["actions"]
 147                                     ):
 148                                         for url in alarm_descriptor["actions"][
 149                                             action_type
 150                                         ]:
 151                                             AlarmActionRepository.create(
 152                                                 type=action_type,
 153                                                 url=url["url"],
 154                                                 alarm=alarm,
 155                                             )
 156                                 alarms_created.append(alarm)
 157
 158         except Exception as e:
 159             log.exception("Error configuring VNF alarms:")
 160             if len(alarms_created) > 0:
 161                 log.debug("Cleaning alarm resources in MON")
 162                 for alarm in alarms_created:
 163                     try:
 164                         await self.mon_client.delete_alarm(
 165                             alarm.nsr_id,
 166                             alarm.vnf_member_index,
 167                             alarm.vdu_name,
 168                             alarm.alarm_uuid,
 169                         )
 170                     except ValueError:
 171                         log.exception(
 172                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 173                         )
 174             raise e
 175         finally:
 176             database.db.close()
 177
 178     async def delete_orphaned_alarms(self, nsr_id):
 179         # TODO: Review as it seems this code is never called
 180         log.info("Deleting orphaned vnf alarms for network service %s", nsr_id)
 181         database.db.connect()
 182         try:
 183             with database.db.atomic():
 184                 for alarm in VnfAlarmRepository.list(VnfAlarm.nsr_id == nsr_id):
 185                     try:
 186                         self.db_client.get_vdur(
 187                             nsr_id, alarm.vnf_member_index, alarm.vdu_name
 188                         )
 189                     except VdurNotFound:
 190                         log.debug("Deleting orphaned alarm %s", alarm.alarm_uuid)
 191                         try:
 192                             await self.mon_client.delete_alarm(
 193                                 alarm.nsr_id,
 194                                 alarm.vnf_member_index,
 195                                 alarm.vdu_name,
 196                                 alarm.alarm_uuid,
 197                             )
 198                         except ValueError:
 199                             log.exception(
 200                                 "Error deleting alarm in MON %s", alarm.alarm_uuid
 201                             )
 202                         alarm.delete_instance()
 203         except Exception as e:
 204             log.exception("Error deleting orphaned alarms:")
 205             raise e
 206         finally:
 207             database.db.close()
 208
 209     async def delete_vnf_alarms(self, nsr_id, vnf_member_index=None):
 210         log.info("Deleting vnf alarms for network service %s", nsr_id)
 211         database.db.connect()
 212         try:
 213             with database.db.atomic():
 214                 if vnf_member_index is None:
 215                     alarm_conditions = VnfAlarm.nsr_id == nsr_id
 216                 else:
 217                     query_list = [
 218                         VnfAlarm.nsr_id == nsr_id,
 219                         VnfAlarm.vnf_member_index == vnf_member_index,
 220                     ]
 221                     alarm_conditions = functools.reduce(operator.and_, query_list)
 222                 for alarm in VnfAlarmRepository.list(alarm_conditions):
 223                     log.debug("Deleting vnf alarm %s", alarm.alarm_uuid)
 224                     try:
 225                         await self.mon_client.delete_alarm(
 226                             alarm.nsr_id,
 227                             alarm.vnf_member_index,
 228                             alarm.vdu_name,
 229                             alarm.alarm_uuid,
 230                         )
 231                     except ValueError:
 232                         log.exception(
 233                             "Error deleting alarm in MON %s", alarm.alarm_uuid
 234                         )
 235                     alarm.delete_instance()
 236
 237         except Exception as e:
 238             log.exception("Error deleting vnf alarms:")
 239             raise e
 240         finally:
 241             database.db.close()
 242
 243     async def handle_alarm(self, alarm_uuid: str, status: str, payload: dict):
 244         alert_timeout = int(self.conf.get("alert", "timeout"))
 245         database.db.connect()
 246         try:
 247             with database.db.atomic():
 248                 alarm = VnfAlarmRepository.get(VnfAlarm.alarm_uuid == alarm_uuid)
 249                 log.debug(
 250                     "Handling vnf alarm %s with status %s", alarm.alarm_id, status
 251                 )
 252                 for action in alarm.actions:
 253                     """
 254                     Compares the current status with the last_action status.
 255                     If both the status are 'alarm', it avoid sending repetitive alarm notification.
 256                     If both the status are 'ok', it avoid sending repetitive ok notification.
 257                     """
 258                     if action.type == status:
 259                         if bool(self.conf.get("alert", "enhanced_alarms")):
 260                             if (
 261                                 status != "ok"
 262                                 or (status == "ok" and alarm.ok_ack is False)
 263                             ) and (
 264                                 status != "alarm"
 265                                 or (status == "alarm" and alarm.alarm_ack is False)
 266                             ):
 267                                 log.info(
 268                                     "Executing request to url %s for vnf alarm %s with status %s",
 269                                     action.url,
 270                                     alarm.alarm_id,
 271                                     status,
 272                                 )
 273                                 try:
 274                                     if status == "alarm" and alarm.last_action == "ok":
 275                                         alarm.id_suffix += 1
 276                                         alarm.ok_ack = False
 277                                     if status == "ok" and alarm.last_action == "alarm":
 278                                         alarm.alarm_ack = False
 279                                     alarm.last_action = status
 280                                     alarm.save()
 281                                 except Exception as e:
 282                                     log.exception(e)
 283
 284                                 payload["notify_details"][
 285                                     "alarm_number"
 286                                 ] = alarm.id_suffix
 287                                 headers = {"content-type": "application/json"}
 288                                 try:
 289                                     resp = requests.post(
 290                                         url=action.url,
 291                                         data=json.dumps(payload),
 292                                         headers=headers,
 293                                         verify=False,
 294                                         timeout=alert_timeout,
 295                                     )
 296                                     log.info("Response %s", resp)
 297                                     if resp.status_code == 200:
 298                                         if status == "ok":
 299                                             alarm.ok_ack = True
 300                                             alarm.save()
 301                                         if status == "alarm":
 302                                             alarm.alarm_ack = True
 303                                             alarm.save()
 304                                         if status == "insufficient-data":
 305                                             alarm.alarm_ack = False
 306                                             alarm.ok_ack = False
 307                                             alarm.save()
 308                                 except ConnectionError:
 309                                     log.exception(
 310                                         "Error connecting to url %s", action.url
 311                                     )
 312                                 except RequestException as e:
 313                                     log.info(
 314                                         "Error: RequestException while connecting to url %s",
 315                                         action.url,
 316                                     )
 317                                     log.debug("RequestException %s", e)
 318
 319                         else:
 320                             log.info(
 321                                 "Executing request to url %s for vnf alarm %s with status %s",
 322                                 action.url,
 323                                 alarm.alarm_id,
 324                                 status,
 325                             )
 326                             try:
 327                                 requests.post(
 328                                     url=action.url,
 329                                     json=json.dumps(payload),
 330                                     timeout=alert_timeout,
 331                                 )
 332                             except ConnectionError:
 333                                 log.exception("Error connecting to url %s", action.url)
 334                             except RequestException as e:
 335                                 log.info(
 336                                     "Error: RequestException while connecting to url %s",
 337                                     action.url,
 338                                 )
 339                                 log.debug("RequestException %s", e)
 340
 341         except VnfAlarm.DoesNotExist:
 342             log.debug(
 343                 "There is no alarming action configured for alarm %s.", alarm_uuid
 344             )
 345         finally:
 346             database.db.close()
 347
 348     def _get_metric_name(self, vnf_monitoring_param: dict):
 349         if "performance-metric" in vnf_monitoring_param:
 350             return vnf_monitoring_param["performance-metric"]
 351         raise ValueError(
 352             "No metric name found for vnf_monitoring_param %s"
 353             % vnf_monitoring_param["id"]
 354         )