osm_mon/plugins/OpenStack/Aodh/alarming.py

   1 # Copyright 2017 Intel Research and Development Ireland Limited
   2 # *************************************************************
   3
   4 # This file is part of OSM Monitoring module
   5 # All Rights Reserved to Intel Corporation
   6
   7 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   8 # not use this file except in compliance with the License. You may obtain
   9 # a copy of the License at
  10
  11 #         http://www.apache.org/licenses/LICENSE-2.0
  12
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  15 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  16 # License for the specific language governing permissions and limitations
  17 # under the License.
  18
  19 # For those usages not covered by the Apache License, Version 2.0 please
  20 # contact: helena.mcgough@intel.com or adrian.hoban@intel.com
  21 ##
  22 """Carry out alarming requests via Aodh API."""
  23
  24 import json
  25 import logging
  26
  27 import six
  28 import yaml
  29
  30 from osm_mon.core.auth import AuthManager
  31 from osm_mon.core.database import DatabaseManager
  32 from osm_mon.core.message_bus.producer import KafkaProducer
  33 from osm_mon.core.settings import Config
  34 from osm_mon.plugins.OpenStack.common import Common
  35 from osm_mon.plugins.OpenStack.response import OpenStack_Response
  36
  37 log = logging.getLogger(__name__)
  38
  39 METRIC_MAPPINGS = {
  40     "average_memory_utilization": "memory.percent",
  41     "disk_read_ops": "disk.read.requests",
  42     "disk_write_ops": "disk.write.requests",
  43     "disk_read_bytes": "disk.read.bytes",
  44     "disk_write_bytes": "disk.write.bytes",
  45     "packets_dropped": "interface.if_dropped",
  46     "packets_received": "interface.if_packets",
  47     "packets_sent": "interface.if_packets",
  48     "cpu_utilization": "cpu_util",
  49 }
  50
  51 SEVERITIES = {
  52     "warning": "low",
  53     "minor": "low",
  54     "major": "moderate",
  55     "critical": "critical",
  56     "indeterminate": "critical"}
  57
  58 STATISTICS = {
  59     "average": "mean",
  60     "minimum": "min",
  61     "maximum": "max",
  62     "count": "count",
  63     "sum": "sum"}
  64
  65
  66 class Alarming(object):
  67     """Carries out alarming requests and responses via Aodh API."""
  68
  69     def __init__(self):
  70         """Create the OpenStack alarming instance."""
  71         # Initialize configuration and notifications
  72         config = Config.instance()
  73         config.read_environ()
  74
  75         self._database_manager = DatabaseManager()
  76         self._auth_manager = AuthManager()
  77
  78         # Use the Response class to generate valid json response messages
  79         self._response = OpenStack_Response()
  80
  81         # Initializer a producer to send responses back to SO
  82         self._producer = KafkaProducer("alarm_response")
  83
  84     def configure_alarm(self, alarm_endpoint, metric_endpoint, auth_token, values, vim_config):
  85         """Create requested alarm in Aodh."""
  86         url = "{}/v2/alarms/".format(alarm_endpoint)
  87
  88         # Check if the desired alarm is supported
  89         alarm_name = values['alarm_name'].lower()
  90         metric_name = values['metric_name'].lower()
  91         resource_id = values['resource_uuid']
  92
  93         if metric_name not in METRIC_MAPPINGS.keys():
  94             log.warning("This metric is not supported.")
  95             return None, False
  96
  97         # Check for the required metric
  98         metric_id = self.check_for_metric(auth_token, metric_endpoint, metric_name, resource_id)
  99
 100         try:
 101             if metric_id is not None:
 102                 # Create the alarm if metric is available
 103                 if 'granularity' in vim_config and 'granularity' not in values:
 104                     values['granularity'] = vim_config['granularity']
 105                 payload = self.check_payload(values, metric_name, resource_id,
 106                                              alarm_name)
 107                 new_alarm = Common.perform_request(
 108                     url, auth_token, req_type="post", payload=payload)
 109                 return json.loads(new_alarm.text)['alarm_id'], True
 110             else:
 111                 log.warning("The required Gnocchi metric does not exist.")
 112                 return None, False
 113
 114         except Exception as exc:
 115             log.warning("Failed to create the alarm: %s", exc)
 116         return None, False
 117
 118     def alarming(self, message, vim_uuid):
 119         """Consume info from the message bus to manage alarms."""
 120         try:
 121             values = json.loads(message.value)
 122         except ValueError:
 123             values = yaml.safe_load(message.value)
 124
 125         log.info("OpenStack alarm action required.")
 126
 127         auth_token = Common.get_auth_token(vim_uuid)
 128
 129         alarm_endpoint = Common.get_endpoint("alarming", vim_uuid)
 130         metric_endpoint = Common.get_endpoint("metric", vim_uuid)
 131
 132         vim_account = self._auth_manager.get_credentials(vim_uuid)
 133         vim_config = json.loads(vim_account.config)
 134
 135         if message.key == "create_alarm_request":
 136             # Configure/Update an alarm
 137             alarm_details = values['alarm_create_request']
 138
 139             alarm_id, alarm_status = self.configure_alarm(
 140                 alarm_endpoint, metric_endpoint, auth_token, alarm_details, vim_config)
 141
 142             # Generate a valid response message, send via producer
 143             if alarm_status is True:
 144                 log.info("Alarm successfully created")
 145                 self._database_manager.save_alarm(alarm_id, vim_uuid)
 146             try:
 147                 resp_message = self._response.generate_response(
 148                     'create_alarm_response', status=alarm_status,
 149                     alarm_id=alarm_id,
 150                     cor_id=alarm_details['correlation_id'])
 151                 log.info("Response Message: %s", resp_message)
 152                 self._producer.create_alarm_response(
 153                     'create_alarm_response', resp_message)
 154             except Exception:
 155                 log.exception("Response creation failed:")
 156
 157         elif message.key == "list_alarm_request":
 158             # Check for a specified: alarm_name, resource_uuid, severity
 159             # and generate the appropriate list
 160             list_details = values['alarm_list_request']
 161
 162             alarm_list = self.list_alarms(
 163                 alarm_endpoint, auth_token, list_details)
 164
 165             try:
 166                 # Generate and send a list response back
 167                 resp_message = self._response.generate_response(
 168                     'list_alarm_response', alarm_list=alarm_list,
 169                     cor_id=list_details['correlation_id'])
 170                 log.info("Response Message: %s", resp_message)
 171                 self._producer.list_alarm_response(
 172                     'list_alarm_response', resp_message)
 173             except Exception:
 174                 log.exception("Failed to send a valid response back.")
 175
 176         elif message.key == "delete_alarm_request":
 177             request_details = values['alarm_delete_request']
 178             alarm_id = request_details['alarm_uuid']
 179
 180             resp_status = self.delete_alarm(
 181                 alarm_endpoint, auth_token, alarm_id)
 182
 183             # Generate and send a response message
 184             try:
 185                 resp_message = self._response.generate_response(
 186                     'delete_alarm_response', alarm_id=alarm_id,
 187                     status=resp_status,
 188                     cor_id=request_details['correlation_id'])
 189                 log.info("Response message: %s", resp_message)
 190                 self._producer.delete_alarm_response(
 191                     'delete_alarm_response', resp_message)
 192             except Exception:
 193                 log.exception("Failed to create delete response: ")
 194
 195         elif message.key == "acknowledge_alarm":
 196             # Acknowledge that an alarm has been dealt with by the SO
 197             alarm_id = values['ack_details']['alarm_uuid']
 198
 199             response = self.update_alarm_state(
 200                 alarm_endpoint, auth_token, alarm_id)
 201
 202             # Log if an alarm was reset
 203             if response is True:
 204                 log.info("Acknowledged the alarm and cleared it.")
 205             else:
 206                 log.warning("Failed to acknowledge/clear the alarm.")
 207
 208         elif message.key == "update_alarm_request":
 209             # Update alarm configurations
 210             alarm_details = values['alarm_update_request']
 211
 212             alarm_id, status = self.update_alarm(
 213                 alarm_endpoint, auth_token, alarm_details, vim_config)
 214
 215             # Generate a response for an update request
 216             try:
 217                 resp_message = self._response.generate_response(
 218                     'update_alarm_response', alarm_id=alarm_id,
 219                     cor_id=alarm_details['correlation_id'],
 220                     status=status)
 221                 log.info("Response message: %s", resp_message)
 222                 self._producer.update_alarm_response(
 223                     'update_alarm_response', resp_message)
 224             except Exception:
 225                 log.exception("Failed to send an update response: ")
 226
 227         else:
 228             log.debug("Unknown key, no action will be performed")
 229
 230         return
 231
 232     def delete_alarm(self, endpoint, auth_token, alarm_id):
 233         """Delete alarm function."""
 234         url = "{}/v2/alarms/%s".format(endpoint) % alarm_id
 235
 236         try:
 237             result = Common.perform_request(
 238                 url, auth_token, req_type="delete")
 239             if str(result.status_code) == "404":
 240                 log.info("Alarm doesn't exist: %s", result.status_code)
 241                 # If status code is 404 alarm did not exist
 242                 return False
 243             else:
 244                 return True
 245
 246         except Exception:
 247             log.exception("Failed to delete alarm %s :", alarm_id)
 248         return False
 249
 250     def list_alarms(self, endpoint, auth_token, list_details):
 251         """Generate the requested list of alarms."""
 252         url = "{}/v2/alarms/".format(endpoint)
 253         a_list, name_list, sev_list, res_list = [], [], [], []
 254
 255         # TODO(mcgoughh): for now resource_id is a mandatory field
 256         # Check for a resource id
 257         try:
 258             resource = list_details['resource_uuid']
 259         except KeyError as exc:
 260             log.warning("Resource id not specified for list request: %s", exc)
 261             return None
 262
 263         # Checking what fields are specified for a list request
 264         try:
 265             name = list_details['alarm_name'].lower()
 266         except KeyError as exc:
 267             log.info("Alarm name isn't specified.")
 268             name = None
 269
 270         try:
 271             severity = list_details['severity'].lower()
 272             sev = SEVERITIES[severity]
 273         except KeyError as exc:
 274             log.info("Severity is unspecified/incorrectly configured")
 275             sev = None
 276
 277         # Perform the request to get the desired list
 278         try:
 279             result = Common.perform_request(
 280                 url, auth_token, req_type="get")
 281
 282             if result is not None:
 283                 # Get list based on resource id
 284                 for alarm in json.loads(result.text):
 285                     rule = alarm['gnocchi_resources_threshold_rule']
 286                     if resource == rule['resource_id']:
 287                         res_list.append(alarm)
 288                     if not res_list:
 289                         log.info("No alarms for this resource")
 290                         return a_list
 291
 292                 # Generate specified listed if requested
 293                 if name is not None and sev is not None:
 294                     log.info("Return a list of %s alarms with %s severity.",
 295                              name, sev)
 296                     for alarm in json.loads(result.text):
 297                         if name == alarm['name']:
 298                             name_list.append(alarm)
 299                     for alarm in json.loads(result.text):
 300                         if sev == alarm['severity']:
 301                             sev_list.append(alarm)
 302                     name_sev_list = list(set(name_list).intersection(sev_list))
 303                     a_list = list(set(name_sev_list).intersection(res_list))
 304                 elif name is not None:
 305                     log.info("Returning a %s list of alarms.", name)
 306                     for alarm in json.loads(result.text):
 307                         if name == alarm['name']:
 308                             name_list.append(alarm)
 309                     a_list = list(set(name_list).intersection(res_list))
 310                 elif sev is not None:
 311                     log.info("Returning %s severity alarm list.", sev)
 312                     for alarm in json.loads(result.text):
 313                         if sev == alarm['severity']:
 314                             sev_list.append(alarm)
 315                     a_list = list(set(sev_list).intersection(res_list))
 316                 else:
 317                     log.info("Returning an entire list of alarms.")
 318                     a_list = res_list
 319             else:
 320                 log.info("There are no alarms!")
 321
 322         except Exception as exc:
 323             log.info("Failed to generate required list: %s", exc)
 324             return None
 325
 326         return a_list
 327
 328     def update_alarm_state(self, endpoint, auth_token, alarm_id):
 329         """Set the state of an alarm to ok when ack message is received."""
 330         url = "{}/v2/alarms/%s/state".format(endpoint) % alarm_id
 331         payload = json.dumps("ok")
 332
 333         try:
 334             Common.perform_request(
 335                 url, auth_token, req_type="put", payload=payload)
 336             return True
 337         except Exception:
 338             log.exception("Unable to update alarm state: ")
 339         return False
 340
 341     def update_alarm(self, endpoint, auth_token, values, vim_config):
 342         """Get alarm name for an alarm configuration update."""
 343         # Get already existing alarm details
 344         url = "{}/v2/alarms/%s".format(endpoint) % values['alarm_uuid']
 345
 346         # Gets current configurations about the alarm
 347         try:
 348             result = Common.perform_request(
 349                 url, auth_token, req_type="get")
 350             alarm_name = json.loads(result.text)['name']
 351             rule = json.loads(result.text)['gnocchi_resources_threshold_rule']
 352             alarm_state = json.loads(result.text)['state']
 353             resource_id = rule['resource_id']
 354             metric_name = [key for key, value in six.iteritems(METRIC_MAPPINGS) if value == rule['metric']][0]
 355         except Exception as exc:
 356             log.exception("Failed to retrieve existing alarm info. Can only update OSM alarms.")
 357             return None, False
 358
 359         # Generates and check payload configuration for alarm update
 360         if 'granularity' in vim_config and 'granularity' not in values:
 361             values['granularity'] = vim_config['granularity']
 362         payload = self.check_payload(values, metric_name, resource_id,
 363                                      alarm_name, alarm_state=alarm_state)
 364
 365         # Updates the alarm configurations with the valid payload
 366         if payload is not None:
 367             try:
 368                 update_alarm = Common.perform_request(
 369                     url, auth_token, req_type="put", payload=payload)
 370
 371                 return json.loads(update_alarm.text)['alarm_id'], True
 372             except Exception as exc:
 373                 log.exception("Alarm update could not be performed: ")
 374         return None, False
 375
 376     def check_payload(self, values, metric_name, resource_id,
 377                       alarm_name, alarm_state=None):
 378         """Check that the payload is configuration for update/create alarm."""
 379         try:
 380             cfg = Config.instance()
 381             # Check state and severity
 382
 383             severity = 'critical'
 384             if 'severity' in values:
 385                 severity = values['severity'].lower()
 386
 387             if severity == "indeterminate":
 388                 alarm_state = "insufficient data"
 389             if alarm_state is None:
 390                 alarm_state = "ok"
 391
 392             statistic = values['statistic'].lower()
 393
 394             granularity = cfg.OS_DEFAULT_GRANULARITY
 395             if 'granularity' in values:
 396                 granularity = values['granularity']
 397
 398             resource_type = 'generic'
 399             if 'resource_type' in values:
 400                 resource_type = values['resource_type'].lower()
 401
 402             # Try to configure the payload for the update/create request
 403             # Can only update: threshold, operation, statistic and
 404             # the severity of the alarm
 405             rule = {'threshold': values['threshold_value'],
 406                     'comparison_operator': values['operation'].lower(),
 407                     'metric': METRIC_MAPPINGS[metric_name],
 408                     'resource_id': resource_id,
 409                     'resource_type': resource_type,
 410                     'aggregation_method': STATISTICS[statistic],
 411                     'granularity': granularity, }
 412             payload = json.dumps({'state': alarm_state,
 413                                   'name': alarm_name,
 414                                   'severity': SEVERITIES[severity],
 415                                   'type': 'gnocchi_resources_threshold',
 416                                   'gnocchi_resources_threshold_rule': rule,
 417                                   'alarm_actions': [cfg.OS_NOTIFIER_URI], })
 418             return payload
 419         except KeyError as exc:
 420             log.warning("Alarm is not configured correctly: %s", exc)
 421         return None
 422
 423     def get_alarm_state(self, endpoint, auth_token, alarm_id):
 424         """Get the state of the alarm."""
 425         url = "{}/v2/alarms/%s/state".format(endpoint) % alarm_id
 426
 427         try:
 428             alarm_state = Common.perform_request(
 429                 url, auth_token, req_type="get")
 430             return json.loads(alarm_state.text)
 431         except Exception as exc:
 432             log.warning("Failed to get the state of the alarm:%s", exc)
 433         return None
 434
 435     def check_for_metric(self, auth_token, metric_endpoint, m_name, r_id):
 436         """Check for the alarm metric."""
 437         try:
 438             url = "{}/v1/metric?sort=name:asc".format(metric_endpoint)
 439             result = Common.perform_request(
 440                 url, auth_token, req_type="get")
 441             metric_list = []
 442             metrics_partial = json.loads(result.text)
 443             for metric in metrics_partial:
 444                 metric_list.append(metric)
 445
 446             while len(json.loads(result.text)) > 0:
 447                 last_metric_id = metrics_partial[-1]['id']
 448                 url = "{}/v1/metric?sort=name:asc&marker={}".format(metric_endpoint, last_metric_id)
 449                 result = Common.perform_request(
 450                     url, auth_token, req_type="get")
 451                 if len(json.loads(result.text)) > 0:
 452                     metrics_partial = json.loads(result.text)
 453                     for metric in metrics_partial:
 454                         metric_list.append(metric)
 455             metric_id = None
 456             for metric in metric_list:
 457                 name = metric['name']
 458                 resource = metric['resource_id']
 459                 if name == METRIC_MAPPINGS[m_name] and resource == r_id:
 460                     metric_id = metric['id']
 461             log.info("The required metric exists, an alarm will be created.")
 462             return metric_id
 463         except Exception as exc:
 464             log.info("Desired Gnocchi metric not found:%s", exc)
 465         return None