X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;ds=inline;f=plugins%2FOpenStack%2FAodh%2Falarming.py;h=234337226b51b86a84af3a9bf0938586212eb9db;hb=fe92f84dab8052613b18fbc40dfb12d8bc8721e0;hp=b5e4a3e896de80d8745561fc6f5931d88592d3c5;hpb=0e7809a12eba0c466923ff4783b8fbc3b932eb0d;p=osm%2FMON.git diff --git a/plugins/OpenStack/Aodh/alarming.py b/plugins/OpenStack/Aodh/alarming.py index b5e4a3e..2343372 100644 --- a/plugins/OpenStack/Aodh/alarming.py +++ b/plugins/OpenStack/Aodh/alarming.py @@ -22,43 +22,42 @@ """Carry out alarming requests via Aodh API.""" import json -import logging as log -from core.message_bus.producer import KafkaProducer +import logging -from kafka import KafkaConsumer +from core.message_bus.producer import KafkaProducer -from plugins.OpenStack.common import Common from plugins.OpenStack.response import OpenStack_Response +from plugins.OpenStack.settings import Config __author__ = "Helena McGough" -ALARM_NAMES = [ - "Average_Memory_Usage_Above_Threshold", - "Read_Latency_Above_Threshold", - "Write_Latency_Above_Threshold", - "DISK_READ_OPS", - "DISK_WRITE_OPS", - "DISK_READ_BYTES", - "DISK_WRITE_BYTES", - "Net_Packets_Dropped", - "Packets_in_Above_Threshold", - "Packets_out_Above_Threshold", - "CPU_Utilization_Above_Threshold"] +log = logging.getLogger(__name__) + +ALARM_NAMES = { + "average_memory_usage_above_threshold": "average_memory_utilization", + "disk_read_ops": "disk_read_ops", + "disk_write_ops": "disk_write_ops", + "disk_read_bytes": "disk_read_bytes", + "disk_write_bytes": "disk_write_bytes", + "net_packets_dropped": "packets_dropped", + "packets_in_above_threshold": "packets_received", + "packets_out_above_threshold": "packets_sent", + "cpu_utilization_above_threshold": "cpu_utilization"} SEVERITIES = { - "WARNING": "low", - "MINOR": "low", - "MAJOR": "moderate", - "CRITICAL": "critical", - "INDETERMINATE": "critical"} + "warning": "low", + "minor": "low", + "major": "moderate", + "critical": "critical", + "indeterminate": "critical"} STATISTICS = { - "AVERAGE": "avg", - "MINIMUM": "min", - "MAXIMUM": "max", - "COUNT": "count", - "SUM": "sum"} + "average": "avg", + "minimum": "min", + "maximum": "max", + "count": "count", + "sum": "sum"} class Alarming(object): @@ -66,14 +65,14 @@ class Alarming(object): def __init__(self): """Create the OpenStack alarming instance.""" - self._common = Common() + # Initialize configuration and notifications + config = Config.instance() + config.read_environ("aodh") - # TODO(mcgoughh): Remove hardcoded kafkaconsumer - # Initialize a generic consumer object to consume message from the SO - server = {'server': 'localhost:9092', 'topic': 'alarm_request'} - self._consumer = KafkaConsumer(server['topic'], - group_id='osm_mon', - bootstrap_servers=server['server']) + # Initialise authentication for API requests + self.auth_token = None + self.endpoint = None + self.common = None # Use the Response class to generate valid json response messages self._response = OpenStack_Response() @@ -81,133 +80,126 @@ class Alarming(object): # Initializer a producer to send responses back to SO self._producer = KafkaProducer("alarm_response") - def alarming(self): + def alarming(self, message, common, auth_token): """Consume info from the message bus to manage alarms.""" - # Check the alarming functionlity that needs to be performed - for message in self._consumer: - - values = json.loads(message.value) - vim_type = values['vim_type'].lower() - - if vim_type == "openstack": - log.info("Alarm action required: %s" % (message.topic)) - - # Generate and auth_token and endpoint for request - auth_token, endpoint = self.authenticate(values) - - if message.key == "create_alarm_request": - # Configure/Update an alarm - alarm_details = values['alarm_create_request'] - - alarm_id, alarm_status = self.configure_alarm( - endpoint, auth_token, alarm_details) - - # Generate a valid response message, send via producer - try: - resp_message = self._response.generate_response( - 'create_alarm_response', status=alarm_status, - alarm_id=alarm_id, - cor_id=alarm_details['correlation_id']) - self._producer.create_alarm_response( - 'create_alarm_resonse', resp_message, - 'alarm_response') - except Exception as exc: - log.warn("Response creation failed: %s", exc) - - elif message.key == "list_alarm_request": - # Check for a specifed: alarm_name, resource_uuid, severity - # and generate the appropriate list - list_details = values['alarm_list_request'] - try: - name = list_details['alarm_name'] - alarm_list = self.list_alarms( - endpoint, auth_token, alarm_name=name) - except Exception as a_name: - log.debug("No name specified for list:%s", a_name) - try: - resource = list_details['resource_uuid'] - alarm_list = self.list_alarms( - endpoint, auth_token, resource_id=resource) - except Exception as r_id: - log.debug("No resource id specified for this list:\ - %s", r_id) - try: - severe = list_details['severity'] - alarm_list = self.list_alarms( - endpoint, auth_token, severity=severe) - except Exception as exc: - log.warn("No severity specified for list: %s.\ - will return full list.", exc) - alarm_list = self.list_alarms( - endpoint, auth_token) - - try: - # Generate and send a list response back - resp_message = self._response.generate_response( - 'list_alarm_response', alarm_list=alarm_list, - cor_id=list_details['correlation_id']) - self._producer.list_alarm_response( - 'list_alarm_response', resp_message, - 'alarm_response') - except Exception as exc: - log.warn("Failed to send a valid response back.") - - elif message.key == "delete_alarm_request": - request_details = values['alarm_delete_request'] - alarm_id = request_details['alarm_uuid'] - - resp_status = self.delete_alarm( - endpoint, auth_token, alarm_id) - - # Generate and send a response message - try: - resp_message = self._response.generate_response( - 'delete_alarm_response', alarm_id=alarm_id, - status=resp_status, - cor_id=request_details['correlation_id']) - self._producer.delete_alarm_response( - 'delete_alarm_response', resp_message, - 'alarm_response') - except Exception as exc: - log.warn("Failed to create delete reponse:%s", exc) - - elif message.key == "acknowledge_alarm": - # Acknowledge that an alarm has been dealt with by the SO - alarm_id = values['ack_details']['alarm_uuid'] - - response = self.update_alarm_state( - endpoint, auth_token, alarm_id) - - # Log if an alarm was reset - if response is True: - log.info("Acknowledged the alarm and cleared it.") - else: - log.warn("Failed to acknowledge/clear the alarm.") - - elif message.key == "update_alarm_request": - # Update alarm configurations - alarm_details = values['alarm_update_request'] - - alarm_id, status = self.update_alarm( - endpoint, auth_token, alarm_details) - - # Generate a response for an update request - try: - resp_message = self._response.generate_response( - 'update_alarm_response', alarm_id=alarm_id, - cor_id=alarm_details['correlation_id'], - status=status) - self._producer.update_alarm_response( - 'update_alarm_response', resp_message, - 'alarm_response') - except Exception as exc: - log.warn("Failed to send an update response:%s", exc) + values = json.loads(message.value) + self.common = common - else: - log.debug("Unknown key, no action will be performed") + log.info("OpenStack alarm action required.") + + # Generate and auth_token and endpoint for request + if auth_token is not None: + if self.auth_token != auth_token: + log.info("Auth_token for alarming set by access_credentials.") + self.auth_token = auth_token + else: + log.info("Auth_token has not been updated.") + else: + log.info("Using environment variables to set auth_token for Aodh.") + self.auth_token = self.common._authenticate() + + if self.endpoint is None: + log.info("Generating a new endpoint for Aodh.") + self.endpoint = self.common.get_endpoint("alarming") + + if message.key == "create_alarm_request": + # Configure/Update an alarm + alarm_details = values['alarm_create_request'] + + alarm_id, alarm_status = self.configure_alarm( + self.endpoint, self.auth_token, alarm_details) + + # Generate a valid response message, send via producer + try: + if alarm_status is True: + log.info("Alarm successfully created") + + resp_message = self._response.generate_response( + 'create_alarm_response', status=alarm_status, + alarm_id=alarm_id, + cor_id=alarm_details['correlation_id']) + log.info("Response Message: %s", resp_message) + self._producer.create_alarm_response( + 'create_alarm_resonse', resp_message, + 'alarm_response') + except Exception as exc: + log.warn("Response creation failed: %s", exc) + + elif message.key == "list_alarm_request": + # Check for a specifed: alarm_name, resource_uuid, severity + # and generate the appropriate list + list_details = values['alarm_list_request'] + + alarm_list = self.list_alarms( + self.endpoint, self.auth_token, list_details) + + try: + # Generate and send a list response back + resp_message = self._response.generate_response( + 'list_alarm_response', alarm_list=alarm_list, + cor_id=list_details['correlation_id']) + log.info("Response Message: %s", resp_message) + self._producer.list_alarm_response( + 'list_alarm_response', resp_message, + 'alarm_response') + except Exception as exc: + log.warn("Failed to send a valid response back.") + + elif message.key == "delete_alarm_request": + request_details = values['alarm_delete_request'] + alarm_id = request_details['alarm_uuid'] + + resp_status = self.delete_alarm( + self.endpoint, self.auth_token, alarm_id) + + # Generate and send a response message + try: + resp_message = self._response.generate_response( + 'delete_alarm_response', alarm_id=alarm_id, + status=resp_status, + cor_id=request_details['correlation_id']) + log.info("Response message: %s", resp_message) + self._producer.delete_alarm_response( + 'delete_alarm_response', resp_message, + 'alarm_response') + except Exception as exc: + log.warn("Failed to create delete reponse:%s", exc) + + elif message.key == "acknowledge_alarm": + # Acknowledge that an alarm has been dealt with by the SO + alarm_id = values['ack_details']['alarm_uuid'] + + response = self.update_alarm_state( + self.endpoint, self.auth_token, alarm_id) + + # Log if an alarm was reset + if response is True: + log.info("Acknowledged the alarm and cleared it.") else: - log.info("Message topic not relevant to this plugin: %s", - message.topic) + log.warn("Failed to acknowledge/clear the alarm.") + + elif message.key == "update_alarm_request": + # Update alarm configurations + alarm_details = values['alarm_update_request'] + + alarm_id, status = self.update_alarm( + self.endpoint, self.auth_token, alarm_details) + + # Generate a response for an update request + try: + resp_message = self._response.generate_response( + 'update_alarm_response', alarm_id=alarm_id, + cor_id=alarm_details['correlation_id'], + status=status) + log.info("Response message: %s", resp_message) + self._producer.update_alarm_response( + 'update_alarm_response', resp_message, + 'alarm_response') + except Exception as exc: + log.warn("Failed to send an update response:%s", exc) + + else: + log.debug("Unknown key, no action will be performed") return @@ -216,23 +208,34 @@ class Alarming(object): url = "{}/v2/alarms/".format(endpoint) # Check if the desired alarm is supported - alarm_name = values['alarm_name'] - if alarm_name not in ALARM_NAMES: + alarm_name = values['alarm_name'].lower() + metric_name = values['metric_name'].lower() + resource_id = values['resource_uuid'] + + if alarm_name not in ALARM_NAMES.keys(): log.warn("This alarm is not supported, by a valid metric.") return None, False + if ALARM_NAMES[alarm_name] != metric_name: + log.warn("This is not the correct metric for this alarm.") + return None, False + + # Check for the required metric + metric_id = self.check_for_metric(auth_token, metric_name, resource_id) try: - metric_name = values['metric_name'] - resource_id = values['resource_uuid'] - # Check the payload for the desired alarm - payload = self.check_payload(values, metric_name, resource_id, - alarm_name) - new_alarm = self._common._perform_request( - url, auth_token, req_type="post", payload=payload) - - return json.loads(new_alarm.text)['alarm_id'], True + if metric_id is not None: + # Create the alarm if metric is available + payload = self.check_payload(values, metric_name, resource_id, + alarm_name) + new_alarm = self.common._perform_request( + url, auth_token, req_type="post", payload=payload) + return json.loads(new_alarm.text)['alarm_id'], True + else: + log.warn("The required Gnocchi metric does not exist.") + return None, False + except Exception as exc: - log.warn("Alarm creation could not be performed: %s", exc) + log.warn("Failed to create the alarm: %s", exc) return None, False def delete_alarm(self, endpoint, auth_token, alarm_id): @@ -240,9 +243,10 @@ class Alarming(object): url = "{}/v2/alarms/%s".format(endpoint) % (alarm_id) try: - result = self._common._perform_request( + result = self.common._perform_request( url, auth_token, req_type="delete") if str(result.status_code) == "404": + log.info("Alarm doesn't exist: %s", result.status_code) # If status code is 404 alarm did not exist return False else: @@ -252,34 +256,86 @@ class Alarming(object): log.warn("Failed to delete alarm: %s because %s.", alarm_id, exc) return False - def list_alarms(self, endpoint, auth_token, - alarm_name=None, resource_id=None, severity=None): + def list_alarms(self, endpoint, auth_token, list_details): """Generate the requested list of alarms.""" url = "{}/v2/alarms/".format(endpoint) - alarm_list = [] - - result = self._common._perform_request( - url, auth_token, req_type="get") - if result is not None: - # Check for a specified list based on: - # alarm_name, severity, resource_id - if alarm_name is not None: - for alarm in json.loads(result.text): - if alarm_name in str(alarm): - alarm_list.append(str(alarm)) - elif resource_id is not None: - for alarm in json.loads(result.text): - if resource_id in str(alarm): - alarm_list.append(str(alarm)) - elif severity is not None: + a_list, name_list, sev_list, res_list = [], [], [], [] + + # TODO(mcgoughh): for now resource_id is a mandatory field + # Check for a reqource is + try: + resource = list_details['resource_uuid'] + except KeyError as exc: + log.warn("Resource id not specified for list request: %s", exc) + return None + + # Checking what fields are specified for a list request + try: + name = list_details['alarm_name'].lower() + if name not in ALARM_NAMES.keys(): + log.warn("This alarm is not supported, won't be used!") + name = None + except KeyError as exc: + log.info("Alarm name isn't specified.") + name = None + + try: + severity = list_details['severity'].lower() + sev = SEVERITIES[severity] + except KeyError as exc: + log.info("Severity is unspecified/incorrectly configured") + sev = None + + # Perform the request to get the desired list + try: + result = self.common._perform_request( + url, auth_token, req_type="get") + + if result is not None: + # Get list based on resource id for alarm in json.loads(result.text): - if severity in str(alarm): - alarm_list.append(str(alarm)) + rule = alarm['gnocchi_resources_threshold_rule'] + if resource == rule['resource_id']: + res_list.append(str(alarm)) + if not res_list: + log.info("No alarms for this resource") + return a_list + + # Generate specified listed if requested + if name is not None and sev is not None: + log.info("Return a list of %s alarms with %s severity.", + name, sev) + for alarm in json.loads(result.text): + if name == alarm['name']: + name_list.append(str(alarm)) + for alarm in json.loads(result.text): + if sev == alarm['severity']: + sev_list.append(str(alarm)) + name_sev_list = list(set(name_list).intersection(sev_list)) + a_list = list(set(name_sev_list).intersection(res_list)) + elif name is not None: + log.info("Returning a %s list of alarms.", name) + for alarm in json.loads(result.text): + if name == alarm['name']: + name_list.append(str(alarm)) + a_list = list(set(name_list).intersection(res_list)) + elif sev is not None: + log.info("Returning %s severity alarm list.", sev) + for alarm in json.loads(result.text): + if sev == alarm['severity']: + sev_list.append(str(alarm)) + a_list = list(set(sev_list).intersection(res_list)) + else: + log.info("Returning an entire list of alarms.") + a_list = res_list else: - alarm_list = result.text - else: + log.info("There are no alarms!") + + except Exception as exc: + log.info("Failed to generate required list: %s", exc) return None - return alarm_list + + return a_list def update_alarm_state(self, endpoint, auth_token, alarm_id): """Set the state of an alarm to ok when ack message is received.""" @@ -287,7 +343,7 @@ class Alarming(object): payload = json.dumps("ok") try: - self._common._perform_request( + self.common._perform_request( url, auth_token, req_type="put", payload=payload) return True except Exception as exc: @@ -301,7 +357,7 @@ class Alarming(object): # Gets current configurations about the alarm try: - result = self._common._perform_request( + result = self.common._perform_request( url, auth_token, req_type="get") alarm_name = json.loads(result.text)['name'] rule = json.loads(result.text)['gnocchi_resources_threshold_rule'] @@ -310,7 +366,7 @@ class Alarming(object): metric_name = rule['metric'] except Exception as exc: log.warn("Failed to retreive existing alarm info: %s.\ - Can only update OSM created alarms.", exc) + Can only update OSM alarms.", exc) return None, False # Generates and check payload configuration for alarm update @@ -320,7 +376,7 @@ class Alarming(object): # Updates the alarm configurations with the valid payload if payload is not None: try: - update_alarm = self._common._perform_request( + update_alarm = self.common._perform_request( url, auth_token, req_type="put", payload=payload) return json.loads(update_alarm.text)['alarm_id'], True @@ -334,13 +390,13 @@ class Alarming(object): """Check that the payload is configuration for update/create alarm.""" try: # Check state and severity - severity = values['severity'] - if severity == "INDETERMINATE": + severity = values['severity'].lower() + if severity == "indeterminate": alarm_state = "insufficient data" if alarm_state is None: alarm_state = "ok" - statistic = values['statistic'] + statistic = values['statistic'].lower() # Try to configure the payload for the update/create request # Can only update: threshold, operation, statistic and # the severity of the alarm @@ -360,29 +416,34 @@ class Alarming(object): log.warn("Alarm is not configured correctly: %s", exc) return None - def authenticate(self, values): - """Generate an authentication token and endpoint for alarm request.""" - try: - # Check for a tenant_id - auth_token = self._common._authenticate( - tenant_id=values['tenant_uuid']) - endpoint = self._common.get_endpoint("alarming") - except Exception as exc: - log.warn("Tenant ID is not specified. Will use a generic\ - authentication: %s", exc) - auth_token = self._common._authenticate() - endpoint = self._common.get_endpoint("alarming") - - return auth_token, endpoint - def get_alarm_state(self, endpoint, auth_token, alarm_id): """Get the state of the alarm.""" url = "{}/v2/alarms/%s/state".format(endpoint) % alarm_id try: - alarm_state = self._common._perform_request( + alarm_state = self.common._perform_request( url, auth_token, req_type="get") return json.loads(alarm_state.text) except Exception as exc: log.warn("Failed to get the state of the alarm:%s", exc) return None + + def check_for_metric(self, auth_token, m_name, r_id): + """Check for the alarm metric.""" + try: + endpoint = self.common.get_endpoint("metric") + + url = "{}/v1/metric/".format(endpoint) + metric_list = self.common._perform_request( + url, auth_token, req_type="get") + + for metric in json.loads(metric_list.text): + name = metric['name'] + resource = metric['resource_id'] + if (name == m_name and resource == r_id): + metric_id = metric['id'] + log.info("The required metric exists, an alarm will be created.") + return metric_id + except Exception as exc: + log.info("Desired Gnocchi metric not found:%s", exc) + return None