X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_mon%2Fplugins%2FOpenStack%2FGnocchi%2Fmetrics.py;h=b41b5c039d3b3ecdffd853a9c4d6efd9b755ea88;hb=75512477988ae5e287433c6c859c61de1bc82318;hp=94641e69f21dcf88a0469a319e0a83bf9bc6d969;hpb=c7397b95dbaeebd7d872779eec809daed9e487cc;p=osm%2FMON.git diff --git a/osm_mon/plugins/OpenStack/Gnocchi/metrics.py b/osm_mon/plugins/OpenStack/Gnocchi/metrics.py index 94641e6..b41b5c0 100644 --- a/osm_mon/plugins/OpenStack/Gnocchi/metrics.py +++ b/osm_mon/plugins/OpenStack/Gnocchi/metrics.py @@ -24,28 +24,28 @@ import datetime import json import logging - import time -from core.message_bus.producer import KafkaProducer - -from plugins.OpenStack.response import OpenStack_Response -from plugins.OpenStack.settings import Config +import six +import yaml -__author__ = "Helena McGough" +from osm_mon.core.message_bus.producer import KafkaProducer +from osm_mon.core.settings import Config +from osm_mon.plugins.OpenStack.common import Common +from osm_mon.plugins.OpenStack.response import OpenStack_Response log = logging.getLogger(__name__) METRIC_MAPPINGS = { - "average_memory_utilization": "memory.percent", - "disk_read_ops": "disk.disk_ops", - "disk_write_ops": "disk.disk_ops", - "disk_read_bytes": "disk.disk_octets", - "disk_write_bytes": "disk.disk_octets", + "average_memory_utilization": "memory.usage", + "disk_read_ops": "disk.read.requests", + "disk_write_ops": "disk.write.requests", + "disk_read_bytes": "disk.read.bytes", + "disk_write_bytes": "disk.write.bytes", "packets_dropped": "interface.if_dropped", "packets_received": "interface.if_packets", "packets_sent": "interface.if_packets", - "cpu_utilization": "cpu.percent", + "cpu_utilization": "cpu_util", } PERIOD_MS = { @@ -64,12 +64,10 @@ class Metrics(object): """Initialize the metric actions.""" # Configure an instance of the OpenStack metric plugin config = Config.instance() - config.read_environ("gnocchi") + config.read_environ() # Initialise authentication for API requests - self.auth_token = None - self.endpoint = None - self._common = None + self._common = Common() # Use the Response class to generate valid json response messages self._response = OpenStack_Response() @@ -77,32 +75,23 @@ class Metrics(object): # Initializer a producer to send responses back to SO self._producer = KafkaProducer("metric_response") - def metric_calls(self, message, common, auth_token): + def metric_calls(self, message): """Consume info from the message bus to manage metric requests.""" - values = json.loads(message.value) - self._common = common + try: + values = json.loads(message.value) + except ValueError: + values = yaml.safe_load(message.value) log.info("OpenStack metric action required.") - # Generate and auth_token and endpoint for request - if auth_token is not None: - if self.auth_token != auth_token: - log.info("Auth_token for metrics set by access_credentials.") - self.auth_token = auth_token - else: - log.info("Auth_token has not been updated.") - else: - log.info("Using environment variables to set Gnocchi auth_token.") - self.auth_token = self._common._authenticate() + auth_token = Common.get_auth_token(values['vim_uuid']) - if self.endpoint is None: - log.info("Generating a new endpoint for Gnocchi.") - self.endpoint = self._common.get_endpoint("metric") + endpoint = Common.get_endpoint("metric", values['vim_uuid']) if message.key == "create_metric_request": # Configure metric metric_details = values['metric_create'] metric_id, resource_id, status = self.configure_metric( - self.endpoint, self.auth_token, metric_details) + endpoint, auth_token, metric_details) # Generate and send a create metric response try: @@ -120,13 +109,16 @@ class Metrics(object): elif message.key == "read_metric_data_request": # Read all metric data related to a specified metric timestamps, metric_data = self.read_metric_data( - self.endpoint, self.auth_token, values) + endpoint, auth_token, values) # Generate and send a response message try: + + metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], + values['resource_uuid']) resp_message = self._response.generate_response( 'read_metric_data_response', - m_id=values['metric_uuid'], + m_id=metric_id, m_name=values['metric_name'], r_id=values['resource_uuid'], cor_id=values['correlation_id'], @@ -140,9 +132,10 @@ class Metrics(object): elif message.key == "delete_metric_request": # delete the specified metric in the request - metric_id = values['metric_uuid'] + metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], + values['resource_uuid']) status = self.delete_metric( - self.endpoint, self.auth_token, metric_id) + endpoint, auth_token, metric_id) # Generate and send a response message try: @@ -167,7 +160,7 @@ class Metrics(object): metric_name = req_details['metric_name'] resource_id = req_details['resource_uuid'] metric_id = self.get_metric_id( - self.endpoint, self.auth_token, metric_name, resource_id) + endpoint, auth_token, metric_name, resource_id) # Generate and send a response message try: @@ -186,7 +179,7 @@ class Metrics(object): list_details = values['metrics_list_request'] metric_list = self.list_metrics( - self.endpoint, self.auth_token, list_details) + endpoint, auth_token, list_details) # Generate and send a response message try: @@ -214,8 +207,8 @@ class Metrics(object): return None, None, False # Check/Normalize metric name - metric_name, norm_name = self.get_metric_name(values) - if norm_name is None: + norm_name, metric_name = self.get_metric_name(values) + if metric_name is None: log.warn("This metric is not supported by this plugin.") return None, resource_id, False @@ -230,7 +223,7 @@ class Metrics(object): res_url = base_url.format(endpoint) % resource_id payload = {metric_name: {'archive_policy_name': 'high', 'unit': values['metric_unit']}} - result = self._common._perform_request( + result = Common.perform_request( res_url, auth_token, req_type="post", payload=json.dumps(payload)) # Get id of newly created metric @@ -254,7 +247,7 @@ class Metrics(object): 'metrics': { metric_name: metric}}) - resource = self._common._perform_request( + resource = Common.perform_request( url, auth_token, req_type="post", payload=resource_payload) @@ -278,10 +271,10 @@ class Metrics(object): def delete_metric(self, endpoint, auth_token, metric_id): """Delete metric.""" - url = "{}/v1/metric/%s".format(endpoint) % (metric_id) + url = "{}/v1/metric/%s".format(endpoint) % metric_id try: - result = self._common._perform_request( + result = Common.perform_request( url, auth_token, req_type="delete") if str(result.status_code) == "404": log.warn("Failed to delete the metric.") @@ -294,7 +287,6 @@ class Metrics(object): def list_metrics(self, endpoint, auth_token, values): """List all metrics.""" - url = "{}/v1/metric/".format(endpoint) # Check for a specified list try: @@ -314,9 +306,23 @@ class Metrics(object): resource = None try: - result = self._common._perform_request( + url = "{}/v1/metric?sort=name:asc".format(endpoint) + result = Common.perform_request( url, auth_token, req_type="get") - metrics = json.loads(result.text) + metrics = [] + metrics_partial = json.loads(result.text) + for metric in metrics_partial: + metrics.append(metric) + + while len(json.loads(result.text)) > 0: + last_metric_id = metrics_partial[-1]['id'] + url = "{}/v1/metric?sort=name:asc&marker={}".format(endpoint, last_metric_id) + result = Common.perform_request( + url, auth_token, req_type="get") + if len(json.loads(result.text)) > 0: + metrics_partial = json.loads(result.text) + for metric in metrics_partial: + metrics.append(metric) if metrics is not None: # Format the list response @@ -348,10 +354,9 @@ class Metrics(object): def get_metric_id(self, endpoint, auth_token, metric_name, resource_id): """Check if the desired metric already exists for the resource.""" url = "{}/v1/resource/generic/%s".format(endpoint) % resource_id - try: # Try return the metric id if it exists - result = self._common._perform_request( + result = Common.perform_request( url, auth_token, req_type="get") return json.loads(result.text)['metrics'][metric_name] except Exception: @@ -360,6 +365,7 @@ class Metrics(object): def get_metric_name(self, values): """Check metric name configuration and normalize.""" + metric_name = None try: # Normalize metric name metric_name = values['metric_name'].lower() @@ -369,16 +375,18 @@ class Metrics(object): return metric_name, None def read_metric_data(self, endpoint, auth_token, values): - """Collectd metric measures over a specified time period.""" + """Collect metric measures over a specified time period.""" timestamps = [] data = [] try: + #get metric_id + metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], values['resource_uuid']) # Try and collect measures - metric_id = values['metric_uuid'] collection_unit = values['collection_unit'].upper() collection_period = values['collection_period'] # Define the start and end time based on configurations + # FIXME: Local timezone may differ from timezone set in Gnocchi, causing discrepancies in measures stop_time = time.strftime("%Y-%m-%d") + "T" + time.strftime("%X") end_time = int(round(time.time() * 1000)) if collection_unit == 'YEAR': @@ -393,7 +401,7 @@ class Metrics(object): "0": metric_id, "1": start_time, "2": stop_time} # Perform metric data request - metric_data = self._common._perform_request( + metric_data = Common.perform_request( url, auth_token, req_type="get") # Generate a list of the requested timestamps and data @@ -414,35 +422,40 @@ class Metrics(object): # Create required lists for row in metric_list: # Only list OSM metrics - if row['name'] in METRIC_MAPPINGS.keys(): - metric = {"metric_name": row['name'], + name = None + if row['name'] in METRIC_MAPPINGS.values(): + for k,v in six.iteritems(METRIC_MAPPINGS): + if row['name'] == v: + name = k + metric = {"metric_name": name, "metric_uuid": row['id'], "metric_unit": row['unit'], "resource_uuid": row['resource_id']} - resp_list.append(str(metric)) + resp_list.append(metric) # Generate metric_name specific list - if metric_name is not None: - if row['name'] == metric_name: - metric = {"metric_name": row['name'], + if metric_name is not None and name is not None: + if metric_name in METRIC_MAPPINGS.keys() and row['name'] == METRIC_MAPPINGS[metric_name]: + metric = {"metric_name": metric_name, "metric_uuid": row['id'], "metric_unit": row['unit'], "resource_uuid": row['resource_id']} - name_list.append(str(metric)) + name_list.append(metric) # Generate resource specific list - if resource is not None: + if resource is not None and name is not None: if row['resource_id'] == resource: - metric = {"metric_name": row['name'], + metric = {"metric_name": name, "metric_uuid": row['id'], "metric_unit": row['unit'], "resource_uuid": row['resource_id']} - res_list.append(str(metric)) + res_list.append(metric) # Join required lists if metric_name is not None and resource is not None: - return list(set(res_list).intersection(name_list)) + # Return intersection of res_list and name_list + return [i for i in res_list for j in name_list if i['metric_uuid'] == j['metric_uuid']] elif metric_name is not None: return name_list elif resource is not None: - return list(set(res_list).intersection(resp_list)) + return res_list else: return resp_list