From 9608081aa7b0ff8823ddbcd1144948ee0369160e Mon Sep 17 00:00:00 2001 From: Benjamin Diaz Date: Wed, 6 Jun 2018 17:08:26 -0300 Subject: [PATCH] [MON] Adds check for 'insecure' vim config param in Openstack plugin Closes-Bug: Bug 504 Signed-off-by: Benjamin Diaz Change-Id: Ib16af4bb3cad2a525b011a8d856311a534149442 --- osm_mon/core/auth.py | 7 + osm_mon/core/settings.py | 4 +- osm_mon/plugins/OpenStack/Gnocchi/metrics.py | 190 +++++++++--------- osm_mon/plugins/OpenStack/common.py | 58 ++++-- .../plugins/vRealiseOps/mon_plugin_vrops.py | 4 +- osm_mon/test/OpenStack/unit/test_common.py | 19 +- .../test/OpenStack/unit/test_metric_calls.py | 93 ++++----- .../test/OpenStack/unit/test_metric_req.py | 51 +++-- osm_mon/test/OpenStack/unit/test_notifier.py | 107 +++------- 9 files changed, 249 insertions(+), 284 deletions(-) diff --git a/osm_mon/core/auth.py b/osm_mon/core/auth.py index bb6dbba..fa80256 100644 --- a/osm_mon/core/auth.py +++ b/osm_mon/core/auth.py @@ -53,3 +53,10 @@ class AuthManager: credentials = self.get_credentials(creds_dict['_id']) if credentials: credentials.delete_instance() + + def get_config(self, vim_uuid): + return json.loads(self.get_credentials(vim_uuid).config) + + def is_verify_ssl(self, vim_uuid): + vim_config = self.get_config(vim_uuid) + return 'insecure' not in vim_config or vim_config['insecure'] is False diff --git a/osm_mon/core/settings.py b/osm_mon/core/settings.py index db78b4a..d27f0ca 100644 --- a/osm_mon/core/settings.py +++ b/osm_mon/core/settings.py @@ -64,6 +64,7 @@ class Config(object): CfgParam('DATABASE', "sqlite:///mon_sqlite.db", six.text_type), CfgParam('OS_NOTIFIER_URI', "http://localhost:8662", six.text_type), CfgParam('OS_DEFAULT_GRANULARITY', "300", six.text_type), + CfgParam('REQUEST_TIMEOUT', 10, int), ] _config_dict = {cfg.key: cfg for cfg in _configuration} @@ -73,12 +74,13 @@ class Config(object): """Set the default values.""" for cfg in self._configuration: setattr(self, cfg.key, cfg.default) + self.read_environ() def read_environ(self): """Check the appropriate environment variables and update defaults.""" for key in self._config_keys: try: - val = str(os.environ[key]) + val = self._config_dict[key].data_type(os.environ[key]) setattr(self, key, val) except KeyError as exc: log.debug("Environment variable not present: %s", exc) diff --git a/osm_mon/plugins/OpenStack/Gnocchi/metrics.py b/osm_mon/plugins/OpenStack/Gnocchi/metrics.py index ea2f2d2..7af9427 100644 --- a/osm_mon/plugins/OpenStack/Gnocchi/metrics.py +++ b/osm_mon/plugins/OpenStack/Gnocchi/metrics.py @@ -29,8 +29,8 @@ import time import six import yaml +from osm_mon.core.auth import AuthManager from osm_mon.core.message_bus.producer import KafkaProducer -from osm_mon.core.settings import Config from osm_mon.plugins.OpenStack.common import Common from osm_mon.plugins.OpenStack.response import OpenStack_Response @@ -62,12 +62,6 @@ class Metrics(object): def __init__(self): """Initialize the metric actions.""" - # Configure an instance of the OpenStack metric plugin - config = Config.instance() - config.read_environ() - - # Initialise authentication for API requests - self._common = Common() # Use the Response class to generate valid json response messages self._response = OpenStack_Response() @@ -75,33 +69,39 @@ class Metrics(object): # Initializer a producer to send responses back to SO self._producer = KafkaProducer("metric_response") + self._auth_manager = AuthManager() + def metric_calls(self, message, vim_uuid): """Consume info from the message bus to manage metric requests.""" + log.info("OpenStack metric action required.") try: values = json.loads(message.value) except ValueError: values = yaml.safe_load(message.value) - log.info("OpenStack metric action required.") - - auth_token = Common.get_auth_token(vim_uuid) - - endpoint = Common.get_endpoint("metric", vim_uuid) if 'metric_name' in values and values['metric_name'] not in METRIC_MAPPINGS.keys(): raise ValueError('Metric ' + values['metric_name'] + ' is not supported.') + verify_ssl = self._auth_manager.is_verify_ssl(vim_uuid) + + endpoint = Common.get_endpoint("metric", vim_uuid, verify_ssl=verify_ssl) + + auth_token = Common.get_auth_token(vim_uuid, verify_ssl=verify_ssl) + if message.key == "create_metric_request": # Configure metric metric_details = values['metric_create_request'] metric_id, resource_id, status = self.configure_metric( - endpoint, auth_token, metric_details) + endpoint, auth_token, metric_details, verify_ssl) # Generate and send a create metric response try: resp_message = self._response.generate_response( - 'create_metric_response', status=status, + 'create_metric_response', + status=status, cor_id=metric_details['correlation_id'], - metric_id=metric_id, r_id=resource_id) + metric_id=metric_id, + r_id=resource_id) log.info("Response messages: %s", resp_message) self._producer.create_metrics_resp( 'create_metric_response', resp_message) @@ -110,109 +110,112 @@ class Metrics(object): elif message.key == "read_metric_data_request": # Read all metric data related to a specified metric - timestamps, metric_data = self.read_metric_data( - endpoint, auth_token, values) + timestamps, metric_data = self.read_metric_data(endpoint, auth_token, values, verify_ssl) # Generate and send a response message try: - - metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], - values['resource_uuid']) + metric_id = self.get_metric_id(endpoint, + auth_token, + METRIC_MAPPINGS[values['metric_name']], + values['resource_uuid'], + verify_ssl) resp_message = self._response.generate_response( 'read_metric_data_response', m_id=metric_id, m_name=values['metric_name'], r_id=values['resource_uuid'], cor_id=values['correlation_id'], - times=timestamps, metrics=metric_data) + times=timestamps, + metrics=metric_data) log.info("Response message: %s", resp_message) self._producer.read_metric_data_response( 'read_metric_data_response', resp_message) except Exception as exc: - log.warning("Failed to send read metric response:%s", exc) + log.warning("Failed to create response: %s", exc) elif message.key == "delete_metric_request": # delete the specified metric in the request metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], - values['resource_uuid']) + values['resource_uuid'], verify_ssl) status = self.delete_metric( - endpoint, auth_token, metric_id) + endpoint, auth_token, metric_id, verify_ssl) # Generate and send a response message try: resp_message = self._response.generate_response( - 'delete_metric_response', m_id=metric_id, + 'delete_metric_response', + m_id=metric_id, m_name=values['metric_name'], - status=status, r_id=values['resource_uuid'], + status=status, + r_id=values['resource_uuid'], cor_id=values['correlation_id']) log.info("Response message: %s", resp_message) self._producer.delete_metric_response( 'delete_metric_response', resp_message) except Exception as exc: - log.warning("Failed to send delete response:%s", exc) + log.warning("Failed to create response: %s", exc) elif message.key == "update_metric_request": # Gnocchi doesn't support configuration updates # Log and send a response back to this effect - log.warning("Gnocchi doesn't support metric configuration\ - updates.") + log.warning("Gnocchi doesn't support metric configuration updates.") req_details = values['metric_create_request'] metric_name = req_details['metric_name'] resource_id = req_details['resource_uuid'] - metric_id = self.get_metric_id( - endpoint, auth_token, metric_name, resource_id) + metric_id = self.get_metric_id(endpoint, auth_token, metric_name, resource_id, verify_ssl) # Generate and send a response message try: resp_message = self._response.generate_response( - 'update_metric_response', status=False, + 'update_metric_response', + status=False, cor_id=req_details['correlation_id'], - r_id=resource_id, m_id=metric_id) + r_id=resource_id, + m_id=metric_id) log.info("Response message: %s", resp_message) self._producer.update_metric_response( 'update_metric_response', resp_message) except Exception as exc: - log.exception("Failed to send an update response:") + log.warning("Failed to create response: %s", exc) elif message.key == "list_metric_request": list_details = values['metrics_list_request'] metric_list = self.list_metrics( - endpoint, auth_token, list_details) + endpoint, auth_token, list_details, verify_ssl) # Generate and send a response message try: resp_message = self._response.generate_response( - 'list_metric_response', m_list=metric_list, + 'list_metric_response', + m_list=metric_list, cor_id=list_details['correlation_id']) log.info("Response message: %s", resp_message) self._producer.list_metric_response( 'list_metric_response', resp_message) except Exception as exc: - log.warning("Failed to send a list response:%s", exc) + log.warning("Failed to create response: %s", exc) else: - log.warning("Unknown key, no action will be performed.") + log.warning("Unknown key %s, no action will be performed.", message.key) - return - - def configure_metric(self, endpoint, auth_token, values): + def configure_metric(self, endpoint, auth_token, values, verify_ssl): """Create the new metric in Gnocchi.""" try: resource_id = values['resource_uuid'] except KeyError: - log.warning("Resource is not defined correctly.") + log.warning("resource_uuid field is missing.") return None, None, False - # Check/Normalize metric name - norm_name, metric_name = self.get_metric_name(values) - if metric_name is None: - log.warning("This metric is not supported by this plugin.") - return None, resource_id, False + try: + metric_name = values['metric_name'].lower() + except KeyError: + log.warning("metric_name field is missing.") + return None, None, False # Check for an existing metric for this resource metric_id = self.get_metric_id( - endpoint, auth_token, metric_name, resource_id) + endpoint, auth_token, metric_name, resource_id, verify_ssl) if metric_id is None: # Try appending metric to existing resource @@ -222,7 +225,10 @@ class Metrics(object): payload = {metric_name: {'archive_policy_name': 'high', 'unit': values['metric_unit']}} result = Common.perform_request( - res_url, auth_token, req_type="post", + res_url, + auth_token, + req_type="post", + verify_ssl=verify_ssl, payload=json.dumps(payload, sort_keys=True)) # Get id of newly created metric for row in json.loads(result.text): @@ -246,8 +252,11 @@ class Metrics(object): metric_name: metric}}, sort_keys=True) resource = Common.perform_request( - url, auth_token, req_type="post", - payload=resource_payload) + url, + auth_token, + req_type="post", + payload=resource_payload, + verify_ssl=verify_ssl) # Return the newly created resource_id for creating alarms new_resource_id = json.loads(resource.text)['id'] @@ -255,11 +264,11 @@ class Metrics(object): new_resource_id) metric_id = self.get_metric_id( - endpoint, auth_token, metric_name, new_resource_id) + endpoint, auth_token, metric_name, new_resource_id, verify_ssl) return metric_id, new_resource_id, True except Exception as exc: - log.warning("Failed to create a new resource:%s", exc) + log.warning("Failed to create a new resource: %s", exc) return None, None, False else: @@ -267,50 +276,42 @@ class Metrics(object): return metric_id, resource_id, False - def delete_metric(self, endpoint, auth_token, metric_id): + def delete_metric(self, endpoint, auth_token, metric_id, verify_ssl): """Delete metric.""" url = "{}/v1/metric/%s".format(endpoint) % metric_id try: result = Common.perform_request( - url, auth_token, req_type="delete") + url, + auth_token, + req_type="delete", + verify_ssl=verify_ssl) if str(result.status_code) == "404": log.warning("Failed to delete the metric.") return False else: return True except Exception as exc: - log.warning("Failed to carry out delete metric request:%s", exc) + log.warning("Failed to delete metric: %s", exc) return False - def list_metrics(self, endpoint, auth_token, values): + def list_metrics(self, endpoint, auth_token, values, verify_ssl): """List all metrics.""" # Check for a specified list - try: - # Check if the metric_name was specified for the list - if values['metric_name']: - metric_name = values['metric_name'].lower() - if metric_name not in METRIC_MAPPINGS.keys(): - log.warning("This metric is not supported, won't be listed.") - metric_name = None - else: - metric_name = None - except KeyError as exc: - log.info("Metric name is not specified: %s", exc) - metric_name = None + metric_name = None + if 'metric_name' in values: + metric_name = values['metric_name'].lower() - try: + resource = None + if 'resource_uuid' in values: resource = values['resource_uuid'] - except KeyError as exc: - log.info("Resource is not specified:%s", exc) - resource = None try: if resource: url = "{}/v1/resource/generic/{}".format(endpoint, resource) result = Common.perform_request( - url, auth_token, req_type="get") + url, auth_token, req_type="get", verify_ssl=verify_ssl) resource_data = json.loads(result.text) metrics = resource_data['metrics'] @@ -319,7 +320,7 @@ class Metrics(object): metric_id = metrics[METRIC_MAPPINGS[metric_name]] url = "{}/v1/metric/{}".format(endpoint, metric_id) result = Common.perform_request( - url, auth_token, req_type="get") + url, auth_token, req_type="get", verify_ssl=verify_ssl) metric_list = json.loads(result.text) log.info("Returning an %s resource list for %s metrics", metric_name, resource) @@ -332,7 +333,7 @@ class Metrics(object): for k, v in metrics.items(): url = "{}/v1/metric/{}".format(endpoint, v) result = Common.perform_request( - url, auth_token, req_type="get") + url, auth_token, req_type="get", verify_ssl=verify_ssl) metric = json.loads(result.text) metric_list.append(metric) if metric_list: @@ -345,7 +346,7 @@ class Metrics(object): else: url = "{}/v1/metric?sort=name:asc".format(endpoint) result = Common.perform_request( - url, auth_token, req_type="get") + url, auth_token, req_type="get", verify_ssl=verify_ssl) metrics = [] metrics_partial = json.loads(result.text) for metric in metrics_partial: @@ -355,7 +356,7 @@ class Metrics(object): last_metric_id = metrics_partial[-1]['id'] url = "{}/v1/metric?sort=name:asc&marker={}".format(endpoint, last_metric_id) result = Common.perform_request( - url, auth_token, req_type="get") + url, auth_token, req_type="get", verify_ssl=verify_ssl) if len(json.loads(result.text)) > 0: metrics_partial = json.loads(result.text) for metric in metrics_partial: @@ -375,40 +376,32 @@ class Metrics(object): log.info("There are no metrics available") return [] except Exception as exc: - log.warning("Failed to generate any metric list. %s", exc) + log.exception("Failed to list metrics. %s", exc) return None - def get_metric_id(self, endpoint, auth_token, metric_name, resource_id): + def get_metric_id(self, endpoint, auth_token, metric_name, resource_id, verify_ssl): """Check if the desired metric already exists for the resource.""" url = "{}/v1/resource/generic/%s".format(endpoint) % resource_id try: # Try return the metric id if it exists result = Common.perform_request( - url, auth_token, req_type="get") + url, + auth_token, + req_type="get", + verify_ssl=verify_ssl) return json.loads(result.text)['metrics'][metric_name] - except Exception: - log.info("Metric doesn't exist. No metric_id available") - return None - - def get_metric_name(self, values): - """Check metric name configuration and normalize.""" - metric_name = None - try: - # Normalize metric name - metric_name = values['metric_name'].lower() - return metric_name, METRIC_MAPPINGS[metric_name] except KeyError: - log.info("Metric name %s is invalid.", metric_name) - return metric_name, None + log.warning("Metric doesn't exist. No metric_id available") + return None - def read_metric_data(self, endpoint, auth_token, values): + def read_metric_data(self, endpoint, auth_token, values, verify_ssl): """Collect metric measures over a specified time period.""" timestamps = [] data = [] try: # get metric_id metric_id = self.get_metric_id(endpoint, auth_token, METRIC_MAPPINGS[values['metric_name']], - values['resource_uuid']) + values['resource_uuid'], verify_ssl) # Try and collect measures collection_unit = values['collection_unit'].upper() collection_period = values['collection_period'] @@ -430,7 +423,10 @@ class Metrics(object): # Perform metric data request metric_data = Common.perform_request( - url, auth_token, req_type="get") + url, + auth_token, + req_type="get", + verify_ssl=verify_ssl) # Generate a list of the requested timestamps and data for r in json.loads(metric_data.text): diff --git a/osm_mon/plugins/OpenStack/common.py b/osm_mon/plugins/OpenStack/common.py index e5a70f9..64cbf7f 100644 --- a/osm_mon/plugins/OpenStack/common.py +++ b/osm_mon/plugins/OpenStack/common.py @@ -25,13 +25,17 @@ import logging import requests import yaml +from keystoneauth1 import session +from keystoneauth1.identity import v3 from keystoneclient.v3 import client from osm_mon.core.auth import AuthManager +from osm_mon.core.settings import Config __author__ = "Helena McGough" log = logging.getLogger(__name__) +cfg = Config.instance() class Common(object): @@ -39,28 +43,37 @@ class Common(object): def __init__(self): """Create the common instance.""" - self.auth_manager = AuthManager() @staticmethod - def get_auth_token(vim_uuid): + def get_auth_token(vim_uuid, verify_ssl=True): """Authenticate and/or renew the authentication token.""" auth_manager = AuthManager() creds = auth_manager.get_credentials(vim_uuid) - ks = client.Client(auth_url=creds.url, - username=creds.user, - password=creds.password, - tenant_name=creds.tenant_name) - return ks.auth_token + sess = session.Session(verify=verify_ssl) + ks = client.Client(session=sess) + token_dict = ks.get_raw_token_from_identity_service(auth_url=creds.url, + username=creds.user, + password=creds.password, + project_name=creds.tenant_name, + project_domain_id='default', + user_domain_id='default') + return token_dict['auth_token'] @staticmethod - def get_endpoint(service_type, vim_uuid): + def get_endpoint(service_type, vim_uuid, verify_ssl=True): """Get the endpoint for Gnocchi/Aodh.""" auth_manager = AuthManager() creds = auth_manager.get_credentials(vim_uuid) - ks = client.Client(auth_url=creds.url, + auth = v3.Password(auth_url=creds.url, username=creds.user, password=creds.password, - tenant_name=creds.tenant_name) + project_name=creds.tenant_name, + project_domain_id='default', + user_domain_id='default') + sess = session.Session(auth=auth, verify=verify_ssl) + ks = client.Client(session=sess, interface='public') + service = ks.services.list(type=service_type)[0] + endpoints = ks.endpoints.list(service) endpoint_type = 'publicURL' region_name = 'RegionOne' if creds.config is not None: @@ -72,16 +85,17 @@ class Common(object): endpoint_type = config['endpoint_type'] if 'region_name' in config: region_name = config['region_name'] - - return ks.service_catalog.url_for( - service_type=service_type, - endpoint_type=endpoint_type, - region_name=region_name) + for endpoint in endpoints: + if endpoint.interface in endpoint_type and endpoint.region == region_name: + return endpoint.url @staticmethod def perform_request(url, auth_token, - req_type=None, payload=None, params=None): + req_type=None, payload=None, params=None, verify_ssl=True): """Perform the POST/PUT/GET/DELETE request.""" + + timeout = cfg.REQUEST_TIMEOUT + # request headers headers = {'X-Auth-Token': auth_token, 'Content-type': 'application/json'} @@ -89,25 +103,25 @@ class Common(object): if req_type == "put": response = requests.put( url, data=payload, headers=headers, - timeout=10) + timeout=timeout, verify=verify_ssl) elif req_type == "get": response = requests.get( - url, params=params, headers=headers, timeout=10) + url, params=params, headers=headers, timeout=timeout, verify=verify_ssl) elif req_type == "delete": response = requests.delete( - url, headers=headers, timeout=10) + url, headers=headers, timeout=timeout, verify=verify_ssl) else: response = requests.post( url, data=payload, headers=headers, - timeout=10) + timeout=timeout, verify=verify_ssl) # Raises exception if there was an error try: response.raise_for_status() # pylint: disable=broad-except except Exception: - # Log out the result of the request for debugging purpose - log.debug( + # Log out the result of the request + log.warning( 'Result: %s, %s', response.status_code, response.text) return response diff --git a/osm_mon/plugins/vRealiseOps/mon_plugin_vrops.py b/osm_mon/plugins/vRealiseOps/mon_plugin_vrops.py index bd86a50..935e624 100644 --- a/osm_mon/plugins/vRealiseOps/mon_plugin_vrops.py +++ b/osm_mon/plugins/vRealiseOps/mon_plugin_vrops.py @@ -24,7 +24,7 @@ """ Monitoring metrics & creating Alarm definitions in vROPs """ - +import pytz import requests import logging @@ -1263,7 +1263,7 @@ class MonPlugin(): """ date_time_formatted = '0000-00-00T00:00:00' if date_time != 0: - complete_datetime = datetime.datetime.fromtimestamp(date_time/1000.0).isoformat('T') + complete_datetime = datetime.datetime.fromtimestamp(date_time/1000.0, tz=pytz.utc).isoformat('T') date_time_formatted = complete_datetime.split('.',1)[0] return date_time_formatted diff --git a/osm_mon/test/OpenStack/unit/test_common.py b/osm_mon/test/OpenStack/unit/test_common.py index 042d15b..e6c52fb 100644 --- a/osm_mon/test/OpenStack/unit/test_common.py +++ b/osm_mon/test/OpenStack/unit/test_common.py @@ -31,7 +31,6 @@ from keystoneclient.v3 import client from osm_mon.core.auth import AuthManager from osm_mon.core.database import VimCredentials -from osm_mon.core.settings import Config from osm_mon.plugins.OpenStack.common import Common __author__ = "Helena McGough" @@ -69,14 +68,14 @@ class TestCommon(unittest.TestCase): self.creds.tenant_name = 'tenant_name' @mock.patch.object(AuthManager, "get_credentials") - @mock.patch.object(Config, "instance") - @mock.patch.object(client, "Client") - def test_get_auth_token(self, key_client, cfg, get_creds): + @mock.patch.object(client.Client, "get_raw_token_from_identity_service") + def test_get_auth_token(self, get_token, get_creds): """Test generating a new authentication token.""" get_creds.return_value = self.creds Common.get_auth_token('test_id') get_creds.assert_called_with('test_id') - key_client.assert_called_with(auth_url='url', password='password', tenant_name='tenant_name', username='user') + get_token.assert_called_with(auth_url='url', password='password', project_name='tenant_name', username='user', + project_domain_id='default', user_domain_id='default') @mock.patch.object(requests, 'post') def test_post_req(self, post): @@ -85,7 +84,7 @@ class TestCommon(unittest.TestCase): payload="payload") post.assert_called_with("url", data="payload", headers=mock.ANY, - timeout=mock.ANY) + timeout=mock.ANY, verify=True) @mock.patch.object(requests, 'get') def test_get_req(self, get): @@ -94,7 +93,7 @@ class TestCommon(unittest.TestCase): Common.perform_request("url", "auth_token", req_type="get") get.assert_called_with("url", params=None, headers=mock.ANY, - timeout=mock.ANY) + timeout=mock.ANY, verify=True) # Test with some parameters specified get.reset_mock() @@ -102,7 +101,7 @@ class TestCommon(unittest.TestCase): params="some parameters") get.assert_called_with("url", params="some parameters", - headers=mock.ANY, timeout=mock.ANY) + headers=mock.ANY, timeout=mock.ANY, verify=True) @mock.patch.object(requests, 'put') def test_put_req(self, put): @@ -110,11 +109,11 @@ class TestCommon(unittest.TestCase): Common.perform_request("url", "auth_token", req_type="put", payload="payload") put.assert_called_with("url", data="payload", headers=mock.ANY, - timeout=mock.ANY) + timeout=mock.ANY, verify=True) @mock.patch.object(requests, 'delete') def test_delete_req(self, delete): """Testing a delete request.""" Common.perform_request("url", "auth_token", req_type="delete") - delete.assert_called_with("url", headers=mock.ANY, timeout=mock.ANY) + delete.assert_called_with("url", headers=mock.ANY, timeout=mock.ANY, verify=True) diff --git a/osm_mon/test/OpenStack/unit/test_metric_calls.py b/osm_mon/test/OpenStack/unit/test_metric_calls.py index 5014133..3f89a91 100644 --- a/osm_mon/test/OpenStack/unit/test_metric_calls.py +++ b/osm_mon/test/OpenStack/unit/test_metric_calls.py @@ -29,6 +29,7 @@ import unittest import mock +from osm_mon.core.auth import AuthManager from osm_mon.plugins.OpenStack.Gnocchi import metrics as metric_req from osm_mon.plugins.OpenStack.common import Common @@ -60,6 +61,8 @@ def perform_request_side_effect(*args, **kwargs): resp = Response() if 'marker' in args[0]: resp.text = json.dumps([]) + if 'resource/generic' in args[0]: + resp.text = json.dumps({'metrics': {'cpu_util': 'test_id'}}) return resp @@ -72,87 +75,78 @@ class TestMetricCalls(unittest.TestCase): self.metrics = metric_req.Metrics() self.metrics._common = Common() - @mock.patch.object(metric_req.Metrics, "get_metric_name") @mock.patch.object(metric_req.Metrics, "get_metric_id") @mock.patch.object(Common, "perform_request") def test_invalid_config_metric_req( - self, perf_req, get_metric, get_metric_name): + self, perf_req, get_metric): """Test the configure metric function, for an invalid metric.""" # Test invalid configuration for creating a metric values = {"metric_details": "invalid_metric"} m_id, r_id, status = self.metrics.configure_metric( - endpoint, auth_token, values) + endpoint, auth_token, values, verify_ssl=False) perf_req.assert_not_called() - self.assertEqual(m_id, None) - self.assertEqual(r_id, None) self.assertEqual(status, False) # Test with an invalid metric name, will not perform request values = {"resource_uuid": "r_id"} - get_metric_name.return_value = "metric_name", None m_id, r_id, status = self.metrics.configure_metric( - endpoint, auth_token, values) + endpoint, auth_token, values, verify_ssl=False) perf_req.assert_not_called() - self.assertEqual(m_id, None) - self.assertEqual(r_id, "r_id") self.assertEqual(status, False) - get_metric_name.reset_mock() # If metric exists, it won't be recreated - get_metric_name.return_value = "metric_name", "norm_name" get_metric.return_value = "metric_id" m_id, r_id, status = self.metrics.configure_metric( - endpoint, auth_token, values) + endpoint, auth_token, values, verify_ssl=False) perf_req.assert_not_called() - self.assertEqual(m_id, "metric_id") - self.assertEqual(r_id, "r_id") self.assertEqual(status, False) - @mock.patch.object(metric_req.Metrics, "get_metric_name") @mock.patch.object(metric_req.Metrics, "get_metric_id") @mock.patch.object(Common, "perform_request") + @mock.patch.object(AuthManager, "get_credentials") def test_valid_config_metric_req( - self, perf_req, get_metric, get_metric_name): + self, get_creds, perf_req, get_metric): """Test the configure metric function, for a valid metric.""" # Test valid configuration and payload for creating a metric + get_creds.return_value = type('obj', (object,), {'config': '{"insecure":true}'}) values = {"resource_uuid": "r_id", - "metric_unit": "units"} - get_metric_name.return_value = "norm_name", "metric_name" + "metric_unit": "units", + "metric_name": "cpu_util"} get_metric.return_value = None payload = {"id": "r_id", - "metrics": {"metric_name": + "metrics": {"cpu_util": {"archive_policy_name": "high", - "name": "metric_name", + "name": "cpu_util", "unit": "units"}}} - perf_req.return_value = type('obj', (object,), {'text': '{"id":"1"}'}) + perf_req.return_value = type('obj', (object,), {'text': '{"metrics":{"cpu_util":1}, "id":1}'}) - self.metrics.configure_metric(endpoint, auth_token, values) + self.metrics.configure_metric(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_called_with( - "/v1/resource/generic", auth_token, req_type="post", + "/v1/resource/generic", auth_token, req_type="post", verify_ssl=False, payload=json.dumps(payload, sort_keys=True)) @mock.patch.object(Common, "perform_request") def test_delete_metric_req(self, perf_req): """Test the delete metric function.""" - self.metrics.delete_metric(endpoint, auth_token, "metric_id") + self.metrics.delete_metric(endpoint, auth_token, "metric_id", verify_ssl=False) perf_req.assert_called_with( - "/v1/metric/metric_id", auth_token, req_type="delete") + "/v1/metric/metric_id", auth_token, req_type="delete", verify_ssl=False) @mock.patch.object(Common, "perform_request") def test_delete_metric_invalid_status(self, perf_req): """Test invalid response for delete request.""" perf_req.return_value = type('obj', (object,), {"status_code": "404"}) - status = self.metrics.delete_metric(endpoint, auth_token, "metric_id") + status = self.metrics.delete_metric(endpoint, auth_token, "metric_id", verify_ssl=False) self.assertEqual(status, False) @@ -163,10 +157,10 @@ class TestMetricCalls(unittest.TestCase): # Test listing metrics without any configuration options values = {} perf_req.side_effect = perform_request_side_effect - self.metrics.list_metrics(endpoint, auth_token, values) + self.metrics.list_metrics(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_any_call( - "/v1/metric?sort=name:asc", auth_token, req_type="get") + "/v1/metric?sort=name:asc", auth_token, req_type="get", verify_ssl=False) resp_list.assert_called_with([{u'id': u'test_id'}]) @mock.patch.object(metric_req.Metrics, "response_list") @@ -176,10 +170,10 @@ class TestMetricCalls(unittest.TestCase): # Test listing metrics with a resource id specified values = {"resource_uuid": "resource_id"} perf_req.side_effect = perform_request_side_effect - self.metrics.list_metrics(endpoint, auth_token, values) + self.metrics.list_metrics(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_any_call( - "/v1/resource/generic/resource_id", auth_token, req_type="get") + "/v1/metric/test_id", auth_token, req_type="get", verify_ssl=False) @mock.patch.object(metric_req.Metrics, "response_list") @mock.patch.object(Common, "perform_request") @@ -188,10 +182,10 @@ class TestMetricCalls(unittest.TestCase): # Test listing metrics with a metric_name specified values = {"metric_name": "disk_write_bytes"} perf_req.side_effect = perform_request_side_effect - self.metrics.list_metrics(endpoint, auth_token, values) + self.metrics.list_metrics(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_any_call( - "/v1/metric?sort=name:asc", auth_token, req_type="get") + "/v1/metric?sort=name:asc", auth_token, req_type="get", verify_ssl=False) resp_list.assert_called_with( [{u'id': u'test_id'}], metric_name="disk_write_bytes") @@ -202,38 +196,21 @@ class TestMetricCalls(unittest.TestCase): # Test listing metrics with a resource id and metric name specified values = {"resource_uuid": "resource_id", - "metric_name": "packets_sent"} + "metric_name": "cpu_utilization"} perf_req.side_effect = perform_request_side_effect - self.metrics.list_metrics(endpoint, auth_token, values) + self.metrics.list_metrics(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_any_call( - "/v1/resource/generic/resource_id", auth_token, req_type="get") + "/v1/metric/test_id", auth_token, req_type="get", verify_ssl=False) @mock.patch.object(Common, "perform_request") def test_get_metric_id(self, perf_req): """Test get_metric_id function.""" - self.metrics.get_metric_id(endpoint, auth_token, "my_metric", "r_id") + perf_req.return_value = type('obj', (object,), {'text': '{"alarm_id":"1"}'}) + self.metrics.get_metric_id(endpoint, auth_token, "my_metric", "r_id", verify_ssl=False) perf_req.assert_called_with( - "/v1/resource/generic/r_id", auth_token, req_type="get") - - def test_get_metric_name(self): - """Test the result from the get_metric_name function.""" - # test with a valid metric_name - values = {"metric_name": "disk_write_ops"} - - metric_name, norm_name = self.metrics.get_metric_name(values) - - self.assertEqual(metric_name, "disk_write_ops") - self.assertEqual(norm_name, "disk.write.requests") - - # test with an invalid metric name - values = {"metric_name": "my_invalid_metric"} - - metric_name, norm_name = self.metrics.get_metric_name(values) - - self.assertEqual(metric_name, "my_invalid_metric") - self.assertEqual(norm_name, None) + "/v1/resource/generic/r_id", auth_token, req_type="get", verify_ssl=False) @mock.patch.object(metric_req.Metrics, "get_metric_id") @mock.patch.object(Common, "perform_request") @@ -247,7 +224,7 @@ class TestMetricCalls(unittest.TestCase): perf_req.return_value = type('obj', (object,), {'text': '{"metric_data":"[]"}'}) get_metric.return_value = "metric_id" - self.metrics.read_metric_data(endpoint, auth_token, values) + self.metrics.read_metric_data(endpoint, auth_token, values, verify_ssl=False) perf_req.assert_called_once() @@ -258,13 +235,13 @@ class TestMetricCalls(unittest.TestCase): values = {} times, data = self.metrics.read_metric_data( - endpoint, auth_token, values) + endpoint, auth_token, values, verify_ssl=False) self.assertEqual(times, []) self.assertEqual(data, []) def test_complete_response_list(self): - """Test the response list function for formating metric lists.""" + """Test the response list function for formatting metric lists.""" # Mock a list for testing purposes, with valid OSM metric resp_list = self.metrics.response_list(metric_list) diff --git a/osm_mon/test/OpenStack/unit/test_metric_req.py b/osm_mon/test/OpenStack/unit/test_metric_req.py index de39ebb..7bb81c9 100644 --- a/osm_mon/test/OpenStack/unit/test_metric_req.py +++ b/osm_mon/test/OpenStack/unit/test_metric_req.py @@ -22,16 +22,14 @@ """Tests for all metric request message keys.""" import json - import logging - import unittest import mock +from osm_mon.core.auth import AuthManager from osm_mon.core.message_bus.producer import KafkaProducer from osm_mon.plugins.OpenStack.Gnocchi import metrics as metric_req - from osm_mon.plugins.OpenStack.common import Common log = logging.getLogger(__name__) @@ -57,38 +55,47 @@ class TestMetricReq(unittest.TestCase): self.metrics = metric_req.Metrics() @mock.patch.object(Common, "get_auth_token", mock.Mock()) - @mock.patch.object(Common, 'get_endpoint', mock.Mock()) + @mock.patch.object(Common, "get_endpoint", mock.Mock()) @mock.patch.object(metric_req.Metrics, "delete_metric") @mock.patch.object(metric_req.Metrics, "get_metric_id") - def test_delete_metric_key(self, get_metric_id, del_metric): + @mock.patch.object(AuthManager, "get_credentials") + def test_delete_metric_key(self, get_creds, get_metric_id, del_metric): """Test the functionality for a delete metric request.""" # Mock a message value and key message = Message() message.key = "delete_metric_request" message.value = json.dumps({"metric_name": "disk_write_ops", "resource_uuid": "my_r_id", "correlation_id": 1}) + get_creds.return_value = type('obj', (object,), { + 'config': '{"insecure":true}' + }) del_metric.return_value = True # Call the metric functionality and check delete request get_metric_id.return_value = "my_metric_id" self.metrics.metric_calls(message, 'test_id') - del_metric.assert_called_with(mock.ANY, mock.ANY, "my_metric_id") + del_metric.assert_called_with(mock.ANY, mock.ANY, "my_metric_id", False) @mock.patch.object(Common, "get_auth_token", mock.Mock()) @mock.patch.object(Common, 'get_endpoint', mock.Mock()) @mock.patch.object(metric_req.Metrics, "list_metrics") - def test_list_metric_key(self, list_metrics): + @mock.patch.object(AuthManager, "get_credentials") + def test_list_metric_key(self, get_creds, list_metrics): """Test the functionality for a list metric request.""" # Mock a message with list metric key and value message = Message() message.key = "list_metric_request" message.value = json.dumps({"metrics_list_request": {"correlation_id": 1}}) + get_creds.return_value = type('obj', (object,), { + 'config': '{"insecure":true}' + }) + list_metrics.return_value = [] # Call the metric functionality and check list functionality self.metrics.metric_calls(message, 'test_id') - list_metrics.assert_called_with(mock.ANY, mock.ANY, {"correlation_id": 1}) + list_metrics.assert_called_with(mock.ANY, mock.ANY, {"correlation_id": 1}, False) @mock.patch.object(Common, "get_auth_token", mock.Mock()) @mock.patch.object(Common, 'get_endpoint', mock.Mock()) @@ -96,7 +103,9 @@ class TestMetricReq(unittest.TestCase): @mock.patch.object(metric_req.Metrics, "list_metrics") @mock.patch.object(metric_req.Metrics, "delete_metric") @mock.patch.object(metric_req.Metrics, "configure_metric") - def test_update_metric_key(self, config_metric, delete_metric, list_metrics, + @mock.patch.object(AuthManager, "get_credentials") + @mock.patch.object(Common, "perform_request") + def test_update_metric_key(self, perf_req, get_creds, config_metric, delete_metric, list_metrics, read_data): """Test the functionality for an update metric request.""" # Mock a message with update metric key and value @@ -107,6 +116,12 @@ class TestMetricReq(unittest.TestCase): "metric_name": "my_metric", "resource_uuid": "my_r_id"}}) + get_creds.return_value = type('obj', (object,), { + 'config': '{"insecure":true}' + }) + + perf_req.return_value = type('obj', (object,), {'text': '{"metric_id":"1"}'}) + # Call metric functionality and confirm no function is called # Gnocchi does not support updating a metric configuration self.metrics.metric_calls(message, 'test_id') @@ -118,30 +133,36 @@ class TestMetricReq(unittest.TestCase): @mock.patch.object(Common, "get_auth_token", mock.Mock()) @mock.patch.object(Common, 'get_endpoint', mock.Mock()) @mock.patch.object(metric_req.Metrics, "configure_metric") - def test_config_metric_key(self, config_metric): + @mock.patch.object(AuthManager, "get_credentials") + def test_config_metric_key(self, get_credentials, config_metric): """Test the functionality for a create metric request.""" # Mock a message with create metric key and value message = Message() message.key = "create_metric_request" - message.value = json.dumps({"metric_create_request": "metric_details"}) - + message.value = json.dumps({"metric_create_request": {"correlation_id": 123}}) + get_credentials.return_value = type('obj', (object,), {'config': '{"insecure":true}'}) # Call metric functionality and check config metric config_metric.return_value = "metric_id", "resource_id", True self.metrics.metric_calls(message, 'test_id') - config_metric.assert_called_with(mock.ANY, mock.ANY, "metric_details") + config_metric.assert_called_with(mock.ANY, mock.ANY, {"correlation_id": 123}, False) @mock.patch.object(Common, "get_auth_token", mock.Mock()) @mock.patch.object(Common, 'get_endpoint', mock.Mock()) @mock.patch.object(metric_req.Metrics, "read_metric_data") - def test_read_data_key(self, read_data): + @mock.patch.object(AuthManager, "get_credentials") + def test_read_data_key(self, get_creds, read_data): """Test the functionality for a read metric data request.""" # Mock a message with a read data key and value message = Message() message.key = "read_metric_data_request" message.value = json.dumps({"alarm_uuid": "alarm_id"}) + get_creds.return_value = type('obj', (object,), { + 'config': '{"insecure":true}' + }) + # Call metric functionality and check read data metrics read_data.return_value = "time_stamps", "data_values" self.metrics.metric_calls(message, 'test_id') read_data.assert_called_with( - mock.ANY, mock.ANY, json.loads(message.value)) + mock.ANY, mock.ANY, json.loads(message.value), False) diff --git a/osm_mon/test/OpenStack/unit/test_notifier.py b/osm_mon/test/OpenStack/unit/test_notifier.py index 4841013..76c824e 100644 --- a/osm_mon/test/OpenStack/unit/test_notifier.py +++ b/osm_mon/test/OpenStack/unit/test_notifier.py @@ -21,15 +21,18 @@ ## """Tests for all common OpenStack methods.""" +# TODO: Mock database calls. Improve assertions. + import json import unittest import mock -from six.moves.BaseHTTPServer import BaseHTTPRequestHandler +from six.moves.BaseHTTPServer import HTTPServer + +from osm_mon.core.database import DatabaseManager, Alarm from osm_mon.core.message_bus.producer import KafkaProducer -from osm_mon.core.settings import Config -from osm_mon.plugins.OpenStack.Aodh.alarming import Alarming +from osm_mon.plugins.OpenStack.Aodh.notifier import NotifierHandler from osm_mon.plugins.OpenStack.common import Common from osm_mon.plugins.OpenStack.response import OpenStack_Response @@ -71,19 +74,18 @@ class Response(object): self.text = text -class NotifierHandler(BaseHTTPRequestHandler): +class RFile(): + def read(self, content_length): + return post_data + + +class MockNotifierHandler(NotifierHandler): """Mock the NotifierHandler class for testing purposes.""" - def __init__(self, request, client_address, server): - """Initilase mock NotifierHandler.""" - self.request = request - self.client_address = client_address - self.server = server - self.setup() - try: - self.handle() - finally: - self.finish() + def __init__(self): + """Initialise mock NotifierHandler.""" + self.headers = {'Content-Length': '20'} + self.rfile = RFile() def setup(self): """Mock setup function.""" @@ -97,61 +99,6 @@ class NotifierHandler(BaseHTTPRequestHandler): """Mock finish function.""" pass - def _set_headers(self): - """Mock getting the request headers.""" - pass - - def do_GET(self): - """Mock functionality for GET request.""" - self._set_headers() - pass - - def do_POST(self): - """Mock functionality for a POST request.""" - self._set_headers() - self.notify_alarm(json.loads(post_data)) - - def notify_alarm(self, values): - """Mock the notify_alarm functionality to generate a valid response.""" - config = Config.instance() - config.read_environ() - self._alarming = Alarming() - self._common = Common() - self._response = OpenStack_Response() - self._producer = KafkaProducer('alarm_response') - alarm_id = values['alarm_id'] - - vim_uuid = 'test_id' - - auth_token = Common.get_auth_token(vim_uuid) - endpoint = Common.get_endpoint("alarming", vim_uuid) - - # If authenticated generate and send response message - if auth_token is not None and endpoint is not None: - url = "{}/v2/alarms/%s".format(endpoint) % alarm_id - - # Get the resource_id of the triggered alarm and the date - result = Common.perform_request( - url, auth_token, req_type="get") - alarm_details = json.loads(result.text) - gnocchi_rule = alarm_details['gnocchi_resources_threshold_rule'] - resource_id = gnocchi_rule['resource_id'] - a_date = "dd-mm-yyyy 00:00" - - # Process an alarm notification if resource_id is valid - if resource_id is not None: - # Try generate and send response - try: - resp_message = self._response.generate_response( - 'notify_alarm', a_id=alarm_id, - r_id=resource_id, - sev=values['severity'], date=a_date, - state=values['current'], vim_type="OpenStack") - self._producer.notify_alarm( - 'notify_alarm', resp_message) - except Exception: - pass - class TestNotifier(unittest.TestCase): """Test the NotifierHandler class for requests from aodh.""" @@ -159,8 +106,7 @@ class TestNotifier(unittest.TestCase): def setUp(self): """Setup tests.""" super(TestNotifier, self).setUp() - self.handler = NotifierHandler( - "mock_request", "mock_address", "mock_server") + self.handler = MockNotifierHandler() @mock.patch.object(NotifierHandler, "_set_headers") def test_do_GET(self, set_head): @@ -227,20 +173,21 @@ class TestNotifier(unittest.TestCase): @mock.patch.object(OpenStack_Response, "generate_response") @mock.patch.object(Common, "get_auth_token") @mock.patch.object(Common, "perform_request") - def test_notify_alarm_resp_call(self, perf_req, auth, response, endpoint, notify): + @mock.patch.object(DatabaseManager, "get_alarm") + def test_notify_alarm_resp_call(self, get_alarm, perf_req, auth, response, endpoint, notify): """Test notify_alarm tries to generate a response for SO.""" # Mock valid auth token and endpoint, valid response from aodh auth.return_value = "my_auth_token" endpoint.returm_value = "my_endpoint" perf_req.return_value = Response(valid_get_resp) + mock_alarm = Alarm() + get_alarm.return_value = mock_alarm self.handler.notify_alarm(json.loads(post_data)) notify.assert_called() - response.assert_called_with('notify_alarm', a_id="my_alarm_id", - r_id="my_resource_id", sev="critical", - date="dd-mm-yyyy 00:00", - state="current_state", - vim_type="OpenStack") + response.assert_called_with('notify_alarm', a_id='my_alarm_id', date=mock.ANY, metric_name=None, + ns_id=None, operation=None, sev='critical', state='current_state', + threshold_value=None, vdu_name=None, vnf_member_index=None) @mock.patch.object(Common, "get_endpoint") @mock.patch.object(KafkaProducer, "notify_alarm") @@ -266,15 +213,17 @@ class TestNotifier(unittest.TestCase): @mock.patch.object(OpenStack_Response, "generate_response") @mock.patch.object(Common, "get_auth_token") @mock.patch.object(Common, "perform_request") + @mock.patch.object(DatabaseManager, "get_alarm") def test_notify_alarm_valid_resp( - self, perf_req, auth, response, notify, endpoint): + self, get_alarm, perf_req, auth, response, notify, endpoint): """Test the notify_alarm function, sends response to the producer.""" # Generate return values for valid notify_alarm operation auth.return_value = "my_auth_token" endpoint.return_value = "my_endpoint" perf_req.return_value = Response(valid_get_resp) response.return_value = valid_notify_resp - + mock_alarm = Alarm() + get_alarm.return_value = mock_alarm self.handler.notify_alarm(json.loads(post_data)) notify.assert_called_with( -- 2.25.1