From b3173221c25ae915eaca041ebd045f6148cc447d Mon Sep 17 00:00:00 2001 From: Daniel Gomes Date: Fri, 10 Feb 2023 17:39:56 +0000 Subject: [PATCH] Bug 2215 fixed - created an attribute on the Alarm class, extra_labels - create a method on common_db to include new extra_labels - changed the _get_metric_value_from_response method of Prometheus Evaluator to return both the metric value and labels Change-Id: I22e0fd8e95de28560e3a3216622c43c8861137ef Signed-off-by: Daniel Gomes --- osm_mon/core/common_db.py | 7 +- osm_mon/core/models.py | 4 + osm_mon/core/response.py | 1 + osm_mon/evaluator/backends/base.py | 5 +- osm_mon/evaluator/backends/prometheus.py | 20 +++-- osm_mon/evaluator/evaluator.py | 4 +- osm_mon/evaluator/service.py | 86 +++++++++++-------- .../tests/unit/core/test_common_db_client.py | 2 + .../unit/evaluator/test_evaluator_service.py | 27 ++++-- 9 files changed, 102 insertions(+), 54 deletions(-) diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py index a9fcfbe..66f81fe 100644 --- a/osm_mon/core/common_db.py +++ b/osm_mon/core/common_db.py @@ -17,7 +17,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## @@ -190,6 +189,12 @@ class CommonDbClient: ) return modified_count + def update_alarm_extra_labels(self, alarm_labels: dict, uuid): + modified_count = self.common_db.set_one( + "alarms", {"uuid": uuid}, {"extra_labels": alarm_labels} + ) + return modified_count + def get_alarm_by_uuid(self, uuid: str): return self.common_db.get_one("alarms", {"uuid": uuid}) diff --git a/osm_mon/core/models.py b/osm_mon/core/models.py index 48fe8c4..d076e5e 100644 --- a/osm_mon/core/models.py +++ b/osm_mon/core/models.py @@ -35,6 +35,7 @@ class Alarm: action: str = None, tags: dict = {}, alarm_status: str = "ok", + extra_labels: dict = {}, ): self.uuid = str(uuid.uuid4()) self.name = name @@ -46,6 +47,7 @@ class Alarm: self.action = action self.tags = tags self.alarm_status = alarm_status + self.extra_labels = extra_labels def to_dict(self) -> dict: alarm = { @@ -58,6 +60,7 @@ class Alarm: "tags": self.tags, "operation": self.operation, "alarm_status": self.alarm_status, + "extra_labels": self.extra_labels, } return alarm @@ -73,4 +76,5 @@ class Alarm: alarm.tags = data.get("tags") alarm.operation = data.get("operation") alarm.alarm_status = data.get("alarm_status") + alarm.extra_labels = data.get("extra_labels") return alarm diff --git a/osm_mon/core/response.py b/osm_mon/core/response.py index 0879fcb..91f594b 100644 --- a/osm_mon/core/response.py +++ b/osm_mon/core/response.py @@ -88,6 +88,7 @@ class ResponseBuilder(object): "status": kwargs["status"], "start_date": kwargs["date"], "tags": kwargs["tags"], + "extra_labels": kwargs["extra_labels"], }, } return notify_alarm_resp diff --git a/osm_mon/evaluator/backends/base.py b/osm_mon/evaluator/backends/base.py index 5ef1598..30793c1 100644 --- a/osm_mon/evaluator/backends/base.py +++ b/osm_mon/evaluator/backends/base.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright 2018 Whitestack, LLC # ************************************************************* @@ -15,7 +17,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## @@ -27,5 +28,5 @@ class BaseBackend: def __init__(self, config: Config): pass - def get_metric_value(self, metric_name: str, tags: dict): + def get_metric_data(self, metric_name: str, tags: dict): pass diff --git a/osm_mon/evaluator/backends/prometheus.py b/osm_mon/evaluator/backends/prometheus.py index ad4919f..c5b935e 100644 --- a/osm_mon/evaluator/backends/prometheus.py +++ b/osm_mon/evaluator/backends/prometheus.py @@ -22,7 +22,7 @@ ## import base64 import logging -from typing import Dict +from typing import Dict, List import requests @@ -33,13 +33,15 @@ log = logging.getLogger(__name__) OSM_METRIC_PREFIX = "osm_" +DEFAULT_QUERY_METRICS = ["ns_id", "vnf_member_index", "vdu_name"] + class PrometheusBackend(BaseBackend): def __init__(self, config: Config): super().__init__(config) self.conf = config - def get_metric_value(self, metric_name: str, tags: dict): + def get_metric_data(self, metric_name: str, tags: dict): query = self._build_query(metric_name, tags) request_url = self._build_url(query) request_headers = self._build_headers() @@ -55,7 +57,7 @@ class PrometheusBackend(BaseBackend): if r.status_code == 200: json_response = r.json() if json_response["status"] == "success": - return self._get_metric_value_from_response(json_response) + return self._get_metric_data_from_response(json_response) else: log.warning( "Prometheus response is not success. Got status %s", @@ -91,11 +93,15 @@ class PrometheusBackend(BaseBackend): headers["Authorization"] = f"Basic {token}" return headers - def _get_metric_value_from_response(self, json_response): + def _get_metric_data_from_response(self, json_response) -> List[Dict[str, str]]: result = json_response["data"]["result"] + metrics_data = [] if len(result): - metric_value = float(result[0]["value"][1]) - log.info("Metric value: %s", metric_value) - return metric_value + for metric in result: + metrics_labels = metric["metric"] + metric_value = float(metric["value"][1]) + log.info("Metric value: %s", metric_value) + metrics_data.append({"labels": metrics_labels, "value": metric_value}) + return metrics_data else: return None diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py index 61b788a..44b0af6 100644 --- a/osm_mon/evaluator/evaluator.py +++ b/osm_mon/evaluator/evaluator.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Copyright 2018 Whitestack, LLC # ************************************************************* @@ -71,6 +69,7 @@ class Evaluator: ) evaluator_service = EvaluatorService(self.conf) evaluator_service.update_alarm_status(status.value, alarm.uuid) + evaluator_service.update_alarm_extra_labels(alarm.extra_labels, alarm.uuid) return def _build_alarm_response(self, alarm: Alarm, status: AlarmStatus): @@ -90,4 +89,5 @@ class Evaluator: status=status.value, date=now, tags=tags, + extra_tabels=alarm.extra_labels, ) diff --git a/osm_mon/evaluator/service.py b/osm_mon/evaluator/service.py index ae6191b..e97586c 100644 --- a/osm_mon/evaluator/service.py +++ b/osm_mon/evaluator/service.py @@ -48,10 +48,10 @@ class EvaluatorService: self.common_db = CommonDbClient(self.conf) self.queue = multiprocessing.Queue() - def _get_metric_value(self, metric_name: str, tags: dict): + def _get_metric_data(self, metric_name: str, tags: dict): return BACKENDS[self.conf.get("evaluator", "backend")]( self.conf - ).get_metric_value(metric_name, tags) + ).get_metric_data(metric_name, tags) def _evaluate_metric(self, alarm: Alarm): """Method to evaluate a metric value comparing it against an alarm threshold. @@ -61,38 +61,52 @@ class EvaluatorService: """ log.debug("_evaluate_metric") - metric_value = self._get_metric_value(alarm.metric, alarm.tags) - if alarm.alarm_status.upper() != AlarmStatus.DISABLED.value.upper(): - if metric_value is None: - log.warning("No metric result for alarm %s", alarm.uuid) - self.queue.put((alarm, AlarmStatus.INSUFFICIENT)) - else: - if ( - (alarm.operation.upper() == "GT" and metric_value > alarm.threshold) - or ( - alarm.operation.upper() == "LT" - and metric_value < alarm.threshold - ) - or ( - alarm.operation.upper() == "GE" - and metric_value >= alarm.threshold - ) - or ( - alarm.operation.upper() == "LE" - and metric_value <= alarm.threshold - ) - or ( - alarm.operation.upper() == "EQ" - and metric_value == alarm.threshold - ) - or ( - alarm.operation.upper() == "NE" - and metric_value != alarm.threshold - ) - ): - self.queue.put((alarm, AlarmStatus.ALARM)) - elif alarm.operation.upper() in ("GT", "LT", "GE", "LE", "EQ", "NE"): - self.queue.put((alarm, AlarmStatus.OK)) + metric_data = self._get_metric_data(alarm.metric, alarm.tags) + if metric_data is None: + log.warning("No metric result for alarm %s", alarm.uuid) + self.queue.put((alarm, AlarmStatus.INSUFFICIENT)) + else: + for metric in metric_data: + metric_value = metric["value"] + metric_labels = metric["labels"] + alarm.extra_labels.update(metric_labels) + if alarm.alarm_status.upper() != AlarmStatus.DISABLED.value.upper(): + if ( + ( + alarm.operation.upper() == "GT" + and metric_value > alarm.threshold + ) + or ( + alarm.operation.upper() == "LT" + and metric_value < alarm.threshold + ) + or ( + alarm.operation.upper() == "GE" + and metric_value >= alarm.threshold + ) + or ( + alarm.operation.upper() == "LE" + and metric_value <= alarm.threshold + ) + or ( + alarm.operation.upper() == "EQ" + and metric_value == alarm.threshold + ) + or ( + alarm.operation.upper() == "NE" + and metric_value != alarm.threshold + ) + ): + self.queue.put((alarm, AlarmStatus.ALARM)) + elif alarm.operation.upper() in ( + "GT", + "LT", + "GE", + "LE", + "EQ", + "NE", + ): + self.queue.put((alarm, AlarmStatus.OK)) def update_alarm_status(self, alarm_state, uuid): alarm_data = self.common_db.get_alarm_by_uuid(uuid) @@ -100,6 +114,10 @@ class EvaluatorService: self.common_db.update_alarm_status(alarm_state, uuid) return + def update_alarm_extra_labels(self, alarm_labels, uuid): + self.common_db.update_alarm_extra_labels(alarm_labels, uuid) + return + def evaluate_alarms(self) -> List[Tuple[Alarm, AlarmStatus]]: log.debug("evaluate_alarms") processes = [] diff --git a/osm_mon/tests/unit/core/test_common_db_client.py b/osm_mon/tests/unit/core/test_common_db_client.py index 3ce117a..7f0768d 100644 --- a/osm_mon/tests/unit/core/test_common_db_client.py +++ b/osm_mon/tests/unit/core/test_common_db_client.py @@ -20,6 +20,7 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## + import unittest from unittest import mock @@ -211,6 +212,7 @@ class CommonDbClientTest(unittest.TestCase): "operation": "operation", "uuid": "1", "alarm_status": "ok", + "extra_labels": {}, }, ) diff --git a/osm_mon/tests/unit/evaluator/test_evaluator_service.py b/osm_mon/tests/unit/evaluator/test_evaluator_service.py index 15af5b6..54368dd 100644 --- a/osm_mon/tests/unit/evaluator/test_evaluator_service.py +++ b/osm_mon/tests/unit/evaluator/test_evaluator_service.py @@ -20,6 +20,7 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## + from unittest import TestCase, mock from osm_mon.core.common_db import CommonDbClient @@ -132,13 +133,23 @@ class EvaluatorTest(TestCase): super().setUp() self.config = Config() - @mock.patch.object(EvaluatorService, "_get_metric_value") - def test_evaluate_metric(self, get_metric_value): + @mock.patch.object(EvaluatorService, "_get_metric_data") + def test_evaluate_metric(self, get_metric_data): mock_alarm = mock.Mock() mock_alarm.operation = "gt" mock_alarm.threshold = 50.0 mock_alarm.metric = "metric_name" - get_metric_value.return_value = 100.0 + get_metric_data.return_value = [ + { + "labels": { + "vdu_name": "cirros_vnfd-VM", + "ns_id": "87776f33-b67c-417a-8119-cb08e4098951", + "vnf_member_index": "1", + "extra_label": "run_time_added_label", + }, + "value": 100.0, + } + ] service = EvaluatorService(self.config) service.queue = mock.Mock() @@ -151,7 +162,7 @@ class EvaluatorTest(TestCase): service.queue.put.assert_called_with((mock_alarm, AlarmStatus.OK)) service.queue.reset_mock() - get_metric_value.return_value = None + get_metric_data.return_value = None service._evaluate_metric(mock_alarm) service.queue.put.assert_called_with((mock_alarm, AlarmStatus.INSUFFICIENT)) @@ -176,10 +187,10 @@ class EvaluatorTest(TestCase): process.assert_called_with(target=evaluate_metric, args=(mock_alarm,)) - @mock.patch.object(PrometheusBackend, "get_metric_value") - def test_get_metric_value_prometheus(self, get_metric_value): + @mock.patch.object(PrometheusBackend, "get_metric_data") + def test_get_metric_data_prometheus(self, get_metric_data): self.config.set("evaluator", "backend", "prometheus") evaluator = EvaluatorService(self.config) - evaluator._get_metric_value("test", {}) + evaluator._get_metric_data("test", {}) - get_metric_value.assert_called_with("test", {}) + get_metric_data.assert_called_with("test", {}) -- 2.25.1