From 058d51d5f8ba1666242a12f52d99bdd74ec7f94f Mon Sep 17 00:00:00 2001 From: Benjamin Diaz Date: Tue, 20 Nov 2018 14:01:43 -0300 Subject: [PATCH] Adds support for VNF metric based alarming Adds collect_one method to juju collector, which is used by mon-evaluator to evaluate alarms. Signed-off-by: Benjamin Diaz Change-Id: I952280c3c8fa7496f0ea818b7a2ef7f3a091deea --- Jenkinsfile | 2 +- docker/Dockerfile | 1 + osm_mon/collector/collectors/base.py | 8 +-- osm_mon/collector/collectors/openstack.py | 19 +----- osm_mon/core/common_db.py | 8 +-- osm_mon/core/settings.py | 3 +- osm_mon/evaluator/evaluator.py | 80 +++++++++++------------ 7 files changed, 52 insertions(+), 69 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 635a5e6..6afef69 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ properties([ string(defaultValue: env.GERRIT_REFSPEC, description: '', name: 'GERRIT_REFSPEC'), string(defaultValue: env.GERRIT_PATCHSET_REVISION, description: '', name: 'GERRIT_PATCHSET_REVISION'), string(defaultValue: 'https://osm.etsi.org/gerrit', description: '', name: 'PROJECT_URL_PREFIX'), - booleanParam(defaultValue: true, description: '', name: 'TEST_INSTALL'), + booleanParam(defaultValue: false, description: '', name: 'TEST_INSTALL'), string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), ]) ]) diff --git a/docker/Dockerfile b/docker/Dockerfile index f843908..e150169 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -50,6 +50,7 @@ ENV OSMMON_VCA_USER admin ENV OSMMON_DATABASE_COMMONKEY changeme ENV OSMMON_COLLECTOR_INTERVAL 30 ENV OSMMON_EVALUATOR_INTERVAL 30 +ENV OSMMON_PROMETHEUS_URL http://prometheus:9090 EXPOSE 8662 8000 diff --git a/osm_mon/collector/collectors/base.py b/osm_mon/collector/collectors/base.py index a97903d..824e106 100644 --- a/osm_mon/collector/collectors/base.py +++ b/osm_mon/collector/collectors/base.py @@ -19,11 +19,11 @@ # For those usages not covered by the Apache License, Version 2.0 please # contact: bdiaz@whitestack.com or glavado@whitestack.com ## +from typing import List +from osm_mon.collector.metric import Metric -class BaseCollector: - def collect(self, vnfr: dict): - pass - def collect_one(self, nsr_id: str, vnf_member_index: int, vdur_name: str, metric_name: str): +class BaseCollector: + def collect(self, vnfr: dict) -> List[Metric]: pass diff --git a/osm_mon/collector/collectors/openstack.py b/osm_mon/collector/collectors/openstack.py index 5053902..395d607 100644 --- a/osm_mon/collector/collectors/openstack.py +++ b/osm_mon/collector/collectors/openstack.py @@ -31,9 +31,8 @@ from keystoneauth1.identity import v3 from osm_mon.collector.collectors.base_vim import BaseVimCollector from osm_mon.collector.metric import Metric -from osm_mon.core.common_db import CommonDbClient from osm_mon.core.auth import AuthManager -from osm_mon.core.exceptions import MetricNotFound +from osm_mon.core.common_db import CommonDbClient from osm_mon.core.settings import Config log = logging.getLogger(__name__) @@ -114,19 +113,3 @@ class OpenstackCollector(BaseVimCollector): log.debug("No metric found: %s", e) pass return metrics - - def collect_one(self, nsr_id: str, vnf_member_index: int, vdur_name: str, metric_name: str) -> Metric: - gnocchi_metric_name = METRIC_MAPPINGS[metric_name] - start_date = datetime.datetime.now() - datetime.timedelta(seconds=self.granularity) - resource_id = self._get_resource_uuid(nsr_id, vnf_member_index, vdur_name) - try: - metrics = self.gnocchi_client.metric.get_measures(gnocchi_metric_name, - start=start_date, - resource_id=resource_id, - granularity=self.granularity) - if len(metrics): - metric = Metric(nsr_id, vnf_member_index, vdur_name, metric_name, metrics[-1][2]) - return metric - except gnocchiclient.exceptions.NotFound as e: - log.debug("No metric found: %s", e) - raise MetricNotFound() diff --git a/osm_mon/core/common_db.py b/osm_mon/core/common_db.py index 9549fe4..beaac3d 100644 --- a/osm_mon/core/common_db.py +++ b/osm_mon/core/common_db.py @@ -61,13 +61,13 @@ class CommonDbClient: {"_id": nslcmop_id}) return nslcmop - def get_vdur(self, nsr_id, member_index, vdu_name): + def get_vdur(self, nsr_id, member_index, vdur_name): vnfr = self.get_vnfr(nsr_id, member_index) for vdur in vnfr['vdur']: - if vdur['name'] == vdu_name: + if vdur['name'] == vdur_name: return vdur - raise ValueError('vdur not found for nsr-id %s, member_index %s and vdu_name %s', nsr_id, member_index, - vdu_name) + raise ValueError('vdur not found for nsr-id %s, member_index %s and vdur_name %s', nsr_id, member_index, + vdur_name) def decrypt_vim_password(self, vim_password: str, schema_version: str, vim_id: str): return self.common_db.decrypt(vim_password, schema_version, vim_id) diff --git a/osm_mon/core/settings.py b/osm_mon/core/settings.py index 3b20c2b..680b58d 100644 --- a/osm_mon/core/settings.py +++ b/osm_mon/core/settings.py @@ -60,7 +60,7 @@ class Config(object): _configuration = [ CfgParam('BROKER_URI', "localhost:9092", six.text_type), - CfgParam('MONGO_URI', "mongo:27017", six.text_type), + CfgParam('MONGO_URI', "mongodb://mongo:27017", six.text_type), CfgParam('DATABASE', "sqlite:///mon_sqlite.db", six.text_type), CfgParam('OS_DEFAULT_GRANULARITY', 300, int), CfgParam('REQUEST_TIMEOUT', 10, int), @@ -72,6 +72,7 @@ class Config(object): CfgParam('OSMMON_VCA_SECRET', "secret", six.text_type), CfgParam('OSMMON_VCA_USER', "admin", six.text_type), CfgParam('OSMMON_DATABASE_COMMONKEY', "changeme", six.text_type), + CfgParam('OSMMON_PROMETHEUS_URL', "http://prometheus:9090", six.text_type), ] _config_dict = {cfg.key: cfg for cfg in _configuration} diff --git a/osm_mon/evaluator/evaluator.py b/osm_mon/evaluator/evaluator.py index b040198..7613e0b 100644 --- a/osm_mon/evaluator/evaluator.py +++ b/osm_mon/evaluator/evaluator.py @@ -25,10 +25,10 @@ import logging import multiprocessing import time +import requests from osm_common.dbbase import DbException -from osm_mon.collector.collector import VIM_COLLECTORS -from osm_mon.collector.collectors.juju import VCACollector +from osm_mon.collector.backends.prometheus import OSM_METRIC_PREFIX from osm_mon.core.common_db import CommonDbClient from osm_mon.core.database import DatabaseManager, Alarm from osm_mon.core.message_bus.producer import Producer @@ -46,44 +46,42 @@ class Evaluator: self.database_manager.create_tables() self.queue = multiprocessing.Queue() - def _evaluate_vim_metric(self, - nsr_id: str, - vnf_member_index: int, - vdur_name: str, - nfvi_metric_name: str, - vim_account_id: str, - alarm: Alarm): - vim_type = self.database_manager.get_vim_type(vim_account_id) - if vim_type in VIM_COLLECTORS: - collector = VIM_COLLECTORS[vim_type](vim_account_id) - metric = collector.collect_one(nsr_id, vnf_member_index, vdur_name, nfvi_metric_name) - if alarm.operation.upper() == 'GT': - if metric.value > alarm.threshold: - self.queue.put(alarm) - elif alarm.operation.upper() == 'LT': - if metric.value < alarm.threshold: - self.queue.put(alarm) - + def _evaluate_metric(self, + nsr_id: str, + vnf_member_index: int, + vdur_name: str, + metric_name: str, + alarm: Alarm): + log.debug("_evaluate_metric") + # TODO: Refactor to fit backend plugin model + cfg = Config.instance() + query_section = "query={0}{{ns_id=\"{1}\",vdu_name=\"{2}\",vnf_member_index=\"{3}\"}}".format( + OSM_METRIC_PREFIX + metric_name, nsr_id, vdur_name, vnf_member_index) + request_url = cfg.OSMMON_PROMETHEUS_URL + "/api/v1/query?" + query_section + log.info("Querying Prometheus: %s", request_url) + r = requests.get(request_url) + if r.status_code == 200: + json_response = r.json() + if json_response['status'] == 'success': + result = json_response['data']['result'] + if len(result): + metric_value = float(result[0]['value'][1]) + log.info("Metric value: %s", metric_value) + if alarm.operation.upper() == 'GT': + if metric_value > alarm.threshold: + self.queue.put(alarm) + elif alarm.operation.upper() == 'LT': + if metric_value < alarm.threshold: + self.queue.put(alarm) + else: + log.warning("No metric result for alarm %s", alarm.id) + else: + log.warning("Prometheus response is not success. Got status %s", json_response['status']) else: - log.debug("vimtype %s is not supported.", vim_type) - - def _evaluate_vca_metric(self, - nsr_id: str, - vnf_member_index: int, - vdur_name: str, - vnf_metric_name: str, - alarm: Alarm): - collector = VCACollector() - metric = collector.collect_one(nsr_id, vnf_member_index, vdur_name, vnf_metric_name) - if alarm.operation.upper() == 'GT': - if metric.value > alarm.threshold: - self.queue.put(alarm) - elif alarm.operation.upper() == 'LT': - if metric.value < alarm.threshold: - self.queue.put(alarm) + log.warning("Error contacting Prometheus. Got status code %s: %s", r.status_code, r.text) def evaluate_forever(self): - log.debug('collect_forever') + log.debug('evaluate_forever') cfg = Config.instance() while True: try: @@ -93,6 +91,7 @@ class Evaluator: log.exception("Error evaluating alarms") def evaluate(self): + log.debug('evaluate') processes = [] for alarm in Alarm.select(): try: @@ -118,19 +117,17 @@ class Evaluator: 'vdu-monitoring-param-ref'], vdu['monitoring-param'])) nfvi_metric = vdu_monitoring_param['nfvi-metric'] - vim_account_id = self.common_db.get_vim_account_id(nsr_id, vnf_member_index) - p = multiprocessing.Process(target=self._evaluate_vim_metric, + p = multiprocessing.Process(target=self._evaluate_metric, args=(nsr_id, vnf_member_index, vdur_name, nfvi_metric, - vim_account_id, alarm)) processes.append(p) p.start() if 'vdu-metric' in vnf_monitoring_param: vnf_metric_name = vnf_monitoring_param['vdu-metric']['vdu-metric-name-ref'] - p = multiprocessing.Process(target=self._evaluate_vca_metric, + p = multiprocessing.Process(target=self._evaluate_metric, args=(nsr_id, vnf_member_index, vdur_name, @@ -154,6 +151,7 @@ class Evaluator: p.start() def notify_alarm(self, alarm: Alarm): + log.debug("notify_alarm") response = ResponseBuilder() now = time.strftime("%d-%m-%Y") + " " + time.strftime("%X") # Generate and send response -- 2.25.1