From 335cb85d4b71fb6a5d12d0b08636047686a2a1ba Mon Sep 17 00:00:00 2001 From: vegall Date: Wed, 27 Apr 2022 14:42:48 +0000 Subject: [PATCH] Feature 10920: Monitoring of NFVI-leve VNF metrics form Prometheus TSDB Change-Id: Iae5961ac19c8880c489a0c7930855310a7931ae5 Signed-off-by: vegall --- osm_mon/collector/vnf_collectors/openstack.py | 48 +++++++++++++++++-- osm_mon/dashboarder/service.py | 9 ++-- requirements.txt | 1 + 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index aba4788..a823b67 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -31,6 +31,7 @@ from gnocchiclient.v1 import client as gnocchi_client from keystoneauth1.exceptions.catalog import EndpointNotFound from keystoneclient.v3 import client as keystone_client from neutronclient.v2_0 import client as neutron_client +from prometheus_api_client import PrometheusConnect as prometheus_client from osm_mon.collector.metric import Metric from osm_mon.collector.utils.openstack import OpenstackUtils @@ -130,7 +131,11 @@ class OpenstackCollector(BaseVimCollector): if "monitoring-parameter" in vdu: for param in vdu["monitoring-parameter"]: metric_name = param["performance-metric"] - openstack_metric_name = METRIC_MAPPINGS[metric_name] + log.debug(f"Using an {type(self.backend)} as backend") + if type(self.backend) is PrometheusTSBDBackend: + openstack_metric_name = self.backend.map_metric(metric_name) + else: + openstack_metric_name = METRIC_MAPPINGS[metric_name] metric_type = self._get_metric_type(metric_name) try: resource_id = self._get_resource_uuid( @@ -156,7 +161,7 @@ class OpenstackCollector(BaseVimCollector): metric_type, openstack_metric_name, resource_id ) - if value is None and metric_name in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES: + if value is None and metric_name in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES and type(self.backend) is not PrometheusTSBDBackend: # Reattempting metric collection with new metric names. # Some metric names have changed in newer Openstack releases log.info( @@ -191,15 +196,23 @@ class OpenstackCollector(BaseVimCollector): return metrics def _get_backend(self, vim_account: dict, vim_session: object): + if vim_account.get("prometheus-config"): + try: + tsbd = PrometheusTSBDBackend(vim_account) + log.debug("Using prometheustsbd backend to collect metric") + return tsbd + except Exception as e: + log.error(f"Can't create prometheus client, {e}") + return None try: gnocchi = GnocchiBackend(vim_account, vim_session) gnocchi.client.metric.list(limit=1) - log.info("Using gnocchi backend to collect metric") + log.debug("Using gnocchi backend to collect metric") return gnocchi except (HTTPException, EndpointNotFound): ceilometer = CeilometerBackend(vim_account, vim_session) ceilometer.client.capabilities.get() - log.info("Using ceilometer backend to collect metric") + log.debug("Using ceilometer backend to collect metric") return ceilometer def _get_metric_type(self, metric_name: str) -> MetricType: @@ -219,6 +232,33 @@ class OpenstackBackend: pass +class PrometheusTSBDBackend(OpenstackBackend): + def __init__(self, vim_account: dict): + self.cred = vim_account["prometheus-config"]["prometheus_cred"] + self.map = vim_account["prometheus-config"]["prometheus_map"] + self.client = self._build_prometheus_client(vim_account) + + def _build_prometheus_client(self, vim_account: dict) -> prometheus_client: + url = vim_account["prometheus-config"]["prometheus_url"] + return prometheus_client(url, disable_ssl = True) + + def collect_metric( + self, metric_type: MetricType, metric_name: str, resource_id: str + ): + metric = self.query_metric(metric_name, resource_id) + return metric["value"][1] if metric else None + + def map_metric(self, metric_name: str): + return self.map[metric_name] + + def query_metric(self, metric_name, resource_id = None): + metrics = self.client.get_current_metric_value(metric_name = metric_name) + if resource_id: + metric = next(filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)) + return metric + return metrics + + class GnocchiBackend(OpenstackBackend): def __init__(self, vim_account: dict, vim_session: object): self.client = self._build_gnocchi_client(vim_account, vim_session) diff --git a/osm_mon/dashboarder/service.py b/osm_mon/dashboarder/service.py index a0a80a7..c826777 100644 --- a/osm_mon/dashboarder/service.py +++ b/osm_mon/dashboarder/service.py @@ -146,9 +146,12 @@ class DashboarderService: vnf_profile["vnfd-id"], create_filter_from_nsr(nsr) ) # If there are metrics, create dashboard (if exists) - vdu_found = find_in_list( - vnfd["vdu"], lambda a_vdu: "monitoring-parameter" in a_vdu - ) + if vnfd.get("vdu"): + vdu_found = find_in_list( + vnfd.get("vdu"), lambda a_vdu: "monitoring-parameter" in a_vdu + ) + else: + vdu_found = None if vdu_found: if nsr_id not in dashboard_uids: nsr_name = nsr["name"] diff --git a/requirements.txt b/requirements.txt index c7449b2..34683e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -189,6 +189,7 @@ prettytable==0.7.2 # python-novaclient prometheus-client==0.12.0 # via -r requirements.in +prometheus-api-client==0.5.0 pycparser==2.21 # via cffi pygments==2.11.2 -- 2.25.1