Feature 10920: Monitoring of NFVI-leve VNF metrics form Prometheus TSDB

Change-Id: Iae5961ac19c8880c489a0c7930855310a7931ae5
Signed-off-by: vegall <lvega@whitestack.com>

diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py
index aba4788..a823b67 100644 (file)
--- a/osm_mon/collector/vnf_collectors/openstack.py
+++ b/osm_mon/collector/vnf_collectors/openstack.py
@@ -31,6 +31,7 @@ from gnocchiclient.v1 import client as gnocchi_client
 from keystoneauth1.exceptions.catalog import EndpointNotFound
 from keystoneclient.v3 import client as keystone_client
 from neutronclient.v2_0 import client as neutron_client
+from prometheus_api_client import PrometheusConnect as prometheus_client
 
 from osm_mon.collector.metric import Metric
 from osm_mon.collector.utils.openstack import OpenstackUtils
@@ -130,7 +131,11 @@ class OpenstackCollector(BaseVimCollector):
             if "monitoring-parameter" in vdu:
                 for param in vdu["monitoring-parameter"]:
                     metric_name = param["performance-metric"]
-                    openstack_metric_name = METRIC_MAPPINGS[metric_name]
+                    log.debug(f"Using an {type(self.backend)} as backend")
+                    if type(self.backend) is PrometheusTSBDBackend:
+                        openstack_metric_name = self.backend.map_metric(metric_name)
+                    else:
+                        openstack_metric_name = METRIC_MAPPINGS[metric_name]
                     metric_type = self._get_metric_type(metric_name)
                     try:
                         resource_id = self._get_resource_uuid(
@@ -156,7 +161,7 @@ class OpenstackCollector(BaseVimCollector):
                             metric_type, openstack_metric_name, resource_id
                         )
 
-                        if value is None and metric_name in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES:
+                        if value is None and metric_name in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES and type(self.backend) is not PrometheusTSBDBackend:
                             # Reattempting metric collection with new metric names.
                             # Some metric names have changed in newer Openstack releases
                             log.info(
@@ -191,15 +196,23 @@ class OpenstackCollector(BaseVimCollector):
         return metrics
 
     def _get_backend(self, vim_account: dict, vim_session: object):
+        if vim_account.get("prometheus-config"):
+            try:
+                tsbd = PrometheusTSBDBackend(vim_account)
+                log.debug("Using prometheustsbd backend to collect metric")
+                return tsbd
+            except Exception as e:
+                log.error(f"Can't create prometheus client, {e}")
+                return None
         try:
             gnocchi = GnocchiBackend(vim_account, vim_session)
             gnocchi.client.metric.list(limit=1)
-            log.info("Using gnocchi backend to collect metric")
+            log.debug("Using gnocchi backend to collect metric")
             return gnocchi
         except (HTTPException, EndpointNotFound):
             ceilometer = CeilometerBackend(vim_account, vim_session)
             ceilometer.client.capabilities.get()
-            log.info("Using ceilometer backend to collect metric")
+            log.debug("Using ceilometer backend to collect metric")
             return ceilometer
 
     def _get_metric_type(self, metric_name: str) -> MetricType:
@@ -219,6 +232,33 @@ class OpenstackBackend:
         pass
 
 
+class PrometheusTSBDBackend(OpenstackBackend):
+    def __init__(self, vim_account: dict):
+        self.cred = vim_account["prometheus-config"]["prometheus_cred"]
+        self.map = vim_account["prometheus-config"]["prometheus_map"]
+        self.client = self._build_prometheus_client(vim_account)
+
+    def _build_prometheus_client(self, vim_account: dict) -> prometheus_client:
+        url = vim_account["prometheus-config"]["prometheus_url"]
+        return prometheus_client(url, disable_ssl = True)
+
+    def collect_metric(
+        self, metric_type: MetricType, metric_name: str, resource_id: str
+        ):
+        metric = self.query_metric(metric_name, resource_id)
+        return metric["value"][1] if metric else None
+
+    def map_metric(self, metric_name: str):
+        return self.map[metric_name]
+
+    def query_metric(self, metric_name, resource_id = None):
+        metrics = self.client.get_current_metric_value(metric_name = metric_name)
+        if resource_id:
+            metric = next(filter(lambda x: resource_id in x["metric"]["resource_id"], metrics))
+            return metric
+        return metrics
+
+
 class GnocchiBackend(OpenstackBackend):
     def __init__(self, vim_account: dict, vim_session: object):
         self.client = self._build_gnocchi_client(vim_account, vim_session)

diff --git a/osm_mon/dashboarder/service.py b/osm_mon/dashboarder/service.py
index a0a80a7..c826777 100644 (file)
--- a/osm_mon/dashboarder/service.py
+++ b/osm_mon/dashboarder/service.py
@@ -146,9 +146,12 @@ class DashboarderService:
                         vnf_profile["vnfd-id"], create_filter_from_nsr(nsr)
                     )
                     # If there are metrics, create dashboard (if exists)
-                    vdu_found = find_in_list(
-                        vnfd["vdu"], lambda a_vdu: "monitoring-parameter" in a_vdu
-                    )
+                    if vnfd.get("vdu"):
+                        vdu_found = find_in_list(
+                            vnfd.get("vdu"), lambda a_vdu: "monitoring-parameter" in a_vdu
+                        )
+                    else:
+                        vdu_found = None
                     if vdu_found:
                         if nsr_id not in dashboard_uids:
                             nsr_name = nsr["name"]

diff --git a/requirements.txt b/requirements.txt
index c7449b2..34683e6 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -189,6 +189,7 @@ prettytable==0.7.2
     #   python-novaclient
 prometheus-client==0.12.0
     # via -r requirements.in
+prometheus-api-client==0.5.0
 pycparser==2.21
     # via cffi
 pygments==2.11.2