X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_mon%2Fcollector%2Fvnf_collectors%2Fopenstack.py;h=a823b67b827ed8fd05522dd9ca553bba6f1ab09b;hb=refs%2Fchanges%2F56%2F11956%2F3;hp=525bd00d3686d9b236383f407e6b4fcf4db0daaf;hpb=8e4179facf22c8096992f0a83caeec9f2f4996c7;p=osm%2FMON.git diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index 525bd00..a823b67 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -31,6 +31,7 @@ from gnocchiclient.v1 import client as gnocchi_client from keystoneauth1.exceptions.catalog import EndpointNotFound from keystoneclient.v3 import client as keystone_client from neutronclient.v2_0 import client as neutron_client +from prometheus_api_client import PrometheusConnect as prometheus_client from osm_mon.collector.metric import Metric from osm_mon.collector.utils.openstack import OpenstackUtils @@ -55,6 +56,16 @@ METRIC_MAPPINGS = { "cpu_utilization": "cpu", } +# Metrics which have new names in Rocky and higher releases +METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES = { + "disk_read_ops": "disk.device.read.requests", + "disk_write_ops": "disk.device.write.requests", + "disk_read_bytes": "disk.device.read.bytes", + "disk_write_bytes": "disk.device.write.bytes", + "packets_received": "network.incoming.packets", + "packets_sent": "network.outgoing.packets" +} + METRIC_MULTIPLIERS = {"cpu": 0.0000001} METRIC_AGGREGATORS = {"cpu": "rate:mean"} @@ -66,19 +77,27 @@ INTERFACE_METRICS = [ "packets_sent", ] +INSTANCE_DISK = [ + "disk_read_ops", + "disk_write_ops", + "disk_read_bytes", + "disk_write_bytes", +] + class MetricType(Enum): INSTANCE = "instance" INTERFACE_ALL = "interface_all" INTERFACE_ONE = "interface_one" + INSTANCEDISK = 'instancedisk' class OpenstackCollector(BaseVimCollector): - def __init__(self, config: Config, vim_account_id: str): + def __init__(self, config: Config, vim_account_id: str, vim_session: object): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) vim_account = self.common_db.get_vim_account(vim_account_id) - self.backend = self._get_backend(vim_account) + self.backend = self._get_backend(vim_account, vim_session) def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client: sess = OpenstackUtils.get_session(vim_account) @@ -112,7 +131,11 @@ class OpenstackCollector(BaseVimCollector): if "monitoring-parameter" in vdu: for param in vdu["monitoring-parameter"]: metric_name = param["performance-metric"] - openstack_metric_name = METRIC_MAPPINGS[metric_name] + log.debug(f"Using an {type(self.backend)} as backend") + if type(self.backend) is PrometheusTSBDBackend: + openstack_metric_name = self.backend.map_metric(metric_name) + else: + openstack_metric_name = METRIC_MAPPINGS[metric_name] metric_type = self._get_metric_type(metric_name) try: resource_id = self._get_resource_uuid( @@ -137,6 +160,20 @@ class OpenstackCollector(BaseVimCollector): value = self.backend.collect_metric( metric_type, openstack_metric_name, resource_id ) + + if value is None and metric_name in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES and type(self.backend) is not PrometheusTSBDBackend: + # Reattempting metric collection with new metric names. + # Some metric names have changed in newer Openstack releases + log.info( + "Reattempting metric collection for type: %s and name: %s and resource_id %s", + metric_type, + metric_name, + resource_id + ) + openstack_metric_name = METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES[metric_name] + value = self.backend.collect_metric( + metric_type, openstack_metric_name, resource_id + ) if value is not None: log.info("value: %s", value) metric = VnfMetric( @@ -158,21 +195,32 @@ class OpenstackCollector(BaseVimCollector): log.info("Error in metric collection: %s" % e) return metrics - def _get_backend(self, vim_account: dict): + def _get_backend(self, vim_account: dict, vim_session: object): + if vim_account.get("prometheus-config"): + try: + tsbd = PrometheusTSBDBackend(vim_account) + log.debug("Using prometheustsbd backend to collect metric") + return tsbd + except Exception as e: + log.error(f"Can't create prometheus client, {e}") + return None try: - gnocchi = GnocchiBackend(vim_account) + gnocchi = GnocchiBackend(vim_account, vim_session) gnocchi.client.metric.list(limit=1) - log.info("Using gnocchi backend to collect metric") + log.debug("Using gnocchi backend to collect metric") return gnocchi except (HTTPException, EndpointNotFound): - ceilometer = CeilometerBackend(vim_account) + ceilometer = CeilometerBackend(vim_account, vim_session) ceilometer.client.capabilities.get() - log.info("Using ceilometer backend to collect metric") + log.debug("Using ceilometer backend to collect metric") return ceilometer def _get_metric_type(self, metric_name: str) -> MetricType: if metric_name not in INTERFACE_METRICS: - return MetricType.INSTANCE + if metric_name not in INSTANCE_DISK: + return MetricType.INSTANCE + else: + return MetricType.INSTANCEDISK else: return MetricType.INTERFACE_ALL @@ -184,18 +232,43 @@ class OpenstackBackend: pass -class GnocchiBackend(OpenstackBackend): +class PrometheusTSBDBackend(OpenstackBackend): def __init__(self, vim_account: dict): - self.client = self._build_gnocchi_client(vim_account) - self.neutron = self._build_neutron_client(vim_account) + self.cred = vim_account["prometheus-config"]["prometheus_cred"] + self.map = vim_account["prometheus-config"]["prometheus_map"] + self.client = self._build_prometheus_client(vim_account) - def _build_gnocchi_client(self, vim_account: dict) -> gnocchi_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return gnocchi_client.Client(session=sess) + def _build_prometheus_client(self, vim_account: dict) -> prometheus_client: + url = vim_account["prometheus-config"]["prometheus_url"] + return prometheus_client(url, disable_ssl = True) - def _build_neutron_client(self, vim_account: dict) -> neutron_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return neutron_client.Client(session=sess) + def collect_metric( + self, metric_type: MetricType, metric_name: str, resource_id: str + ): + metric = self.query_metric(metric_name, resource_id) + return metric["value"][1] if metric else None + + def map_metric(self, metric_name: str): + return self.map[metric_name] + + def query_metric(self, metric_name, resource_id = None): + metrics = self.client.get_current_metric_value(metric_name = metric_name) + if resource_id: + metric = next(filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)) + return metric + return metrics + + +class GnocchiBackend(OpenstackBackend): + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_gnocchi_client(vim_account, vim_session) + self.neutron = self._build_neutron_client(vim_account, vim_session) + + def _build_gnocchi_client(self, vim_account: dict, vim_session: object) -> gnocchi_client.Client: + return gnocchi_client.Client(session=vim_session) + + def _build_neutron_client(self, vim_account: dict, vim_session: object) -> neutron_client.Client: + return neutron_client.Client(session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str @@ -206,6 +279,9 @@ class GnocchiBackend(OpenstackBackend): elif metric_type == MetricType.INSTANCE: return self._collect_instance_metric(metric_name, resource_id) + elif metric_type == MetricType.INSTANCEDISK: + return self._collect_instance_disk_metric(metric_name, resource_id) + else: raise Exception("Unknown metric type %s" % metric_type.value) @@ -224,7 +300,6 @@ class GnocchiBackend(OpenstackBackend): if not total_measure: total_measure = 0.0 total_measure += measures[-1][2] - except (gnocchiclient.exceptions.NotFound, TypeError) as e: # Gnocchi in some Openstack versions raise TypeError instead of NotFound log.debug( @@ -235,6 +310,25 @@ class GnocchiBackend(OpenstackBackend): ) return total_measure + def _collect_instance_disk_metric(self, openstack_metric_name, resource_id): + value = None + instances = self.client.resource.search( + resource_type='instance_disk', + query={'=': {'instance_id': resource_id}}, + ) + for instance in instances: + try: + measures = self.client.metric.get_measures( + openstack_metric_name, resource_id=instance['id'], limit=1 + ) + if measures: + value = measures[-1][2] + + except gnocchiclient.exceptions.NotFound as e: + log.debug("No metric %s found for instance disk %s: %s", openstack_metric_name, + instance['id'], e) + return value + def _collect_instance_metric(self, openstack_metric_name, resource_id): value = None try: @@ -299,12 +393,11 @@ class GnocchiBackend(OpenstackBackend): class CeilometerBackend(OpenstackBackend): - def __init__(self, vim_account: dict): - self.client = self._build_ceilometer_client(vim_account) + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_ceilometer_client(vim_account, vim_session) - def _build_ceilometer_client(self, vim_account: dict) -> ceilometer_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return ceilometer_client.Client("2", session=sess) + def _build_ceilometer_client(self, vim_account: dict, vim_session: object) -> ceilometer_client.Client: + return ceilometer_client.Client("2", session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str