X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_mon%2Fcollector%2Fvnf_collectors%2Fopenstack.py;h=6aedf88377438567973f0b081eb1c017abddbc23;hb=HEAD;hp=525bd00d3686d9b236383f407e6b4fcf4db0daaf;hpb=8e4179facf22c8096992f0a83caeec9f2f4996c7;p=osm%2FMON.git diff --git a/osm_mon/collector/vnf_collectors/openstack.py b/osm_mon/collector/vnf_collectors/openstack.py index 525bd00..6aedf88 100644 --- a/osm_mon/collector/vnf_collectors/openstack.py +++ b/osm_mon/collector/vnf_collectors/openstack.py @@ -31,6 +31,7 @@ from gnocchiclient.v1 import client as gnocchi_client from keystoneauth1.exceptions.catalog import EndpointNotFound from keystoneclient.v3 import client as keystone_client from neutronclient.v2_0 import client as neutron_client +from prometheus_api_client import PrometheusConnect as prometheus_client from osm_mon.collector.metric import Metric from osm_mon.collector.utils.openstack import OpenstackUtils @@ -55,6 +56,29 @@ METRIC_MAPPINGS = { "cpu_utilization": "cpu", } +METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = { + "cpu_utilization": "cpu", + "average_memory_utilization": "memory_usage", + "disk_read_ops": "disk_device_read_requests", + "disk_write_ops": "disk_device_write_requests", + "disk_read_bytes": "disk_device_read_bytes", + "disk_write_bytes": "disk_device_write_bytes", + "packets_in_dropped": "network_incoming_packets_drop", + "packets_out_dropped": "network_outgoing_packets_drop", + "packets_received": "network_incoming_packets", + "packets_sent": "network_outgoing_packets", +} + +# Metrics which have new names in Rocky and higher releases +METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES = { + "disk_read_ops": "disk.device.read.requests", + "disk_write_ops": "disk.device.write.requests", + "disk_read_bytes": "disk.device.read.bytes", + "disk_write_bytes": "disk.device.write.bytes", + "packets_received": "network.incoming.packets", + "packets_sent": "network.outgoing.packets", +} + METRIC_MULTIPLIERS = {"cpu": 0.0000001} METRIC_AGGREGATORS = {"cpu": "rate:mean"} @@ -66,19 +90,27 @@ INTERFACE_METRICS = [ "packets_sent", ] +INSTANCE_DISK = [ + "disk_read_ops", + "disk_write_ops", + "disk_read_bytes", + "disk_write_bytes", +] + class MetricType(Enum): INSTANCE = "instance" INTERFACE_ALL = "interface_all" INTERFACE_ONE = "interface_one" + INSTANCEDISK = "instancedisk" class OpenstackCollector(BaseVimCollector): - def __init__(self, config: Config, vim_account_id: str): + def __init__(self, config: Config, vim_account_id: str, vim_session: object): super().__init__(config, vim_account_id) self.common_db = CommonDbClient(config) vim_account = self.common_db.get_vim_account(vim_account_id) - self.backend = self._get_backend(vim_account) + self.backend = self._get_backend(vim_account, vim_session) def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client: sess = OpenstackUtils.get_session(vim_account) @@ -112,7 +144,14 @@ class OpenstackCollector(BaseVimCollector): if "monitoring-parameter" in vdu: for param in vdu["monitoring-parameter"]: metric_name = param["performance-metric"] - openstack_metric_name = METRIC_MAPPINGS[metric_name] + log.info(f"Using an {type(self.backend)} as backend") + if type(self.backend) is PrometheusTSBDBackend: + openstack_metric_name = self.backend.map_metric(metric_name) + else: + try: + openstack_metric_name = METRIC_MAPPINGS[metric_name] + except KeyError: + continue metric_type = self._get_metric_type(metric_name) try: resource_id = self._get_resource_uuid( @@ -137,6 +176,29 @@ class OpenstackCollector(BaseVimCollector): value = self.backend.collect_metric( metric_type, openstack_metric_name, resource_id ) + + if ( + value is None + and metric_name + in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES + and type(self.backend) is not PrometheusTSBDBackend + ): + # Reattempting metric collection with new metric names. + # Some metric names have changed in newer Openstack releases + log.info( + "Reattempting metric collection for type: %s and name: %s and resource_id %s", + metric_type, + metric_name, + resource_id, + ) + openstack_metric_name = ( + METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES[ + metric_name + ] + ) + value = self.backend.collect_metric( + metric_type, openstack_metric_name, resource_id + ) if value is not None: log.info("value: %s", value) metric = VnfMetric( @@ -158,21 +220,32 @@ class OpenstackCollector(BaseVimCollector): log.info("Error in metric collection: %s" % e) return metrics - def _get_backend(self, vim_account: dict): + def _get_backend(self, vim_account: dict, vim_session: object): + if vim_account.get("prometheus-config"): + try: + tsbd = PrometheusTSBDBackend(vim_account) + log.debug("Using prometheustsbd backend to collect metric") + return tsbd + except Exception as e: + log.error(f"Can't create prometheus client, {e}") + return None try: - gnocchi = GnocchiBackend(vim_account) + gnocchi = GnocchiBackend(vim_account, vim_session) gnocchi.client.metric.list(limit=1) - log.info("Using gnocchi backend to collect metric") + log.debug("Using gnocchi backend to collect metric") return gnocchi except (HTTPException, EndpointNotFound): - ceilometer = CeilometerBackend(vim_account) + ceilometer = CeilometerBackend(vim_account, vim_session) ceilometer.client.capabilities.get() - log.info("Using ceilometer backend to collect metric") + log.debug("Using ceilometer backend to collect metric") return ceilometer def _get_metric_type(self, metric_name: str) -> MetricType: if metric_name not in INTERFACE_METRICS: - return MetricType.INSTANCE + if metric_name not in INSTANCE_DISK: + return MetricType.INSTANCE + else: + return MetricType.INSTANCEDISK else: return MetricType.INTERFACE_ALL @@ -184,18 +257,56 @@ class OpenstackBackend: pass -class GnocchiBackend(OpenstackBackend): +class PrometheusTSBDBackend(OpenstackBackend): def __init__(self, vim_account: dict): - self.client = self._build_gnocchi_client(vim_account) - self.neutron = self._build_neutron_client(vim_account) + self.map = self._build_map(vim_account) + self.cred = vim_account["prometheus-config"].get("prometheus-cred") + self.client = self._build_prometheus_client( + vim_account["prometheus-config"]["prometheus-url"] + ) - def _build_gnocchi_client(self, vim_account: dict) -> gnocchi_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return gnocchi_client.Client(session=sess) + def _build_prometheus_client(self, url: str) -> prometheus_client: + return prometheus_client(url, disable_ssl=True) - def _build_neutron_client(self, vim_account: dict) -> neutron_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return neutron_client.Client(session=sess) + def _build_map(self, vim_account: dict) -> dict: + custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD + if "prometheus-map" in vim_account["prometheus-config"]: + custom_map.update(vim_account["prometheus-config"]["prometheus-map"]) + return custom_map + + def collect_metric( + self, metric_type: MetricType, metric_name: str, resource_id: str + ): + metric = self.query_metric(metric_name, resource_id) + return metric["value"][1] if metric else None + + def map_metric(self, metric_name: str): + return self.map[metric_name] + + def query_metric(self, metric_name, resource_id=None): + metrics = self.client.get_current_metric_value(metric_name=metric_name) + if resource_id: + metric = next( + filter(lambda x: resource_id in x["metric"]["resource_id"], metrics) + ) + return metric + return metrics + + +class GnocchiBackend(OpenstackBackend): + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_gnocchi_client(vim_account, vim_session) + self.neutron = self._build_neutron_client(vim_account, vim_session) + + def _build_gnocchi_client( + self, vim_account: dict, vim_session: object + ) -> gnocchi_client.Client: + return gnocchi_client.Client(session=vim_session) + + def _build_neutron_client( + self, vim_account: dict, vim_session: object + ) -> neutron_client.Client: + return neutron_client.Client(session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str @@ -206,6 +317,9 @@ class GnocchiBackend(OpenstackBackend): elif metric_type == MetricType.INSTANCE: return self._collect_instance_metric(metric_name, resource_id) + elif metric_type == MetricType.INSTANCEDISK: + return self._collect_instance_disk_metric(metric_name, resource_id) + else: raise Exception("Unknown metric type %s" % metric_type.value) @@ -224,7 +338,6 @@ class GnocchiBackend(OpenstackBackend): if not total_measure: total_measure = 0.0 total_measure += measures[-1][2] - except (gnocchiclient.exceptions.NotFound, TypeError) as e: # Gnocchi in some Openstack versions raise TypeError instead of NotFound log.debug( @@ -235,6 +348,29 @@ class GnocchiBackend(OpenstackBackend): ) return total_measure + def _collect_instance_disk_metric(self, openstack_metric_name, resource_id): + value = None + instances = self.client.resource.search( + resource_type="instance_disk", + query={"=": {"instance_id": resource_id}}, + ) + for instance in instances: + try: + measures = self.client.metric.get_measures( + openstack_metric_name, resource_id=instance["id"], limit=1 + ) + if measures: + value = measures[-1][2] + + except gnocchiclient.exceptions.NotFound as e: + log.debug( + "No metric %s found for instance disk %s: %s", + openstack_metric_name, + instance["id"], + e, + ) + return value + def _collect_instance_metric(self, openstack_metric_name, resource_id): value = None try: @@ -299,12 +435,13 @@ class GnocchiBackend(OpenstackBackend): class CeilometerBackend(OpenstackBackend): - def __init__(self, vim_account: dict): - self.client = self._build_ceilometer_client(vim_account) + def __init__(self, vim_account: dict, vim_session: object): + self.client = self._build_ceilometer_client(vim_account, vim_session) - def _build_ceilometer_client(self, vim_account: dict) -> ceilometer_client.Client: - sess = OpenstackUtils.get_session(vim_account) - return ceilometer_client.Client("2", session=sess) + def _build_ceilometer_client( + self, vim_account: dict, vim_session: object + ) -> ceilometer_client.Client: + return ceilometer_client.Client("2", session=vim_session) def collect_metric( self, metric_type: MetricType, metric_name: str, resource_id: str