Coverage for osm_mon/collector/vnf_collectors/openstack.py: 45%
214 statements
« prev ^ index » next coverage.py v7.6.12, created at 2025-05-06 19:04 +0000
1# Copyright 2018 Whitestack, LLC
2# *************************************************************
4# This file is part of OSM Monitoring module
5# All Rights Reserved to Whitestack, LLC
7# Licensed under the Apache License, Version 2.0 (the "License"); you may
8# not use this file except in compliance with the License. You may obtain
9# a copy of the License at
11# http://www.apache.org/licenses/LICENSE-2.0
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16# License for the specific language governing permissions and limitations
17# under the License.
19# For those usages not covered by the Apache License, Version 2.0 please
20# contact: bdiaz@whitestack.com or glavado@whitestack.com
21##
22from enum import Enum
23import logging
24import time
25from typing import List
27from ceilometerclient import client as ceilometer_client
28from ceilometerclient.exc import HTTPException
29import gnocchiclient.exceptions
30from gnocchiclient.v1 import client as gnocchi_client
31from keystoneauth1.exceptions.catalog import EndpointNotFound
32from keystoneclient.v3 import client as keystone_client
33from neutronclient.v2_0 import client as neutron_client
34from prometheus_api_client import PrometheusConnect as prometheus_client
36from osm_mon.collector.metric import Metric
37from osm_mon.collector.utils.openstack import OpenstackUtils
38from osm_mon.collector.vnf_collectors.base_vim import BaseVimCollector
39from osm_mon.collector.vnf_metric import VnfMetric
40from osm_mon.core.common_db import CommonDbClient
41from osm_mon.core.config import Config
44log = logging.getLogger(__name__)
46METRIC_MAPPINGS = {
47 "average_memory_utilization": "memory.usage",
48 "disk_read_ops": "disk.read.requests.rate",
49 "disk_write_ops": "disk.write.requests.rate",
50 "disk_read_bytes": "disk.read.bytes.rate",
51 "disk_write_bytes": "disk.write.bytes.rate",
52 "packets_in_dropped": "network.outgoing.packets.drop",
53 "packets_out_dropped": "network.incoming.packets.drop",
54 "packets_received": "network.incoming.packets.rate",
55 "packets_sent": "network.outgoing.packets.rate",
56 "cpu_utilization": "cpu",
57}
59METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = {
60 "cpu_utilization": "cpu",
61 "average_memory_utilization": "memory_usage",
62 "disk_read_ops": "disk_device_read_requests",
63 "disk_write_ops": "disk_device_write_requests",
64 "disk_read_bytes": "disk_device_read_bytes",
65 "disk_write_bytes": "disk_device_write_bytes",
66 "packets_in_dropped": "network_incoming_packets_drop",
67 "packets_out_dropped": "network_outgoing_packets_drop",
68 "packets_received": "network_incoming_packets",
69 "packets_sent": "network_outgoing_packets",
70}
72# Metrics which have new names in Rocky and higher releases
73METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES = {
74 "disk_read_ops": "disk.device.read.requests",
75 "disk_write_ops": "disk.device.write.requests",
76 "disk_read_bytes": "disk.device.read.bytes",
77 "disk_write_bytes": "disk.device.write.bytes",
78 "packets_received": "network.incoming.packets",
79 "packets_sent": "network.outgoing.packets",
80}
82METRIC_MULTIPLIERS = {"cpu": 0.0000001}
84METRIC_AGGREGATORS = {"cpu": "rate:mean"}
86INTERFACE_METRICS = [
87 "packets_in_dropped",
88 "packets_out_dropped",
89 "packets_received",
90 "packets_sent",
91]
93INSTANCE_DISK = [
94 "disk_read_ops",
95 "disk_write_ops",
96 "disk_read_bytes",
97 "disk_write_bytes",
98]
101class MetricType(Enum):
102 INSTANCE = "instance"
103 INTERFACE_ALL = "interface_all"
104 INTERFACE_ONE = "interface_one"
105 INSTANCEDISK = "instancedisk"
108class OpenstackCollector(BaseVimCollector):
109 def __init__(self, config: Config, vim_account_id: str, vim_session: object):
110 super().__init__(config, vim_account_id)
111 self.common_db = CommonDbClient(config)
112 vim_account = self.common_db.get_vim_account(vim_account_id)
113 self.backend = self._get_backend(vim_account, vim_session)
115 def _build_keystone_client(self, vim_account: dict) -> keystone_client.Client:
116 sess = OpenstackUtils.get_session(vim_account)
117 return keystone_client.Client(session=sess)
119 def _get_resource_uuid(
120 self, nsr_id: str, vnf_member_index: str, vdur_name: str
121 ) -> str:
122 vdur = self.common_db.get_vdur(nsr_id, vnf_member_index, vdur_name)
123 return vdur["vim-id"]
125 def collect(self, vnfr: dict) -> List[Metric]:
126 nsr_id = vnfr["nsr-id-ref"]
127 vnf_member_index = vnfr["member-vnf-index-ref"]
128 vnfd = self.common_db.get_vnfd(vnfr["vnfd-id"])
129 # Populate extra tags for metrics
130 tags = {}
131 tags["ns_name"] = self.common_db.get_nsr(nsr_id)["name"]
132 if vnfr["_admin"]["projects_read"]:
133 tags["project_id"] = vnfr["_admin"]["projects_read"][0]
134 else:
135 tags["project_id"] = ""
137 metrics = []
139 for vdur in vnfr["vdur"]:
140 # This avoids errors when vdur records have not been completely filled
141 if "name" not in vdur:
142 continue
143 vdu = next(filter(lambda vdu: vdu["id"] == vdur["vdu-id-ref"], vnfd["vdu"]))
144 if "monitoring-parameter" in vdu:
145 for param in vdu["monitoring-parameter"]:
146 metric_name = param["performance-metric"]
147 log.info(f"Using an {type(self.backend)} as backend")
148 if type(self.backend) is PrometheusTSBDBackend:
149 openstack_metric_name = self.backend.map_metric(metric_name)
150 else:
151 try:
152 openstack_metric_name = METRIC_MAPPINGS[metric_name]
153 except KeyError:
154 continue
155 metric_type = self._get_metric_type(metric_name)
156 try:
157 resource_id = self._get_resource_uuid(
158 nsr_id, vnf_member_index, vdur["name"]
159 )
160 except ValueError:
161 log.warning(
162 "Could not find resource_uuid for vdur %s, vnf_member_index %s, nsr_id %s. "
163 "Was it recently deleted?",
164 vdur["name"],
165 vnf_member_index,
166 nsr_id,
167 )
168 continue
169 try:
170 log.info(
171 "Collecting metric type: %s and metric_name: %s and resource_id %s and ",
172 metric_type,
173 metric_name,
174 resource_id,
175 )
176 value = self.backend.collect_metric(
177 metric_type, openstack_metric_name, resource_id
178 )
180 if (
181 value is None
182 and metric_name
183 in METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES
184 and type(self.backend) is not PrometheusTSBDBackend
185 ):
186 # Reattempting metric collection with new metric names.
187 # Some metric names have changed in newer Openstack releases
188 log.info(
189 "Reattempting metric collection for type: %s and name: %s and resource_id %s",
190 metric_type,
191 metric_name,
192 resource_id,
193 )
194 openstack_metric_name = (
195 METRIC_MAPPINGS_FOR_ROCKY_AND_NEWER_RELEASES[
196 metric_name
197 ]
198 )
199 value = self.backend.collect_metric(
200 metric_type, openstack_metric_name, resource_id
201 )
202 if value is not None:
203 log.info("value: %s", value)
204 metric = VnfMetric(
205 nsr_id,
206 vnf_member_index,
207 vdur["name"],
208 metric_name,
209 value,
210 tags,
211 )
212 metrics.append(metric)
213 else:
214 log.info("metric value is empty")
215 except Exception as e:
216 log.exception(
217 "Error collecting metric %s for vdu %s"
218 % (metric_name, vdur["name"])
219 )
220 log.info("Error in metric collection: %s" % e)
221 return metrics
223 def _get_backend(self, vim_account: dict, vim_session: object):
224 if vim_account.get("prometheus-config"):
225 try:
226 tsbd = PrometheusTSBDBackend(vim_account)
227 log.debug("Using prometheustsbd backend to collect metric")
228 return tsbd
229 except Exception as e:
230 log.error(f"Can't create prometheus client, {e}")
231 return None
232 try:
233 gnocchi = GnocchiBackend(vim_account, vim_session)
234 gnocchi.client.metric.list(limit=1)
235 log.debug("Using gnocchi backend to collect metric")
236 return gnocchi
237 except (HTTPException, EndpointNotFound):
238 ceilometer = CeilometerBackend(vim_account, vim_session)
239 ceilometer.client.capabilities.get()
240 log.debug("Using ceilometer backend to collect metric")
241 return ceilometer
243 def _get_metric_type(self, metric_name: str) -> MetricType:
244 if metric_name not in INTERFACE_METRICS:
245 if metric_name not in INSTANCE_DISK:
246 return MetricType.INSTANCE
247 else:
248 return MetricType.INSTANCEDISK
249 else:
250 return MetricType.INTERFACE_ALL
253class OpenstackBackend:
254 def collect_metric(
255 self, metric_type: MetricType, metric_name: str, resource_id: str
256 ):
257 pass
260class PrometheusTSBDBackend(OpenstackBackend):
261 def __init__(self, vim_account: dict):
262 self.map = self._build_map(vim_account)
263 self.cred = vim_account["prometheus-config"].get("prometheus-cred")
264 self.client = self._build_prometheus_client(
265 vim_account["prometheus-config"]["prometheus-url"]
266 )
268 def _build_prometheus_client(self, url: str) -> prometheus_client:
269 return prometheus_client(url, disable_ssl=True)
271 def _build_map(self, vim_account: dict) -> dict:
272 custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD
273 if "prometheus-map" in vim_account["prometheus-config"]:
274 custom_map.update(vim_account["prometheus-config"]["prometheus-map"])
275 return custom_map
277 def collect_metric(
278 self, metric_type: MetricType, metric_name: str, resource_id: str
279 ):
280 metric = self.query_metric(metric_name, resource_id)
281 return metric["value"][1] if metric else None
283 def map_metric(self, metric_name: str):
284 return self.map[metric_name]
286 def query_metric(self, metric_name, resource_id=None):
287 metrics = self.client.get_current_metric_value(metric_name=metric_name)
288 if resource_id:
289 metric = next(
290 filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)
291 )
292 return metric
293 return metrics
296class GnocchiBackend(OpenstackBackend):
297 def __init__(self, vim_account: dict, vim_session: object):
298 self.client = self._build_gnocchi_client(vim_account, vim_session)
299 self.neutron = self._build_neutron_client(vim_account, vim_session)
301 def _build_gnocchi_client(
302 self, vim_account: dict, vim_session: object
303 ) -> gnocchi_client.Client:
304 return gnocchi_client.Client(session=vim_session)
306 def _build_neutron_client(
307 self, vim_account: dict, vim_session: object
308 ) -> neutron_client.Client:
309 return neutron_client.Client(session=vim_session)
311 def collect_metric(
312 self, metric_type: MetricType, metric_name: str, resource_id: str
313 ):
314 if metric_type == MetricType.INTERFACE_ALL:
315 return self._collect_interface_all_metric(metric_name, resource_id)
317 elif metric_type == MetricType.INSTANCE:
318 return self._collect_instance_metric(metric_name, resource_id)
320 elif metric_type == MetricType.INSTANCEDISK:
321 return self._collect_instance_disk_metric(metric_name, resource_id)
323 else:
324 raise Exception("Unknown metric type %s" % metric_type.value)
326 def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
327 total_measure = None
328 interfaces = self.client.resource.search(
329 resource_type="instance_network_interface",
330 query={"=": {"instance_id": resource_id}},
331 )
332 for interface in interfaces:
333 try:
334 measures = self.client.metric.get_measures(
335 openstack_metric_name, resource_id=interface["id"], limit=1
336 )
337 if measures:
338 if not total_measure:
339 total_measure = 0.0
340 total_measure += measures[-1][2]
341 except (gnocchiclient.exceptions.NotFound, TypeError) as e:
342 # Gnocchi in some Openstack versions raise TypeError instead of NotFound
343 log.debug(
344 "No metric %s found for interface %s: %s",
345 openstack_metric_name,
346 interface["id"],
347 e,
348 )
349 return total_measure
351 def _collect_instance_disk_metric(self, openstack_metric_name, resource_id):
352 value = None
353 instances = self.client.resource.search(
354 resource_type="instance_disk",
355 query={"=": {"instance_id": resource_id}},
356 )
357 for instance in instances:
358 try:
359 measures = self.client.metric.get_measures(
360 openstack_metric_name, resource_id=instance["id"], limit=1
361 )
362 if measures:
363 value = measures[-1][2]
365 except gnocchiclient.exceptions.NotFound as e:
366 log.debug(
367 "No metric %s found for instance disk %s: %s",
368 openstack_metric_name,
369 instance["id"],
370 e,
371 )
372 return value
374 def _collect_instance_metric(self, openstack_metric_name, resource_id):
375 value = None
376 try:
377 aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
379 try:
380 measures = self.client.metric.get_measures(
381 openstack_metric_name,
382 aggregation=aggregation,
383 start=time.time() - 1200,
384 resource_id=resource_id,
385 )
386 if measures:
387 value = measures[-1][2]
388 except (
389 gnocchiclient.exceptions.NotFound,
390 gnocchiclient.exceptions.BadRequest,
391 TypeError,
392 ) as e:
393 # CPU metric in previous Openstack versions do not support rate:mean aggregation method
394 # Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
395 if openstack_metric_name == "cpu":
396 log.debug(
397 "No metric %s found for instance %s: %s",
398 openstack_metric_name,
399 resource_id,
400 e,
401 )
402 log.info(
403 "Retrying to get metric %s for instance %s without aggregation",
404 openstack_metric_name,
405 resource_id,
406 )
407 measures = self.client.metric.get_measures(
408 openstack_metric_name, resource_id=resource_id, limit=1
409 )
410 else:
411 raise e
412 # measures[-1] is the last measure
413 # measures[-2] is the previous measure
414 # measures[x][2] is the value of the metric
415 if measures and len(measures) >= 2:
416 value = measures[-1][2] - measures[-2][2]
417 if value:
418 # measures[-1][0] is the time of the reporting interval
419 # measures[-1][1] is the duration of the reporting interval
420 if aggregation:
421 # If this is an aggregate, we need to divide the total over the reported time period.
422 # Even if the aggregation method is not supported by Openstack, the code will execute it
423 # because aggregation is specified in METRIC_AGGREGATORS
424 value = value / measures[-1][1]
425 if openstack_metric_name in METRIC_MULTIPLIERS:
426 value = value * METRIC_MULTIPLIERS[openstack_metric_name]
427 except gnocchiclient.exceptions.NotFound as e:
428 log.debug(
429 "No metric %s found for instance %s: %s",
430 openstack_metric_name,
431 resource_id,
432 e,
433 )
434 return value
437class CeilometerBackend(OpenstackBackend):
438 def __init__(self, vim_account: dict, vim_session: object):
439 self.client = self._build_ceilometer_client(vim_account, vim_session)
441 def _build_ceilometer_client(
442 self, vim_account: dict, vim_session: object
443 ) -> ceilometer_client.Client:
444 return ceilometer_client.Client("2", session=vim_session)
446 def collect_metric(
447 self, metric_type: MetricType, metric_name: str, resource_id: str
448 ):
449 if metric_type != MetricType.INSTANCE:
450 raise NotImplementedError(
451 "Ceilometer backend only support instance metrics"
452 )
453 measures = self.client.samples.list(
454 meter_name=metric_name,
455 limit=1,
456 q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
457 )
458 return measures[0].counter_volume if measures else None