src/osm_ngsa/osm_mon/vim_connectors/openstack.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 from enum import Enum
  18 import logging
  19 import time
  20 from typing import Dict, List
  21
  22 from ceilometerclient import client as ceilometer_client
  23 from ceilometerclient.exc import HTTPException
  24 import gnocchiclient.exceptions
  25 from gnocchiclient.v1 import client as gnocchi_client
  26 from keystoneauth1 import session
  27 from keystoneauth1.exceptions.catalog import EndpointNotFound
  28 from keystoneauth1.identity import v3
  29 from novaclient import client as nova_client
  30 from osm_mon.vim_connectors.base_vim import VIMConnector
  31 from osm_mon.vim_connectors.vrops_helper import vROPS_Helper
  32 from prometheus_api_client import PrometheusConnect as prometheus_client
  33
  34 log = logging.getLogger(__name__)
  35
  36 METRIC_MULTIPLIERS = {"cpu": 0.0000001}
  37
  38 METRIC_AGGREGATORS = {"cpu": "rate:mean"}
  39
  40 INTERFACE_METRICS = [
  41     "packets_in_dropped",
  42     "packets_out_dropped",
  43     "packets_received",
  44     "packets_sent",
  45 ]
  46
  47 INSTANCE_DISK = [
  48     "disk_read_ops",
  49     "disk_write_ops",
  50     "disk_read_bytes",
  51     "disk_write_bytes",
  52 ]
  53
  54 METRIC_MAPPINGS = {
  55     "average_memory_utilization": "memory.usage",
  56     "disk_read_ops": "disk.device.read.requests",
  57     "disk_write_ops": "disk.device.write.requests",
  58     "disk_read_bytes": "disk.device.read.bytes",
  59     "disk_write_bytes": "disk.device.write.bytes",
  60     "packets_in_dropped": "network.outgoing.packets.drop",
  61     "packets_out_dropped": "network.incoming.packets.drop",
  62     "packets_received": "network.incoming.packets",
  63     "packets_sent": "network.outgoing.packets",
  64     "cpu_utilization": "cpu",
  65 }
  66
  67 METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = {
  68     "cpu_utilization": "cpu",
  69     "average_memory_utilization": "memory_usage",
  70     "disk_read_ops": "disk_device_read_requests",
  71     "disk_write_ops": "disk_device_write_requests",
  72     "disk_read_bytes": "disk_device_read_bytes",
  73     "disk_write_bytes": "disk_device_write_bytes",
  74     "packets_in_dropped": "network_incoming_packets_drop",
  75     "packets_out_dropped": "network_outgoing_packets_drop",
  76     "packets_received": "network_incoming_packets",
  77     "packets_sent": "network_outgoing_packets",
  78 }
  79
  80
  81 class MetricType(Enum):
  82     INSTANCE = "instance"
  83     INTERFACE_ALL = "interface_all"
  84     INTERFACE_ONE = "interface_one"
  85     INSTANCEDISK = "instancedisk"
  86
  87
  88 class CertificateNotCreated(Exception):
  89     pass
  90
  91
  92 class OpenStackCollector(VIMConnector):
  93     def __init__(self, vim_account: Dict):
  94         log.debug("__init__")
  95         self.vim_account = vim_account
  96         self.vim_session = None
  97         self.vim_session = self._get_session(vim_account)
  98         self.nova = self._build_nova_client()
  99         self.backend = self._get_backend(vim_account, self.vim_session)
 100
 101     def _get_session(self, creds: Dict):
 102         log.debug("_get_session")
 103         verify_ssl = True
 104         project_domain_name = "Default"
 105         user_domain_name = "Default"
 106         try:
 107             if "config" in creds:
 108                 vim_config = creds["config"]
 109                 if "insecure" in vim_config and vim_config["insecure"]:
 110                     verify_ssl = False
 111                 if "ca_cert" in vim_config:
 112                     verify_ssl = vim_config["ca_cert"]
 113                 elif "ca_cert_content" in vim_config:
 114                     # vim_config = self._create_file_cert(vim_config, creds["_id"])
 115                     verify_ssl = vim_config["ca_cert"]
 116                 if "project_domain_name" in vim_config:
 117                     project_domain_name = vim_config["project_domain_name"]
 118                 if "user_domain_name" in vim_config:
 119                     user_domain_name = vim_config["user_domain_name"]
 120             auth = v3.Password(
 121                 auth_url=creds["vim_url"],
 122                 username=creds["vim_user"],
 123                 password=creds["vim_password"],
 124                 project_name=creds["vim_tenant_name"],
 125                 project_domain_name=project_domain_name,
 126                 user_domain_name=user_domain_name,
 127             )
 128             return session.Session(auth=auth, verify=verify_ssl, timeout=10)
 129         except CertificateNotCreated as e:
 130             log.error(e)
 131
 132     def _get_backend(self, vim_account: dict, vim_session: object):
 133         if vim_account.get("prometheus-config"):
 134             # try:
 135             #     tsbd = PrometheusTSBDBackend(vim_account)
 136             #     log.debug("Using prometheustsbd backend to collect metric")
 137             #     return tsbd
 138             # except Exception as e:
 139             #     log.error(f"Can't create prometheus client, {e}")
 140             #     return None
 141             return None
 142
 143         if "config" in vim_account and "vim_type" in vim_account["config"]:
 144             vim_type = vim_account["config"]["vim_type"].lower()
 145             log.debug(f"vim_type: {vim_type}")
 146             log.debug(f"vim_account[config]: {vim_account['config']}")
 147             if vim_type == "vio" and "vrops_site" in vim_account["config"]:
 148                 try:
 149                     log.debug("Using vROPS backend to collect metric")
 150                     vrops = VropsBackend(vim_account)
 151                     return vrops
 152                 except Exception as e:
 153                     log.error(f"Can't create vROPS client, {e}")
 154             return None
 155
 156         try:
 157             gnocchi = GnocchiBackend(vim_account, vim_session)
 158             gnocchi.client.metric.list(limit=1)
 159             log.debug("Using gnocchi backend to collect metric")
 160             return gnocchi
 161         except (HTTPException, EndpointNotFound):
 162             ceilometer = CeilometerBackend(vim_account, vim_session)
 163             ceilometer.client.capabilities.get()
 164             log.debug("Using ceilometer backend to collect metric")
 165             return ceilometer
 166
 167     def _build_nova_client(self) -> nova_client.Client:
 168         return nova_client.Client("2", session=self.vim_session, timeout=10)
 169
 170     def _build_gnocchi_client(self) -> gnocchi_client.Client:
 171         return gnocchi_client.Client(session=self.vim_session)
 172
 173     def collect_servers_status(self) -> List[Dict]:
 174         log.debug("collect_servers_status")
 175         servers = []
 176         for server in self.nova.servers.list(detailed=True):
 177             vm = {
 178                 "id": server.id,
 179                 "name": server.name,
 180                 "status": (0 if (server.status == "ERROR") else 1),
 181             }
 182             servers.append(vm)
 183         return servers
 184
 185     def is_vim_ok(self) -> bool:
 186         log.debug("is_vim_ok")
 187         try:
 188             self.nova.servers.list()
 189             return True
 190         except Exception as e:
 191             log.warning("VIM status is not OK: %s" % e)
 192             return False
 193
 194     def _get_metric_type(self, metric_name: str) -> MetricType:
 195         if metric_name not in INTERFACE_METRICS:
 196             if metric_name not in INSTANCE_DISK:
 197                 return MetricType.INSTANCE
 198             else:
 199                 return MetricType.INSTANCEDISK
 200         else:
 201             return MetricType.INTERFACE_ALL
 202
 203     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 204         log.debug("collect_metrics")
 205         if not self.backend:
 206             log.error("Undefined backend")
 207             return []
 208
 209         if type(self.backend) is PrometheusTSBDBackend:
 210             log.info("Using Prometheus as backend (NOT SUPPORTED)")
 211             return []
 212
 213         if type(self.backend) is VropsBackend:
 214             log.info("Using vROPS as backend")
 215             return self.backend.collect_metrics(metric_list)
 216
 217         metric_results = []
 218         for metric in metric_list:
 219             server = metric["vm_id"]
 220             metric_name = metric["metric"]
 221             try:
 222                 openstack_metric_name = METRIC_MAPPINGS[metric_name]
 223             except KeyError:
 224                 continue
 225             metric_type = self._get_metric_type(metric_name)
 226             log.info(f"Collecting metric {openstack_metric_name} for {server}")
 227             try:
 228                 value = self.backend.collect_metric(
 229                     metric_type, openstack_metric_name, server
 230                 )
 231                 if value is not None:
 232                     log.info(f"value: {value}")
 233                     metric["value"] = value
 234                     metric_results.append(metric)
 235                 else:
 236                     log.info("metric value is empty")
 237             except Exception as e:
 238                 log.error("Error in metric collection: %s" % e)
 239         return metric_results
 240
 241
 242 class OpenstackBackend:
 243     def collect_metric(
 244         self, metric_type: MetricType, metric_name: str, resource_id: str
 245     ):
 246         pass
 247
 248     def collect_metrics(self, metrics_list: List[Dict]):
 249         pass
 250
 251
 252 class PrometheusTSBDBackend(OpenstackBackend):
 253     def __init__(self, vim_account: dict):
 254         self.map = self._build_map(vim_account)
 255         self.cred = vim_account["prometheus-config"].get("prometheus-cred")
 256         self.client = self._build_prometheus_client(
 257             vim_account["prometheus-config"]["prometheus-url"]
 258         )
 259
 260     def _build_prometheus_client(self, url: str) -> prometheus_client:
 261         return prometheus_client(url, disable_ssl=True)
 262
 263     def _build_map(self, vim_account: dict) -> dict:
 264         custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD
 265         if "prometheus-map" in vim_account["prometheus-config"]:
 266             custom_map.update(vim_account["prometheus-config"]["prometheus-map"])
 267         return custom_map
 268
 269     def collect_metric(
 270         self, metric_type: MetricType, metric_name: str, resource_id: str
 271     ):
 272         metric = self.query_metric(metric_name, resource_id)
 273         return metric["value"][1] if metric else None
 274
 275     def map_metric(self, metric_name: str):
 276         return self.map[metric_name]
 277
 278     def query_metric(self, metric_name, resource_id=None):
 279         metrics = self.client.get_current_metric_value(metric_name=metric_name)
 280         if resource_id:
 281             metric = next(
 282                 filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)
 283             )
 284             return metric
 285         return metrics
 286
 287
 288 class GnocchiBackend(OpenstackBackend):
 289     def __init__(self, vim_account: dict, vim_session: object):
 290         self.client = self._build_gnocchi_client(vim_account, vim_session)
 291
 292     def _build_gnocchi_client(
 293         self, vim_account: dict, vim_session: object
 294     ) -> gnocchi_client.Client:
 295         return gnocchi_client.Client(session=vim_session)
 296
 297     def collect_metric(
 298         self, metric_type: MetricType, metric_name: str, resource_id: str
 299     ):
 300         if metric_type == MetricType.INTERFACE_ALL:
 301             return self._collect_interface_all_metric(metric_name, resource_id)
 302
 303         elif metric_type == MetricType.INSTANCE:
 304             return self._collect_instance_metric(metric_name, resource_id)
 305
 306         elif metric_type == MetricType.INSTANCEDISK:
 307             return self._collect_instance_disk_metric(metric_name, resource_id)
 308
 309         else:
 310             raise Exception("Unknown metric type %s" % metric_type.value)
 311
 312     def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
 313         total_measure = None
 314         interfaces = self.client.resource.search(
 315             resource_type="instance_network_interface",
 316             query={"=": {"instance_id": resource_id}},
 317         )
 318         for interface in interfaces:
 319             try:
 320                 measures = self.client.metric.get_measures(
 321                     openstack_metric_name, resource_id=interface["id"], limit=1
 322                 )
 323                 if measures:
 324                     if not total_measure:
 325                         total_measure = 0.0
 326                     total_measure += measures[-1][2]
 327             except (gnocchiclient.exceptions.NotFound, TypeError) as e:
 328                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound
 329                 log.debug(
 330                     "No metric %s found for interface %s: %s",
 331                     openstack_metric_name,
 332                     interface["id"],
 333                     e,
 334                 )
 335         return total_measure
 336
 337     def _collect_instance_disk_metric(self, openstack_metric_name, resource_id):
 338         value = None
 339         instances = self.client.resource.search(
 340             resource_type="instance_disk",
 341             query={"=": {"instance_id": resource_id}},
 342         )
 343         for instance in instances:
 344             try:
 345                 measures = self.client.metric.get_measures(
 346                     openstack_metric_name, resource_id=instance["id"], limit=1
 347                 )
 348                 if measures:
 349                     value = measures[-1][2]
 350
 351             except gnocchiclient.exceptions.NotFound as e:
 352                 log.debug(
 353                     "No metric %s found for instance disk %s: %s",
 354                     openstack_metric_name,
 355                     instance["id"],
 356                     e,
 357                 )
 358         return value
 359
 360     def _collect_instance_metric(self, openstack_metric_name, resource_id):
 361         value = None
 362         try:
 363             aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
 364
 365             try:
 366                 measures = self.client.metric.get_measures(
 367                     openstack_metric_name,
 368                     aggregation=aggregation,
 369                     start=time.time() - 1200,
 370                     resource_id=resource_id,
 371                 )
 372                 if measures:
 373                     value = measures[-1][2]
 374             except (
 375                 gnocchiclient.exceptions.NotFound,
 376                 gnocchiclient.exceptions.BadRequest,
 377                 TypeError,
 378             ) as e:
 379                 # CPU metric in previous Openstack versions do not support rate:mean aggregation method
 380                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
 381                 if openstack_metric_name == "cpu":
 382                     log.debug(
 383                         "No metric %s found for instance %s: %s",
 384                         openstack_metric_name,
 385                         resource_id,
 386                         e,
 387                     )
 388                     log.info(
 389                         "Retrying to get metric %s for instance %s without aggregation",
 390                         openstack_metric_name,
 391                         resource_id,
 392                     )
 393                     measures = self.client.metric.get_measures(
 394                         openstack_metric_name, resource_id=resource_id, limit=1
 395                     )
 396                 else:
 397                     raise e
 398                 # measures[-1] is the last measure
 399                 # measures[-2] is the previous measure
 400                 # measures[x][2] is the value of the metric
 401                 if measures and len(measures) >= 2:
 402                     value = measures[-1][2] - measures[-2][2]
 403             if value:
 404                 # measures[-1][0] is the time of the reporting interval
 405                 # measures[-1][1] is the duration of the reporting interval
 406                 if aggregation:
 407                     # If this is an aggregate, we need to divide the total over the reported time period.
 408                     # Even if the aggregation method is not supported by Openstack, the code will execute it
 409                     # because aggregation is specified in METRIC_AGGREGATORS
 410                     value = value / measures[-1][1]
 411                 if openstack_metric_name in METRIC_MULTIPLIERS:
 412                     value = value * METRIC_MULTIPLIERS[openstack_metric_name]
 413         except gnocchiclient.exceptions.NotFound as e:
 414             log.debug(
 415                 "No metric %s found for instance %s: %s",
 416                 openstack_metric_name,
 417                 resource_id,
 418                 e,
 419             )
 420         return value
 421
 422
 423 class CeilometerBackend(OpenstackBackend):
 424     def __init__(self, vim_account: dict, vim_session: object):
 425         self.client = self._build_ceilometer_client(vim_account, vim_session)
 426
 427     def _build_ceilometer_client(
 428         self, vim_account: dict, vim_session: object
 429     ) -> ceilometer_client.Client:
 430         return ceilometer_client.Client("2", session=vim_session)
 431
 432     def collect_metric(
 433         self, metric_type: MetricType, metric_name: str, resource_id: str
 434     ):
 435         if metric_type != MetricType.INSTANCE:
 436             raise NotImplementedError(
 437                 "Ceilometer backend only support instance metrics"
 438             )
 439         measures = self.client.samples.list(
 440             meter_name=metric_name,
 441             limit=1,
 442             q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
 443         )
 444         return measures[0].counter_volume if measures else None
 445
 446
 447 class VropsBackend(OpenstackBackend):
 448     def __init__(self, vim_account: dict):
 449         self.vrops = vROPS_Helper(
 450             vrops_site=vim_account["config"]["vrops_site"],
 451             vrops_user=vim_account["config"]["vrops_user"],
 452             vrops_password=vim_account["config"]["vrops_password"],
 453         )
 454
 455     def collect_metrics(self, metrics_list: List[Dict]):
 456         # Fetch the list of all known resources from vROPS.
 457         resource_list = self.vrops.get_vm_resource_list_from_vrops()
 458
 459         vdu_mappings = {}
 460         extended_metrics = []
 461         for metric in metrics_list:
 462             vim_id = metric["vm_id"]
 463             # Map the vROPS instance id to the vim-id so we can look it up.
 464             for resource in resource_list:
 465                 for resourceIdentifier in resource["resourceKey"][
 466                     "resourceIdentifiers"
 467                 ]:
 468                     if (
 469                         resourceIdentifier["identifierType"]["name"]
 470                         == "VMEntityInstanceUUID"
 471                     ):
 472                         if resourceIdentifier["value"] != vim_id:
 473                             continue
 474                         vdu_mappings[vim_id] = resource["identifier"]
 475             if vim_id in vdu_mappings:
 476                 metric["vrops_id"] = vdu_mappings[vim_id]
 477                 extended_metrics.append(metric)
 478
 479         if len(extended_metrics) != 0:
 480             return self.vrops.get_metrics(extended_metrics)
 481         else:
 482             return []