src/osm_ngsa/osm_mon/vim_connectors/openstack.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 from enum import Enum
  18 import logging
  19 import time
  20 from typing import Dict, List
  21
  22 from ceilometerclient import client as ceilometer_client
  23 from ceilometerclient.exc import HTTPException
  24 import gnocchiclient.exceptions
  25 from gnocchiclient.v1 import client as gnocchi_client
  26 from keystoneauth1 import session
  27 from keystoneauth1.exceptions.catalog import EndpointNotFound
  28 from keystoneauth1.identity import v3
  29 from novaclient import client as nova_client
  30 from osm_mon.vim_connectors.base_vim import VIMConnector
  31 from osm_mon.vim_connectors.vrops_helper import vROPS_Helper
  32 from prometheus_api_client import PrometheusConnect as prometheus_client
  33
  34 log = logging.getLogger(__name__)
  35
  36 METRIC_MULTIPLIERS = {"cpu": 0.0000001}
  37
  38 METRIC_AGGREGATORS = {"cpu": "rate:mean"}
  39
  40 INTERFACE_METRICS = [
  41     "packets_in_dropped",
  42     "packets_out_dropped",
  43     "packets_received",
  44     "packets_sent",
  45 ]
  46
  47 INSTANCE_DISK = [
  48     "disk_read_ops",
  49     "disk_write_ops",
  50     "disk_read_bytes",
  51     "disk_write_bytes",
  52 ]
  53
  54 METRIC_MAPPINGS = {
  55     "average_memory_utilization": "memory.usage",
  56     "disk_read_ops": "disk.device.read.requests",
  57     "disk_write_ops": "disk.device.write.requests",
  58     "disk_read_bytes": "disk.device.read.bytes",
  59     "disk_write_bytes": "disk.device.write.bytes",
  60     "packets_in_dropped": "network.outgoing.packets.drop",
  61     "packets_out_dropped": "network.incoming.packets.drop",
  62     "packets_received": "network.incoming.packets",
  63     "packets_sent": "network.outgoing.packets",
  64     "cpu_utilization": "cpu",
  65 }
  66
  67 METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = {
  68     "cpu_utilization": "cpu",
  69     "average_memory_utilization": "memory_usage",
  70     "disk_read_ops": "disk_device_read_requests",
  71     "disk_write_ops": "disk_device_write_requests",
  72     "disk_read_bytes": "disk_device_read_bytes",
  73     "disk_write_bytes": "disk_device_write_bytes",
  74     "packets_in_dropped": "network_incoming_packets_drop",
  75     "packets_out_dropped": "network_outgoing_packets_drop",
  76     "packets_received": "network_incoming_packets",
  77     "packets_sent": "network_outgoing_packets",
  78 }
  79
  80
  81 class MetricType(Enum):
  82     INSTANCE = "instance"
  83     INTERFACE_ALL = "interface_all"
  84     INTERFACE_ONE = "interface_one"
  85     INSTANCEDISK = "instancedisk"
  86
  87
  88 class CertificateNotCreated(Exception):
  89     pass
  90
  91
  92 class OpenStackCollector(VIMConnector):
  93     def __init__(self, vim_account: Dict):
  94         log.debug("__init__")
  95         self.vim_account = vim_account
  96         self.vim_session = None
  97         self.vim_session = self._get_session(vim_account)
  98         self.nova = self._build_nova_client()
  99         self.backend = self._get_backend(vim_account, self.vim_session)
 100
 101     def _get_session(self, creds: Dict):
 102         log.debug("_get_session")
 103         verify_ssl = True
 104         project_domain_name = "Default"
 105         user_domain_name = "Default"
 106         try:
 107             if "config" in creds:
 108                 vim_config = creds["config"]
 109                 if "insecure" in vim_config and vim_config["insecure"]:
 110                     verify_ssl = False
 111                 if "ca_cert" in vim_config:
 112                     verify_ssl = vim_config["ca_cert"]
 113                 elif "ca_cert_content" in vim_config:
 114                     # vim_config = self._create_file_cert(vim_config, creds["_id"])
 115                     verify_ssl = vim_config["ca_cert"]
 116                 if "project_domain_name" in vim_config:
 117                     project_domain_name = vim_config["project_domain_name"]
 118                 if "user_domain_name" in vim_config:
 119                     user_domain_name = vim_config["user_domain_name"]
 120             auth = v3.Password(
 121                 auth_url=creds["vim_url"],
 122                 username=creds["vim_user"],
 123                 password=creds["vim_password"],
 124                 project_name=creds["vim_tenant_name"],
 125                 project_domain_name=project_domain_name,
 126                 user_domain_name=user_domain_name,
 127             )
 128             return session.Session(auth=auth, verify=verify_ssl, timeout=10)
 129         except CertificateNotCreated as e:
 130             log.error(e)
 131
 132     def _get_backend(self, vim_account: dict, vim_session: object):
 133         if vim_account.get("prometheus-config"):
 134             # try:
 135             #     tsbd = PrometheusTSBDBackend(vim_account)
 136             #     log.debug("Using prometheustsbd backend to collect metric")
 137             #     return tsbd
 138             # except Exception as e:
 139             #     log.error(f"Can't create prometheus client, {e}")
 140             #     return None
 141             return None
 142
 143         if "config" in vim_account and "vim_type" in vim_account["config"]:
 144             vim_type = vim_account["config"]["vim_type"].lower()
 145             log.debug(f"vim_type: {vim_type}")
 146             log.debug(f"vim_account[config]: {vim_account['config']}")
 147             if vim_type == "vio" and "vrops_site" in vim_account["config"]:
 148                 try:
 149                     log.debug("Using vROPS backend to collect metric")
 150                     vrops = VropsBackend(vim_account)
 151                     return vrops
 152                 except Exception as e:
 153                     log.error(f"Can't create vROPS client, {e}")
 154             return None
 155
 156         try:
 157             gnocchi = GnocchiBackend(vim_account, vim_session)
 158             gnocchi.client.metric.list(limit=1)
 159             log.debug("Using gnocchi backend to collect metric")
 160             return gnocchi
 161         except (HTTPException, EndpointNotFound):
 162             ceilometer = CeilometerBackend(vim_account, vim_session)
 163             ceilometer.client.capabilities.get()
 164             log.debug("Using ceilometer backend to collect metric")
 165             return ceilometer
 166
 167     def _build_nova_client(self) -> nova_client.Client:
 168         return nova_client.Client("2", session=self.vim_session, timeout=10)
 169
 170     def _build_gnocchi_client(self) -> gnocchi_client.Client:
 171         return gnocchi_client.Client(session=self.vim_session)
 172
 173     def collect_servers_status(self) -> List[Dict]:
 174         log.debug("collect_servers_status")
 175         servers = []
 176         for server in self.nova.servers.list(detailed=True):
 177             vm = {
 178                 "id": server.id,
 179                 "name": server.name,
 180                 "status": (0 if (server.status == "ERROR") else 1),
 181             }
 182             servers.append(vm)
 183         return servers
 184
 185     def is_vim_ok(self) -> bool:
 186         log.debug("is_vim_ok")
 187         try:
 188             self.nova.servers.list()
 189             return True
 190         except Exception as e:
 191             log.warning("VIM status is not OK: %s" % e)
 192             return False
 193
 194     def _get_metric_type(self, metric_name: str) -> MetricType:
 195         if metric_name not in INTERFACE_METRICS:
 196             if metric_name not in INSTANCE_DISK:
 197                 return MetricType.INSTANCE
 198             else:
 199                 return MetricType.INSTANCEDISK
 200         else:
 201             return MetricType.INTERFACE_ALL
 202
 203     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 204         log.debug("collect_metrics")
 205         if not self.backend:
 206             log.error("Undefined backend")
 207             return []
 208
 209         if type(self.backend) is PrometheusTSBDBackend:
 210             log.info("Using Prometheus as backend (NOT SUPPORTED)")
 211             return []
 212
 213         if type(self.backend) is VropsBackend:
 214             log.info("Using vROPS as backend")
 215             return self.backend.collect_metrics(metric_list)
 216
 217         metric_results = []
 218         for metric in metric_list:
 219             server = metric["vm_id"]
 220             metric_name = metric["metric"]
 221             openstack_metric_name = METRIC_MAPPINGS[metric_name]
 222             metric_type = self._get_metric_type(metric_name)
 223             log.info(f"Collecting metric {openstack_metric_name} for {server}")
 224             try:
 225                 value = self.backend.collect_metric(
 226                     metric_type, openstack_metric_name, server
 227                 )
 228                 if value is not None:
 229                     log.info(f"value: {value}")
 230                     metric["value"] = value
 231                     metric_results.append(metric)
 232                 else:
 233                     log.info("metric value is empty")
 234             except Exception as e:
 235                 log.error("Error in metric collection: %s" % e)
 236         return metric_results
 237
 238
 239 class OpenstackBackend:
 240     def collect_metric(
 241         self, metric_type: MetricType, metric_name: str, resource_id: str
 242     ):
 243         pass
 244
 245     def collect_metrics(self, metrics_list: List[Dict]):
 246         pass
 247
 248
 249 class PrometheusTSBDBackend(OpenstackBackend):
 250     def __init__(self, vim_account: dict):
 251         self.map = self._build_map(vim_account)
 252         self.cred = vim_account["prometheus-config"].get("prometheus-cred")
 253         self.client = self._build_prometheus_client(
 254             vim_account["prometheus-config"]["prometheus-url"]
 255         )
 256
 257     def _build_prometheus_client(self, url: str) -> prometheus_client:
 258         return prometheus_client(url, disable_ssl=True)
 259
 260     def _build_map(self, vim_account: dict) -> dict:
 261         custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD
 262         if "prometheus-map" in vim_account["prometheus-config"]:
 263             custom_map.update(vim_account["prometheus-config"]["prometheus-map"])
 264         return custom_map
 265
 266     def collect_metric(
 267         self, metric_type: MetricType, metric_name: str, resource_id: str
 268     ):
 269         metric = self.query_metric(metric_name, resource_id)
 270         return metric["value"][1] if metric else None
 271
 272     def map_metric(self, metric_name: str):
 273         return self.map[metric_name]
 274
 275     def query_metric(self, metric_name, resource_id=None):
 276         metrics = self.client.get_current_metric_value(metric_name=metric_name)
 277         if resource_id:
 278             metric = next(
 279                 filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)
 280             )
 281             return metric
 282         return metrics
 283
 284
 285 class GnocchiBackend(OpenstackBackend):
 286     def __init__(self, vim_account: dict, vim_session: object):
 287         self.client = self._build_gnocchi_client(vim_account, vim_session)
 288
 289     def _build_gnocchi_client(
 290         self, vim_account: dict, vim_session: object
 291     ) -> gnocchi_client.Client:
 292         return gnocchi_client.Client(session=vim_session)
 293
 294     def collect_metric(
 295         self, metric_type: MetricType, metric_name: str, resource_id: str
 296     ):
 297         if metric_type == MetricType.INTERFACE_ALL:
 298             return self._collect_interface_all_metric(metric_name, resource_id)
 299
 300         elif metric_type == MetricType.INSTANCE:
 301             return self._collect_instance_metric(metric_name, resource_id)
 302
 303         elif metric_type == MetricType.INSTANCEDISK:
 304             return self._collect_instance_disk_metric(metric_name, resource_id)
 305
 306         else:
 307             raise Exception("Unknown metric type %s" % metric_type.value)
 308
 309     def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
 310         total_measure = None
 311         interfaces = self.client.resource.search(
 312             resource_type="instance_network_interface",
 313             query={"=": {"instance_id": resource_id}},
 314         )
 315         for interface in interfaces:
 316             try:
 317                 measures = self.client.metric.get_measures(
 318                     openstack_metric_name, resource_id=interface["id"], limit=1
 319                 )
 320                 if measures:
 321                     if not total_measure:
 322                         total_measure = 0.0
 323                     total_measure += measures[-1][2]
 324             except (gnocchiclient.exceptions.NotFound, TypeError) as e:
 325                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound
 326                 log.debug(
 327                     "No metric %s found for interface %s: %s",
 328                     openstack_metric_name,
 329                     interface["id"],
 330                     e,
 331                 )
 332         return total_measure
 333
 334     def _collect_instance_disk_metric(self, openstack_metric_name, resource_id):
 335         value = None
 336         instances = self.client.resource.search(
 337             resource_type="instance_disk",
 338             query={"=": {"instance_id": resource_id}},
 339         )
 340         for instance in instances:
 341             try:
 342                 measures = self.client.metric.get_measures(
 343                     openstack_metric_name, resource_id=instance["id"], limit=1
 344                 )
 345                 if measures:
 346                     value = measures[-1][2]
 347
 348             except gnocchiclient.exceptions.NotFound as e:
 349                 log.debug(
 350                     "No metric %s found for instance disk %s: %s",
 351                     openstack_metric_name,
 352                     instance["id"],
 353                     e,
 354                 )
 355         return value
 356
 357     def _collect_instance_metric(self, openstack_metric_name, resource_id):
 358         value = None
 359         try:
 360             aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
 361
 362             try:
 363                 measures = self.client.metric.get_measures(
 364                     openstack_metric_name,
 365                     aggregation=aggregation,
 366                     start=time.time() - 1200,
 367                     resource_id=resource_id,
 368                 )
 369                 if measures:
 370                     value = measures[-1][2]
 371             except (
 372                 gnocchiclient.exceptions.NotFound,
 373                 gnocchiclient.exceptions.BadRequest,
 374                 TypeError,
 375             ) as e:
 376                 # CPU metric in previous Openstack versions do not support rate:mean aggregation method
 377                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
 378                 if openstack_metric_name == "cpu":
 379                     log.debug(
 380                         "No metric %s found for instance %s: %s",
 381                         openstack_metric_name,
 382                         resource_id,
 383                         e,
 384                     )
 385                     log.info(
 386                         "Retrying to get metric %s for instance %s without aggregation",
 387                         openstack_metric_name,
 388                         resource_id,
 389                     )
 390                     measures = self.client.metric.get_measures(
 391                         openstack_metric_name, resource_id=resource_id, limit=1
 392                     )
 393                 else:
 394                     raise e
 395                 # measures[-1] is the last measure
 396                 # measures[-2] is the previous measure
 397                 # measures[x][2] is the value of the metric
 398                 if measures and len(measures) >= 2:
 399                     value = measures[-1][2] - measures[-2][2]
 400             if value:
 401                 # measures[-1][0] is the time of the reporting interval
 402                 # measures[-1][1] is the duration of the reporting interval
 403                 if aggregation:
 404                     # If this is an aggregate, we need to divide the total over the reported time period.
 405                     # Even if the aggregation method is not supported by Openstack, the code will execute it
 406                     # because aggregation is specified in METRIC_AGGREGATORS
 407                     value = value / measures[-1][1]
 408                 if openstack_metric_name in METRIC_MULTIPLIERS:
 409                     value = value * METRIC_MULTIPLIERS[openstack_metric_name]
 410         except gnocchiclient.exceptions.NotFound as e:
 411             log.debug(
 412                 "No metric %s found for instance %s: %s",
 413                 openstack_metric_name,
 414                 resource_id,
 415                 e,
 416             )
 417         return value
 418
 419
 420 class CeilometerBackend(OpenstackBackend):
 421     def __init__(self, vim_account: dict, vim_session: object):
 422         self.client = self._build_ceilometer_client(vim_account, vim_session)
 423
 424     def _build_ceilometer_client(
 425         self, vim_account: dict, vim_session: object
 426     ) -> ceilometer_client.Client:
 427         return ceilometer_client.Client("2", session=vim_session)
 428
 429     def collect_metric(
 430         self, metric_type: MetricType, metric_name: str, resource_id: str
 431     ):
 432         if metric_type != MetricType.INSTANCE:
 433             raise NotImplementedError(
 434                 "Ceilometer backend only support instance metrics"
 435             )
 436         measures = self.client.samples.list(
 437             meter_name=metric_name,
 438             limit=1,
 439             q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
 440         )
 441         return measures[0].counter_volume if measures else None
 442
 443
 444 class VropsBackend(OpenstackBackend):
 445     def __init__(self, vim_account: dict):
 446         self.vrops = vROPS_Helper(
 447             vrops_site=vim_account["config"]["vrops_site"],
 448             vrops_user=vim_account["config"]["vrops_user"],
 449             vrops_password=vim_account["config"]["vrops_password"],
 450         )
 451
 452     def collect_metrics(self, metrics_list: List[Dict]):
 453         # Fetch the list of all known resources from vROPS.
 454         resource_list = self.vrops.get_vm_resource_list_from_vrops()
 455
 456         vdu_mappings = {}
 457         extended_metrics = []
 458         for metric in metrics_list:
 459             vim_id = metric["vm_id"]
 460             # Map the vROPS instance id to the vim-id so we can look it up.
 461             for resource in resource_list:
 462                 for resourceIdentifier in resource["resourceKey"][
 463                     "resourceIdentifiers"
 464                 ]:
 465                     if (
 466                         resourceIdentifier["identifierType"]["name"]
 467                         == "VMEntityInstanceUUID"
 468                     ):
 469                         if resourceIdentifier["value"] != vim_id:
 470                             continue
 471                         vdu_mappings[vim_id] = resource["identifier"]
 472             if vim_id in vdu_mappings:
 473                 metric["vrops_id"] = vdu_mappings[vim_id]
 474                 extended_metrics.append(metric)
 475
 476         if len(extended_metrics) != 0:
 477             return self.vrops.get_metrics(extended_metrics)
 478         else:
 479             return []