src/osm_ngsa/osm_mon/vim_connectors/openstack.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 from enum import Enum
  18 import logging
  19 import time
  20 from typing import Dict, List
  21
  22 from ceilometerclient import client as ceilometer_client
  23 from ceilometerclient.exc import HTTPException
  24 import gnocchiclient.exceptions
  25 from gnocchiclient.v1 import client as gnocchi_client
  26 from keystoneauth1 import session
  27 from keystoneauth1.exceptions.catalog import EndpointNotFound
  28 from keystoneauth1.identity import v3
  29 from novaclient import client as nova_client
  30 from osm_mon.vim_connectors.base_vim import VIMConnector
  31 from prometheus_api_client import PrometheusConnect as prometheus_client
  32
  33 log = logging.getLogger(__name__)
  34
  35 METRIC_MULTIPLIERS = {"cpu": 0.0000001}
  36
  37 METRIC_AGGREGATORS = {"cpu": "rate:mean"}
  38
  39 INTERFACE_METRICS = [
  40     "packets_in_dropped",
  41     "packets_out_dropped",
  42     "packets_received",
  43     "packets_sent",
  44 ]
  45
  46 INSTANCE_DISK = [
  47     "disk_read_ops",
  48     "disk_write_ops",
  49     "disk_read_bytes",
  50     "disk_write_bytes",
  51 ]
  52
  53 METRIC_MAPPINGS = {
  54     "average_memory_utilization": "memory.usage",
  55     "disk_read_ops": "disk.device.read.requests",
  56     "disk_write_ops": "disk.device.write.requests",
  57     "disk_read_bytes": "disk.device.read.bytes",
  58     "disk_write_bytes": "disk.device.write.bytes",
  59     "packets_in_dropped": "network.outgoing.packets.drop",
  60     "packets_out_dropped": "network.incoming.packets.drop",
  61     "packets_received": "network.incoming.packets",
  62     "packets_sent": "network.outgoing.packets",
  63     "cpu_utilization": "cpu",
  64 }
  65
  66 METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD = {
  67     "cpu_utilization": "cpu",
  68     "average_memory_utilization": "memory_usage",
  69     "disk_read_ops": "disk_device_read_requests",
  70     "disk_write_ops": "disk_device_write_requests",
  71     "disk_read_bytes": "disk_device_read_bytes",
  72     "disk_write_bytes": "disk_device_write_bytes",
  73     "packets_in_dropped": "network_incoming_packets_drop",
  74     "packets_out_dropped": "network_outgoing_packets_drop",
  75     "packets_received": "network_incoming_packets",
  76     "packets_sent": "network_outgoing_packets",
  77 }
  78
  79
  80 class MetricType(Enum):
  81     INSTANCE = "instance"
  82     INTERFACE_ALL = "interface_all"
  83     INTERFACE_ONE = "interface_one"
  84     INSTANCEDISK = "instancedisk"
  85
  86
  87 class CertificateNotCreated(Exception):
  88     pass
  89
  90
  91 class OpenStackCollector(VIMConnector):
  92     def __init__(self, vim_account: Dict):
  93         log.debug("__init__")
  94         self.vim_account = vim_account
  95         self.vim_session = None
  96         self.vim_session = self._get_session(vim_account)
  97         self.nova = self._build_nova_client()
  98         # self.gnocchi = self._build_gnocchi_client()
  99         self.backend = self._get_backend(vim_account, self.vim_session)
 100
 101     def _get_session(self, creds: Dict):
 102         log.debug("_get_session")
 103         verify_ssl = True
 104         project_domain_name = "Default"
 105         user_domain_name = "Default"
 106         try:
 107             if "config" in creds:
 108                 vim_config = creds["config"]
 109                 if "insecure" in vim_config and vim_config["insecure"]:
 110                     verify_ssl = False
 111                 if "ca_cert" in vim_config:
 112                     verify_ssl = vim_config["ca_cert"]
 113                 elif "ca_cert_content" in vim_config:
 114                     # vim_config = self._create_file_cert(vim_config, creds["_id"])
 115                     verify_ssl = vim_config["ca_cert"]
 116                 if "project_domain_name" in vim_config:
 117                     project_domain_name = vim_config["project_domain_name"]
 118                 if "user_domain_name" in vim_config:
 119                     user_domain_name = vim_config["user_domain_name"]
 120             auth = v3.Password(
 121                 auth_url=creds["vim_url"],
 122                 username=creds["vim_user"],
 123                 password=creds["vim_password"],
 124                 project_name=creds["vim_tenant_name"],
 125                 project_domain_name=project_domain_name,
 126                 user_domain_name=user_domain_name,
 127             )
 128             return session.Session(auth=auth, verify=verify_ssl, timeout=10)
 129         except CertificateNotCreated as e:
 130             log.error(e)
 131
 132     def _get_backend(self, vim_account: dict, vim_session: object):
 133         if vim_account.get("prometheus-config"):
 134             # try:
 135             #     tsbd = PrometheusTSBDBackend(vim_account)
 136             #     log.debug("Using prometheustsbd backend to collect metric")
 137             #     return tsbd
 138             # except Exception as e:
 139             #     log.error(f"Can't create prometheus client, {e}")
 140             #     return None
 141             return None
 142         try:
 143             gnocchi = GnocchiBackend(vim_account, vim_session)
 144             gnocchi.client.metric.list(limit=1)
 145             log.debug("Using gnocchi backend to collect metric")
 146             return gnocchi
 147         except (HTTPException, EndpointNotFound):
 148             ceilometer = CeilometerBackend(vim_account, vim_session)
 149             ceilometer.client.capabilities.get()
 150             log.debug("Using ceilometer backend to collect metric")
 151             return ceilometer
 152
 153     def _build_nova_client(self) -> nova_client.Client:
 154         return nova_client.Client("2", session=self.vim_session, timeout=10)
 155
 156     def _build_gnocchi_client(self) -> gnocchi_client.Client:
 157         return gnocchi_client.Client(session=self.vim_session)
 158
 159     def collect_servers_status(self) -> List[Dict]:
 160         log.debug("collect_servers_status")
 161         servers = []
 162         for server in self.nova.servers.list(detailed=True):
 163             vm = {
 164                 "id": server.id,
 165                 "name": server.name,
 166                 "status": (0 if (server.status == "ERROR") else 1),
 167             }
 168             servers.append(vm)
 169         return servers
 170
 171     def is_vim_ok(self) -> bool:
 172         log.debug("is_vim_ok")
 173         try:
 174             self.nova.servers.list()
 175             return True
 176         except Exception as e:
 177             log.warning("VIM status is not OK: %s" % e)
 178             return False
 179
 180     def _get_metric_type(self, metric_name: str) -> MetricType:
 181         if metric_name not in INTERFACE_METRICS:
 182             if metric_name not in INSTANCE_DISK:
 183                 return MetricType.INSTANCE
 184             else:
 185                 return MetricType.INSTANCEDISK
 186         else:
 187             return MetricType.INTERFACE_ALL
 188
 189     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 190         log.debug("collect_metrics")
 191         if not self.backend:
 192             log.error("Undefined backend")
 193             return []
 194
 195         if type(self.backend) is PrometheusTSBDBackend:
 196             log.info("Using Prometheus as backend (NOT SUPPORTED)")
 197             return []
 198
 199         metric_results = []
 200         for metric in metric_list:
 201             server = metric["vm_id"]
 202             metric_name = metric["metric"]
 203             openstack_metric_name = METRIC_MAPPINGS[metric_name]
 204             metric_type = self._get_metric_type(metric_name)
 205             log.info(f"Collecting metric {openstack_metric_name} for {server}")
 206             try:
 207                 value = self.backend.collect_metric(
 208                     metric_type, openstack_metric_name, server
 209                 )
 210                 if value is not None:
 211                     log.info(f"value: {value}")
 212                     metric["value"] = value
 213                     metric_results.append(metric)
 214                 else:
 215                     log.info("metric value is empty")
 216             except Exception as e:
 217                 log.error("Error in metric collection: %s" % e)
 218         return metric_results
 219
 220
 221 class OpenstackBackend:
 222     def collect_metric(
 223         self, metric_type: MetricType, metric_name: str, resource_id: str
 224     ):
 225         pass
 226
 227
 228 class PrometheusTSBDBackend(OpenstackBackend):
 229     def __init__(self, vim_account: dict):
 230         self.map = self._build_map(vim_account)
 231         self.cred = vim_account["prometheus-config"].get("prometheus-cred")
 232         self.client = self._build_prometheus_client(
 233             vim_account["prometheus-config"]["prometheus-url"]
 234         )
 235
 236     def _build_prometheus_client(self, url: str) -> prometheus_client:
 237         return prometheus_client(url, disable_ssl=True)
 238
 239     def _build_map(self, vim_account: dict) -> dict:
 240         custom_map = METRIC_MAPPINGS_FOR_PROMETHEUS_TSBD
 241         if "prometheus-map" in vim_account["prometheus-config"]:
 242             custom_map.update(vim_account["prometheus-config"]["prometheus-map"])
 243         return custom_map
 244
 245     def collect_metric(
 246         self, metric_type: MetricType, metric_name: str, resource_id: str
 247     ):
 248         metric = self.query_metric(metric_name, resource_id)
 249         return metric["value"][1] if metric else None
 250
 251     def map_metric(self, metric_name: str):
 252         return self.map[metric_name]
 253
 254     def query_metric(self, metric_name, resource_id=None):
 255         metrics = self.client.get_current_metric_value(metric_name=metric_name)
 256         if resource_id:
 257             metric = next(
 258                 filter(lambda x: resource_id in x["metric"]["resource_id"], metrics)
 259             )
 260             return metric
 261         return metrics
 262
 263
 264 class GnocchiBackend(OpenstackBackend):
 265     def __init__(self, vim_account: dict, vim_session: object):
 266         self.client = self._build_gnocchi_client(vim_account, vim_session)
 267
 268     def _build_gnocchi_client(
 269         self, vim_account: dict, vim_session: object
 270     ) -> gnocchi_client.Client:
 271         return gnocchi_client.Client(session=vim_session)
 272
 273     def collect_metric(
 274         self, metric_type: MetricType, metric_name: str, resource_id: str
 275     ):
 276         if metric_type == MetricType.INTERFACE_ALL:
 277             return self._collect_interface_all_metric(metric_name, resource_id)
 278
 279         elif metric_type == MetricType.INSTANCE:
 280             return self._collect_instance_metric(metric_name, resource_id)
 281
 282         elif metric_type == MetricType.INSTANCEDISK:
 283             return self._collect_instance_disk_metric(metric_name, resource_id)
 284
 285         else:
 286             raise Exception("Unknown metric type %s" % metric_type.value)
 287
 288     def _collect_interface_all_metric(self, openstack_metric_name, resource_id):
 289         total_measure = None
 290         interfaces = self.client.resource.search(
 291             resource_type="instance_network_interface",
 292             query={"=": {"instance_id": resource_id}},
 293         )
 294         for interface in interfaces:
 295             try:
 296                 measures = self.client.metric.get_measures(
 297                     openstack_metric_name, resource_id=interface["id"], limit=1
 298                 )
 299                 if measures:
 300                     if not total_measure:
 301                         total_measure = 0.0
 302                     total_measure += measures[-1][2]
 303             except (gnocchiclient.exceptions.NotFound, TypeError) as e:
 304                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound
 305                 log.debug(
 306                     "No metric %s found for interface %s: %s",
 307                     openstack_metric_name,
 308                     interface["id"],
 309                     e,
 310                 )
 311         return total_measure
 312
 313     def _collect_instance_disk_metric(self, openstack_metric_name, resource_id):
 314         value = None
 315         instances = self.client.resource.search(
 316             resource_type="instance_disk",
 317             query={"=": {"instance_id": resource_id}},
 318         )
 319         for instance in instances:
 320             try:
 321                 measures = self.client.metric.get_measures(
 322                     openstack_metric_name, resource_id=instance["id"], limit=1
 323                 )
 324                 if measures:
 325                     value = measures[-1][2]
 326
 327             except gnocchiclient.exceptions.NotFound as e:
 328                 log.debug(
 329                     "No metric %s found for instance disk %s: %s",
 330                     openstack_metric_name,
 331                     instance["id"],
 332                     e,
 333                 )
 334         return value
 335
 336     def _collect_instance_metric(self, openstack_metric_name, resource_id):
 337         value = None
 338         try:
 339             aggregation = METRIC_AGGREGATORS.get(openstack_metric_name)
 340
 341             try:
 342                 measures = self.client.metric.get_measures(
 343                     openstack_metric_name,
 344                     aggregation=aggregation,
 345                     start=time.time() - 1200,
 346                     resource_id=resource_id,
 347                 )
 348                 if measures:
 349                     value = measures[-1][2]
 350             except (
 351                 gnocchiclient.exceptions.NotFound,
 352                 gnocchiclient.exceptions.BadRequest,
 353                 TypeError,
 354             ) as e:
 355                 # CPU metric in previous Openstack versions do not support rate:mean aggregation method
 356                 # Gnocchi in some Openstack versions raise TypeError instead of NotFound or BadRequest
 357                 if openstack_metric_name == "cpu":
 358                     log.debug(
 359                         "No metric %s found for instance %s: %s",
 360                         openstack_metric_name,
 361                         resource_id,
 362                         e,
 363                     )
 364                     log.info(
 365                         "Retrying to get metric %s for instance %s without aggregation",
 366                         openstack_metric_name,
 367                         resource_id,
 368                     )
 369                     measures = self.client.metric.get_measures(
 370                         openstack_metric_name, resource_id=resource_id, limit=1
 371                     )
 372                 else:
 373                     raise e
 374                 # measures[-1] is the last measure
 375                 # measures[-2] is the previous measure
 376                 # measures[x][2] is the value of the metric
 377                 if measures and len(measures) >= 2:
 378                     value = measures[-1][2] - measures[-2][2]
 379             if value:
 380                 # measures[-1][0] is the time of the reporting interval
 381                 # measures[-1][1] is the duration of the reporting interval
 382                 if aggregation:
 383                     # If this is an aggregate, we need to divide the total over the reported time period.
 384                     # Even if the aggregation method is not supported by Openstack, the code will execute it
 385                     # because aggregation is specified in METRIC_AGGREGATORS
 386                     value = value / measures[-1][1]
 387                 if openstack_metric_name in METRIC_MULTIPLIERS:
 388                     value = value * METRIC_MULTIPLIERS[openstack_metric_name]
 389         except gnocchiclient.exceptions.NotFound as e:
 390             log.debug(
 391                 "No metric %s found for instance %s: %s",
 392                 openstack_metric_name,
 393                 resource_id,
 394                 e,
 395             )
 396         return value
 397
 398
 399 class CeilometerBackend(OpenstackBackend):
 400     def __init__(self, vim_account: dict, vim_session: object):
 401         self.client = self._build_ceilometer_client(vim_account, vim_session)
 402
 403     def _build_ceilometer_client(
 404         self, vim_account: dict, vim_session: object
 405     ) -> ceilometer_client.Client:
 406         return ceilometer_client.Client("2", session=vim_session)
 407
 408     def collect_metric(
 409         self, metric_type: MetricType, metric_name: str, resource_id: str
 410     ):
 411         if metric_type != MetricType.INSTANCE:
 412             raise NotImplementedError(
 413                 "Ceilometer backend only support instance metrics"
 414             )
 415         measures = self.client.samples.list(
 416             meter_name=metric_name,
 417             limit=1,
 418             q=[{"field": "resource_id", "op": "eq", "value": resource_id}],
 419         )
 420         return measures[0].counter_volume if measures else None