src/osm_ngsa/osm_mon/vim_connectors/gcp.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 # pylint: disable=E1101
  18
  19 import logging
  20 import time
  21 from typing import Dict, List
  22
  23 from google.cloud import monitoring_v3
  24 from google.oauth2 import service_account
  25 import googleapiclient.discovery
  26 from osm_mon.vim_connectors.base_vim import VIMConnector
  27
  28 log = logging.getLogger(__name__)
  29
  30
  31 METRIC_MAPPINGS = {
  32     "cpu_utilization": {
  33         "metrictype": "compute.googleapis.com/instance/cpu/utilization",
  34         "multiplier": 100,
  35     },
  36     "average_memory_utilization": {
  37         # metric only available in e2 family
  38         "metrictype": "compute.googleapis.com/instance/memory/balloon/ram_used",
  39         "multiplier": 0.000001,
  40     },
  41     "disk_read_ops": {
  42         "metrictype": "compute.googleapis.com/instance/disk/read_ops_count",
  43     },
  44     "disk_write_ops": {
  45         "metrictype": "compute.googleapis.com/instance/disk/write_ops_count",
  46     },
  47     "disk_read_bytes": {
  48         "metrictype": "compute.googleapis.com/instance/disk/read_bytes_count",
  49     },
  50     "disk_write_bytes": {
  51         "metrictype": "compute.googleapis.com/instance/disk/write_bytes_count",
  52     },
  53     "packets_received": {
  54         "metrictype": "compute.googleapis.com/instance/network/received_packets_count",
  55     },
  56     "packets_sent": {
  57         "metrictype": "compute.googleapis.com/instance/network/sent_packets_count",
  58     },
  59     # "packets_in_dropped": {},
  60     # "packets_out_dropped": {},
  61 }
  62
  63
  64 class GcpCollector(VIMConnector):
  65     def __init__(self, vim_account: Dict):
  66         self.vim_account = vim_account
  67         self.project = vim_account["vim_tenant_name"] or vim_account["vim_tenant_id"]
  68
  69         # REGION - Google Cloud considers regions and zones. A specific region
  70         # can have more than one zone (for instance: region us-west1 with the
  71         # zones us-west1-a, us-west1-b and us-west1-c). So the region name
  72         # specified in the config will be considered as a specific zone for GC
  73         # and the region will be calculated from that without the preffix.
  74         if "config" in vim_account:
  75             config = vim_account["config"]
  76             if "region_name" in config:
  77                 self.zone = config.get("region_name")
  78                 self.region = self.zone.rsplit("-", 1)[0]
  79             else:
  80                 log.error("Google Cloud region_name not specified in config")
  81         else:
  82             log.error("config is not specified in VIM")
  83
  84         # Credentials
  85         scopes = ["https://www.googleapis.com/auth/cloud-platform"]
  86         self.credentials = None
  87         if "credentials" in config:
  88             log.debug("Setting credentials")
  89             # Settings Google Cloud credentials dict
  90             creds_body = config["credentials"]
  91             creds = service_account.Credentials.from_service_account_info(creds_body)
  92             if "sa_file" in config:
  93                 creds = service_account.Credentials.from_service_account_file(
  94                     config.get("sa_file"), scopes=scopes
  95                 )
  96                 log.debug("Credentials: %s", creds)
  97             # Construct a Resource for interacting with an API.
  98             self.credentials = creds
  99             try:
 100                 self.conn_compute = googleapiclient.discovery.build(
 101                     "compute", "v1", credentials=creds
 102                 )
 103             except Exception as e:
 104                 log.error(e)
 105             # Construct a client for interacting with metrics API.
 106             try:
 107                 self.metric_client = monitoring_v3.MetricServiceClient(
 108                     credentials=creds
 109                 )
 110             except Exception as e:
 111                 log.error(e)
 112         else:
 113             log.error("It is not possible to init GCP with no credentials")
 114
 115     def collect_servers_status(self) -> List[Dict]:
 116         servers = []
 117         try:
 118             response = (
 119                 self.conn_compute.instances()
 120                 .list(project=self.project, zone=self.zone)
 121                 .execute()
 122             )
 123             if "items" in response:
 124                 log.info(response["items"])
 125                 for server in response["items"]:
 126                     vm = {
 127                         "id": server["name"],
 128                         "name": server["name"],
 129                         "status": (1 if (server["status"] == "RUNNING") else 0),
 130                     }
 131                     servers.append(vm)
 132         except Exception as e:
 133             log.error(e)
 134         return servers
 135
 136     def is_vim_ok(self) -> bool:
 137         status = False
 138         try:
 139             self.conn_compute.zones().get(
 140                 project=self.project, zone=self.zone
 141             ).execute()
 142             status = True
 143         except Exception as e:
 144             log.error(e)
 145         return status
 146
 147     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 148         log.debug("collect_metrics")
 149
 150         metric_results = []
 151         log.info(metric_list)
 152         for metric in metric_list:
 153             server = metric["vm_id"]
 154             metric_name = metric["metric"]
 155             metric_mapping = METRIC_MAPPINGS.get(metric_name)
 156             if not metric_mapping:
 157                 # log.info(f"Metric {metric_name} not available in GCP")
 158                 continue
 159             gcp_metric_type = metric_mapping["metrictype"]
 160             metric_multiplier = metric_mapping.get("multiplier", 1)
 161             log.info(f"server: {server}, gcp_metric_type: {gcp_metric_type}")
 162
 163             end = int(time.time())
 164             start = end - 600
 165             interval = monitoring_v3.TimeInterval(
 166                 {
 167                     "end_time": {"seconds": end},
 168                     "start_time": {"seconds": start},
 169                 }
 170             )
 171             aggregation = monitoring_v3.Aggregation(
 172                 {
 173                     "alignment_period": {"seconds": 600},
 174                     "per_series_aligner": monitoring_v3.Aggregation.Aligner.ALIGN_MEAN,
 175                 }
 176             )
 177             filter = f'metric.type = "{gcp_metric_type}" AND metric.labels.instance_name = "{server}"'
 178             project = f"projects/{self.project}"
 179             log.info(f"filter: {filter}")
 180             results = self.metric_client.list_time_series(
 181                 request={
 182                     "name": project,
 183                     "filter": filter,
 184                     "interval": interval,
 185                     "view": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,
 186                     "aggregation": aggregation,
 187                 }
 188             )
 189             value = None
 190             for result in results:
 191                 for point in result.points:
 192                     value = point.value.double_value
 193             if value is not None:
 194                 metric["value"] = value * metric_multiplier
 195                 log.info(f'value: {metric["value"]}')
 196                 metric_results.append(metric)
 197
 198         return metric_results