src/osm_ngsa/osm_mon/vim_connectors/azure.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 import datetime
  18 import logging
  19 from typing import Dict, List
  20
  21 from azure.identity import ClientSecretCredential
  22 from azure.mgmt.compute import ComputeManagementClient
  23 from azure.mgmt.monitor import MonitorManagementClient
  24 from azure.profiles import ProfileDefinition
  25 from osm_mon.vim_connectors.base_vim import VIMConnector
  26
  27
  28 log = logging.getLogger(__name__)
  29
  30
  31 METRIC_MAPPINGS = {
  32     "cpu_utilization": {
  33         "metricname": "Percentage CPU",
  34         "aggregation": "Average",
  35     },
  36     "disk_read_ops": {
  37         "metricname": "Disk Read Operations/Sec",
  38         "aggregation": "Average",
  39     },
  40     "disk_write_ops": {
  41         "metricname": "Disk Write Operations/Sec",
  42         "aggregation": "Average",
  43     },
  44     "disk_read_bytes": {
  45         "metricname": "Disk Read Bytes",
  46         "aggregation": "Total",
  47     },
  48     "disk_write_bytes": {
  49         "metricname": "Disk Write Bytes",
  50         "aggregation": "Total",
  51     },
  52     # "average_memory_utilization": {},
  53     # "packets_in_dropped": {},
  54     # "packets_out_dropped": {},
  55     # "packets_received": {},
  56     # "packets_sent": {},
  57 }
  58
  59
  60 class AzureCollector(VIMConnector):
  61     # Translate azure provisioning state to OSM provision state.
  62     # The first three ones are the transitional status once a user initiated
  63     # action has been requested. Once the operation is complete, it will
  64     # transition into the states Succeeded or Failed
  65     # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
  66     provision_state2osm = {
  67         "Creating": "BUILD",
  68         "Updating": "BUILD",
  69         "Deleting": "INACTIVE",
  70         "Succeeded": "ACTIVE",
  71         "Failed": "ERROR",
  72     }
  73
  74     # Translate azure power state to OSM provision state
  75     power_state2osm = {
  76         "starting": "INACTIVE",
  77         "running": "ACTIVE",
  78         "stopping": "INACTIVE",
  79         "stopped": "INACTIVE",
  80         "unknown": "OTHER",
  81         "deallocated": "BUILD",
  82         "deallocating": "BUILD",
  83     }
  84
  85     AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01"
  86     AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient"
  87     AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition(
  88         {
  89             AZURE_COMPUTE_MGMT_PROFILE_TAG: {
  90                 None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION,
  91                 "availability_sets": "2020-12-01",
  92                 "dedicated_host_groups": "2020-12-01",
  93                 "dedicated_hosts": "2020-12-01",
  94                 "disk_accesses": "2020-12-01",
  95                 "disk_encryption_sets": "2020-12-01",
  96                 "disk_restore_point": "2020-12-01",
  97                 "disks": "2020-12-01",
  98                 "galleries": "2020-09-30",
  99                 "gallery_application_versions": "2020-09-30",
 100                 "gallery_applications": "2020-09-30",
 101                 "gallery_image_versions": "2020-09-30",
 102                 "gallery_images": "2020-09-30",
 103                 "gallery_sharing_profile": "2020-09-30",
 104                 "images": "2020-12-01",
 105                 "log_analytics": "2020-12-01",
 106                 "operations": "2020-12-01",
 107                 "proximity_placement_groups": "2020-12-01",
 108                 "resource_skus": "2019-04-01",
 109                 "shared_galleries": "2020-09-30",
 110                 "shared_gallery_image_versions": "2020-09-30",
 111                 "shared_gallery_images": "2020-09-30",
 112                 "snapshots": "2020-12-01",
 113                 "ssh_public_keys": "2020-12-01",
 114                 "usage": "2020-12-01",
 115                 "virtual_machine_extension_images": "2020-12-01",
 116                 "virtual_machine_extensions": "2020-12-01",
 117                 "virtual_machine_images": "2020-12-01",
 118                 "virtual_machine_images_edge_zone": "2020-12-01",
 119                 "virtual_machine_run_commands": "2020-12-01",
 120                 "virtual_machine_scale_set_extensions": "2020-12-01",
 121                 "virtual_machine_scale_set_rolling_upgrades": "2020-12-01",
 122                 "virtual_machine_scale_set_vm_extensions": "2020-12-01",
 123                 "virtual_machine_scale_set_vm_run_commands": "2020-12-01",
 124                 "virtual_machine_scale_set_vms": "2020-12-01",
 125                 "virtual_machine_scale_sets": "2020-12-01",
 126                 "virtual_machine_sizes": "2020-12-01",
 127                 "virtual_machines": "2020-12-01",
 128             }
 129         },
 130         AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm",
 131     )
 132
 133     def __init__(self, vim_account: Dict):
 134         self.vim_account = vim_account
 135         self.reload_client = True
 136         logger = logging.getLogger("azure")
 137         logger.setLevel(logging.ERROR)
 138         # Store config to create azure subscription later
 139         self._config = {
 140             "user": vim_account["vim_user"],
 141             "passwd": vim_account["vim_password"],
 142             "tenant": vim_account["vim_tenant_name"],
 143         }
 144
 145         # SUBSCRIPTION
 146         config = vim_account["config"]
 147         if "subscription_id" in config:
 148             self._config["subscription_id"] = config.get("subscription_id")
 149             log.info("Subscription: %s", self._config["subscription_id"])
 150         else:
 151             log.error("Subscription not specified")
 152             return
 153
 154         # RESOURCE_GROUP
 155         if "resource_group" in config:
 156             self.resource_group = config.get("resource_group")
 157         else:
 158             log.error("Azure resource_group is not specified at config")
 159             return
 160
 161     def _reload_connection(self):
 162         if self.reload_client:
 163             log.debug("reloading azure client")
 164             try:
 165                 self.credentials = ClientSecretCredential(
 166                     client_id=self._config["user"],
 167                     client_secret=self._config["passwd"],
 168                     tenant_id=self._config["tenant"],
 169                 )
 170                 self.conn_compute = ComputeManagementClient(
 171                     self.credentials,
 172                     self._config["subscription_id"],
 173                     profile=self.AZURE_COMPUTE_MGMT_PROFILE,
 174                 )
 175                 # create client
 176                 self.conn_monitor = MonitorManagementClient(
 177                     self.credentials,
 178                     self._config["subscription_id"],
 179                 )
 180                 # Set to client created
 181                 self.reload_client = False
 182             except Exception as e:
 183                 log.error(e)
 184
 185     def collect_servers_status(self) -> List[Dict]:
 186         servers = []
 187         log.debug("collect_servers_status")
 188         self._reload_connection()
 189         try:
 190             for vm in self.conn_compute.virtual_machines.list(self.resource_group):
 191                 id = vm.id
 192                 array = id.split("/")
 193                 name = array[-1]
 194                 status = self.provision_state2osm.get(vm.provisioning_state, "OTHER")
 195                 if vm.provisioning_state == "Succeeded":
 196                     # check if machine is running or stopped
 197                     instance_view = self.conn_compute.virtual_machines.instance_view(
 198                         self.resource_group, name
 199                     )
 200                     for status in instance_view.statuses:
 201                         splitted_status = status.code.split("/")
 202                         if (
 203                             len(splitted_status) == 2
 204                             and splitted_status[0] == "PowerState"
 205                         ):
 206                             status = self.power_state2osm.get(
 207                                 splitted_status[1], "OTHER"
 208                             )
 209                 # log.info(f'id: {id}, name: {name}, status: {status}')
 210                 vm = {
 211                     "id": id,
 212                     "name": name,
 213                     "status": (1 if (status == "ACTIVE") else 0),
 214                 }
 215                 servers.append(vm)
 216         except Exception as e:
 217             log.error(e)
 218         return servers
 219
 220     def is_vim_ok(self) -> bool:
 221         status = False
 222         self.reload_client = True
 223         try:
 224             self._reload_connection()
 225             status = True
 226         except Exception as e:
 227             log.error(e)
 228         return status
 229
 230     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 231         log.debug("collect_metrics")
 232         self._reload_connection()
 233
 234         metric_results = []
 235         log.info(metric_list)
 236         for metric in metric_list:
 237             server = metric["vm_id"]
 238             metric_name = metric["metric"]
 239             metric_mapping = METRIC_MAPPINGS.get(metric_name)
 240             if not metric_mapping:
 241                 # log.info(f"Metric {metric_name} not available in Azure")
 242                 continue
 243             azure_metric_name = metric_mapping["metricname"]
 244             azure_aggregation = metric_mapping["aggregation"]
 245             end = datetime.datetime.now()
 246             init = end - datetime.timedelta(minutes=5)
 247             try:
 248                 metrics_data = self.conn_monitor.metrics.list(
 249                     server,
 250                     timespan="{}/{}".format(init, end),
 251                     interval="PT1M",
 252                     metricnames=azure_metric_name,
 253                     aggregation=azure_aggregation,
 254                 )
 255             except Exception as e:
 256                 log.error(e)
 257                 continue
 258             total = 0
 259             n_metrics = 0
 260             for item in metrics_data.value:
 261                 log.info("{} ({})".format(item.name.localized_value, item.unit))
 262                 for timeserie in item.timeseries:
 263                     for data in timeserie.data:
 264                         if azure_aggregation == "Average":
 265                             val = data.average
 266                         elif azure_aggregation == "Total":
 267                             val = data.total
 268                         else:
 269                             val = None
 270                         log.info("{}: {}".format(data.time_stamp, val))
 271                         if val is not None:
 272                             total += val
 273                             n_metrics += 1
 274             if n_metrics > 0:
 275                 value = total / n_metrics
 276                 log.info(f"value = {value}")
 277                 metric["value"] = value
 278                 metric_results.append(metric)
 279             else:
 280                 log.info("No metric available")
 281
 282         return metric_results