src/osm_ngsa/osm_mon/vim_connectors/azure.py

   1 #######################################################################################
   2 # Copyright ETSI Contributors and Others.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #    http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  13 # implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #######################################################################################
  17 import datetime
  18 import logging
  19 from typing import Dict, List
  20
  21 from azure.identity import ClientSecretCredential
  22 from azure.mgmt.compute import ComputeManagementClient
  23 from azure.mgmt.monitor import MonitorManagementClient
  24 from azure.profiles import ProfileDefinition
  25 from osm_mon.vim_connectors.base_vim import VIMConnector
  26
  27
  28 log = logging.getLogger(__name__)
  29
  30
  31 METRIC_MAPPINGS = {
  32     "cpu_utilization": {
  33         "metricname": "Percentage CPU",
  34         "aggregation": "Average",
  35     },
  36     "disk_read_ops": {
  37         "metricname": "Disk Read Operations/Sec",
  38         "aggregation": "Average",
  39     },
  40     "disk_write_ops": {
  41         "metricname": "Disk Write Operations/Sec",
  42         "aggregation": "Average",
  43     },
  44     "disk_read_bytes": {
  45         "metricname": "Disk Read Bytes",
  46         "aggregation": "Total",
  47     },
  48     "disk_write_bytes": {
  49         "metricname": "Disk Write Bytes",
  50         "aggregation": "Total",
  51     },
  52     # "average_memory_utilization": {},
  53     # "packets_in_dropped": {},
  54     # "packets_out_dropped": {},
  55     # "packets_received": {},
  56     # "packets_sent": {},
  57 }
  58
  59
  60 class AzureCollector(VIMConnector):
  61     # Translate azure provisioning state to OSM provision state.
  62     # The first three ones are the transitional status once a user initiated
  63     # action has been requested. Once the operation is complete, it will
  64     # transition into the states Succeeded or Failed
  65     # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
  66     provision_state2osm = {
  67         "Creating": "BUILD",
  68         "Updating": "BUILD",
  69         "Deleting": "INACTIVE",
  70         "Succeeded": "ACTIVE",
  71         "Failed": "ERROR",
  72     }
  73
  74     # Translate azure power state to OSM provision state
  75     power_state2osm = {
  76         "starting": "INACTIVE",
  77         "running": "ACTIVE",
  78         "stopping": "INACTIVE",
  79         "stopped": "INACTIVE",
  80         "unknown": "OTHER",
  81         "deallocated": "BUILD",
  82         "deallocating": "BUILD",
  83     }
  84
  85     AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01"
  86     AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient"
  87     AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition(
  88         {
  89             AZURE_COMPUTE_MGMT_PROFILE_TAG: {
  90                 None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION,
  91                 "availability_sets": "2020-12-01",
  92                 "dedicated_host_groups": "2020-12-01",
  93                 "dedicated_hosts": "2020-12-01",
  94                 "disk_accesses": "2020-12-01",
  95                 "disk_encryption_sets": "2020-12-01",
  96                 "disk_restore_point": "2020-12-01",
  97                 "disks": "2020-12-01",
  98                 "galleries": "2020-09-30",
  99                 "gallery_application_versions": "2020-09-30",
 100                 "gallery_applications": "2020-09-30",
 101                 "gallery_image_versions": "2020-09-30",
 102                 "gallery_images": "2020-09-30",
 103                 "gallery_sharing_profile": "2020-09-30",
 104                 "images": "2020-12-01",
 105                 "log_analytics": "2020-12-01",
 106                 "operations": "2020-12-01",
 107                 "proximity_placement_groups": "2020-12-01",
 108                 "resource_skus": "2019-04-01",
 109                 "shared_galleries": "2020-09-30",
 110                 "shared_gallery_image_versions": "2020-09-30",
 111                 "shared_gallery_images": "2020-09-30",
 112                 "snapshots": "2020-12-01",
 113                 "ssh_public_keys": "2020-12-01",
 114                 "usage": "2020-12-01",
 115                 "virtual_machine_extension_images": "2020-12-01",
 116                 "virtual_machine_extensions": "2020-12-01",
 117                 "virtual_machine_images": "2020-12-01",
 118                 "virtual_machine_images_edge_zone": "2020-12-01",
 119                 "virtual_machine_run_commands": "2020-12-01",
 120                 "virtual_machine_scale_set_extensions": "2020-12-01",
 121                 "virtual_machine_scale_set_rolling_upgrades": "2020-12-01",
 122                 "virtual_machine_scale_set_vm_extensions": "2020-12-01",
 123                 "virtual_machine_scale_set_vm_run_commands": "2020-12-01",
 124                 "virtual_machine_scale_set_vms": "2020-12-01",
 125                 "virtual_machine_scale_sets": "2020-12-01",
 126                 "virtual_machine_sizes": "2020-12-01",
 127                 "virtual_machines": "2020-12-01",
 128             }
 129         },
 130         AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm",
 131     )
 132
 133     def __init__(self, vim_account: Dict):
 134         self.vim_account = vim_account
 135         self.reload_client = True
 136
 137         # Store config to create azure subscription later
 138         self._config = {
 139             "user": vim_account["vim_user"],
 140             "passwd": vim_account["vim_password"],
 141             "tenant": vim_account["vim_tenant_name"],
 142         }
 143
 144         # SUBSCRIPTION
 145         config = vim_account["config"]
 146         if "subscription_id" in config:
 147             self._config["subscription_id"] = config.get("subscription_id")
 148             log.info("Subscription: %s", self._config["subscription_id"])
 149         else:
 150             log.error("Subscription not specified")
 151             return
 152
 153         # RESOURCE_GROUP
 154         if "resource_group" in config:
 155             self.resource_group = config.get("resource_group")
 156         else:
 157             log.error("Azure resource_group is not specified at config")
 158             return
 159
 160     def _reload_connection(self):
 161         if self.reload_client:
 162             log.debug("reloading azure client")
 163             try:
 164                 self.credentials = ClientSecretCredential(
 165                     client_id=self._config["user"],
 166                     client_secret=self._config["passwd"],
 167                     tenant_id=self._config["tenant"],
 168                 )
 169                 self.conn_compute = ComputeManagementClient(
 170                     self.credentials,
 171                     self._config["subscription_id"],
 172                     profile=self.AZURE_COMPUTE_MGMT_PROFILE,
 173                 )
 174                 # create client
 175                 self.conn_monitor = MonitorManagementClient(
 176                     self.credentials,
 177                     self._config["subscription_id"],
 178                 )
 179                 # Set to client created
 180                 self.reload_client = False
 181             except Exception as e:
 182                 log.error(e)
 183
 184     def collect_servers_status(self) -> List[Dict]:
 185         servers = []
 186         log.debug("collect_servers_status")
 187         self._reload_connection()
 188         try:
 189             for vm in self.conn_compute.virtual_machines.list(self.resource_group):
 190                 id = vm.id
 191                 array = id.split("/")
 192                 name = array[-1]
 193                 status = self.provision_state2osm.get(vm.provisioning_state, "OTHER")
 194                 if vm.provisioning_state == "Succeeded":
 195                     # check if machine is running or stopped
 196                     instance_view = self.conn_compute.virtual_machines.instance_view(
 197                         self.resource_group, name
 198                     )
 199                     for status in instance_view.statuses:
 200                         splitted_status = status.code.split("/")
 201                         if (
 202                             len(splitted_status) == 2
 203                             and splitted_status[0] == "PowerState"
 204                         ):
 205                             status = self.power_state2osm.get(
 206                                 splitted_status[1], "OTHER"
 207                             )
 208                 # log.info(f'id: {id}, name: {name}, status: {status}')
 209                 vm = {
 210                     "id": id,
 211                     "name": name,
 212                     "status": (1 if (status == "ACTIVE") else 0),
 213                 }
 214                 servers.append(vm)
 215         except Exception as e:
 216             log.error(e)
 217         return servers
 218
 219     def is_vim_ok(self) -> bool:
 220         status = False
 221         self.reload_client = True
 222         try:
 223             self._reload_connection()
 224             status = True
 225         except Exception as e:
 226             log.error(e)
 227         return status
 228
 229     def collect_metrics(self, metric_list: List[Dict]) -> List[Dict]:
 230         log.debug("collect_metrics")
 231         self._reload_connection()
 232
 233         metric_results = []
 234         log.info(metric_list)
 235         for metric in metric_list:
 236             server = metric["vm_id"]
 237             metric_name = metric["metric"]
 238             metric_mapping = METRIC_MAPPINGS.get(metric_name)
 239             if not metric_mapping:
 240                 # log.info(f"Metric {metric_name} not available in Azure")
 241                 continue
 242             azure_metric_name = metric_mapping["metricname"]
 243             azure_aggregation = metric_mapping["aggregation"]
 244             end = datetime.datetime.now()
 245             init = end - datetime.timedelta(minutes=5)
 246             try:
 247                 metrics_data = self.conn_monitor.metrics.list(
 248                     server,
 249                     timespan="{}/{}".format(init, end),
 250                     interval="PT1M",
 251                     metricnames=azure_metric_name,
 252                     aggregation=azure_aggregation,
 253                 )
 254             except Exception as e:
 255                 log.error(e)
 256                 continue
 257             total = 0
 258             n_metrics = 0
 259             for item in metrics_data.value:
 260                 log.info("{} ({})".format(item.name.localized_value, item.unit))
 261                 for timeserie in item.timeseries:
 262                     for data in timeserie.data:
 263                         if azure_aggregation == "Average":
 264                             val = data.average
 265                         elif azure_aggregation == "Total":
 266                             val = data.total
 267                         else:
 268                             val = None
 269                         log.info("{}: {}".format(data.time_stamp, val))
 270                         if val is not None:
 271                             total += val
 272                             n_metrics += 1
 273             if n_metrics > 0:
 274                 value = total / n_metrics
 275                 log.info(f"value = {value}")
 276                 metric["value"] = value
 277                 metric_results.append(metric)
 278             else:
 279                 log.info("No metric available")
 280
 281         return metric_results