RO-VIM-azure/osm_rovim_azure/vimconn_azure.py

   1 # -*- coding: utf-8 -*-
   2 ##
   3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 # not use this file except in compliance with the License. You may obtain
   5 # a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 # License for the specific language governing permissions and limitations
  13 # under the License.
  14 ##
  15
  16 import base64
  17 import logging
  18 from os import getenv
  19 import re
  20
  21 from azure.core.exceptions import ResourceNotFoundError
  22 from azure.identity import ClientSecretCredential
  23 from azure.mgmt.compute import ComputeManagementClient
  24 from azure.mgmt.network import NetworkManagementClient
  25 from azure.mgmt.resource import ResourceManagementClient
  26 from azure.profiles import ProfileDefinition
  27 from cryptography.hazmat.backends import default_backend as crypto_default_backend
  28 from cryptography.hazmat.primitives import serialization as crypto_serialization
  29 from cryptography.hazmat.primitives.asymmetric import rsa
  30 from msrest.exceptions import AuthenticationError
  31 from msrestazure.azure_exceptions import CloudError
  32 import msrestazure.tools as azure_tools
  33 import netaddr
  34 from osm_ro_plugin import vimconn
  35 from requests.exceptions import ConnectionError
  36
  37 __author__ = "Isabel Lloret, Sergio Gonzalez, Alfonso Tierno, Gerardo Garcia"
  38 __date__ = "$18-apr-2019 23:59:59$"
  39
  40
  41 if getenv("OSMRO_PDB_DEBUG"):
  42     import sys
  43
  44     print(sys.path)
  45     import pdb
  46
  47     pdb.set_trace()
  48
  49
  50 def find_in_list(the_list, condition_lambda):
  51     for item in the_list:
  52         if condition_lambda(item):
  53             return item
  54     else:
  55         return None
  56
  57
  58 class vimconnector(vimconn.VimConnector):
  59
  60     # Translate azure provisioning state to OSM provision state
  61     # The first three ones are the transitional status once a user initiated action has been requested
  62     # Once the operation is complete, it will transition into the states Succeeded or Failed
  63     # https://docs.microsoft.com/en-us/azure/virtual-machines/windows/states-lifecycle
  64     provision_state2osm = {
  65         "Creating": "BUILD",
  66         "Updating": "BUILD",
  67         "Deleting": "INACTIVE",
  68         "Succeeded": "ACTIVE",
  69         "Failed": "ERROR",
  70     }
  71
  72     # Translate azure power state to OSM provision state
  73     power_state2osm = {
  74         "starting": "INACTIVE",
  75         "running": "ACTIVE",
  76         "stopping": "INACTIVE",
  77         "stopped": "INACTIVE",
  78         "unknown": "OTHER",
  79         "deallocated": "BUILD",
  80         "deallocating": "BUILD",
  81     }
  82
  83     # TODO - review availability zones
  84     AZURE_ZONES = ["1", "2", "3"]
  85
  86     AZURE_COMPUTE_MGMT_CLIENT_API_VERSION = "2021-03-01"
  87     AZURE_COMPUTE_MGMT_PROFILE_TAG = "azure.mgmt.compute.ComputeManagementClient"
  88     AZURE_COMPUTE_MGMT_PROFILE = ProfileDefinition(
  89         {
  90             AZURE_COMPUTE_MGMT_PROFILE_TAG: {
  91                 None: AZURE_COMPUTE_MGMT_CLIENT_API_VERSION,
  92                 "availability_sets": "2020-12-01",
  93                 "dedicated_host_groups": "2020-12-01",
  94                 "dedicated_hosts": "2020-12-01",
  95                 "disk_accesses": "2020-12-01",
  96                 "disk_encryption_sets": "2020-12-01",
  97                 "disk_restore_point": "2020-12-01",
  98                 "disks": "2020-12-01",
  99                 "galleries": "2020-09-30",
 100                 "gallery_application_versions": "2020-09-30",
 101                 "gallery_applications": "2020-09-30",
 102                 "gallery_image_versions": "2020-09-30",
 103                 "gallery_images": "2020-09-30",
 104                 "gallery_sharing_profile": "2020-09-30",
 105                 "images": "2020-12-01",
 106                 "log_analytics": "2020-12-01",
 107                 "operations": "2020-12-01",
 108                 "proximity_placement_groups": "2020-12-01",
 109                 "resource_skus": "2019-04-01",
 110                 "shared_galleries": "2020-09-30",
 111                 "shared_gallery_image_versions": "2020-09-30",
 112                 "shared_gallery_images": "2020-09-30",
 113                 "snapshots": "2020-12-01",
 114                 "ssh_public_keys": "2020-12-01",
 115                 "usage": "2020-12-01",
 116                 "virtual_machine_extension_images": "2020-12-01",
 117                 "virtual_machine_extensions": "2020-12-01",
 118                 "virtual_machine_images": "2020-12-01",
 119                 "virtual_machine_images_edge_zone": "2020-12-01",
 120                 "virtual_machine_run_commands": "2020-12-01",
 121                 "virtual_machine_scale_set_extensions": "2020-12-01",
 122                 "virtual_machine_scale_set_rolling_upgrades": "2020-12-01",
 123                 "virtual_machine_scale_set_vm_extensions": "2020-12-01",
 124                 "virtual_machine_scale_set_vm_run_commands": "2020-12-01",
 125                 "virtual_machine_scale_set_vms": "2020-12-01",
 126                 "virtual_machine_scale_sets": "2020-12-01",
 127                 "virtual_machine_sizes": "2020-12-01",
 128                 "virtual_machines": "2020-12-01",
 129             }
 130         },
 131         AZURE_COMPUTE_MGMT_PROFILE_TAG + " osm",
 132     )
 133
 134     AZURE_RESOURCE_MGMT_CLIENT_API_VERSION = "2020-10-01"
 135     AZURE_RESOURCE_MGMT_PROFILE_TAG = (
 136         "azure.mgmt.resource.resources.ResourceManagementClient"
 137     )
 138     AZURE_RESOURCE_MGMT_PROFILE = ProfileDefinition(
 139         {
 140             AZURE_RESOURCE_MGMT_PROFILE_TAG: {
 141                 None: AZURE_RESOURCE_MGMT_CLIENT_API_VERSION,
 142             }
 143         },
 144         AZURE_RESOURCE_MGMT_PROFILE_TAG + " osm",
 145     )
 146
 147     AZURE_NETWORK_MGMT_CLIENT_API_VERSION = "2020-11-01"
 148     AZURE_NETWORK_MGMT_PROFILE_TAG = "azure.mgmt.network.NetworkManagementClient"
 149     AZURE_NETWORK_MGMT_PROFILE = ProfileDefinition(
 150         {
 151             AZURE_NETWORK_MGMT_PROFILE_TAG: {
 152                 None: AZURE_NETWORK_MGMT_CLIENT_API_VERSION,
 153                 "firewall_policy_rule_groups": "2020-04-01",
 154                 "interface_endpoints": "2019-02-01",
 155                 "p2_svpn_server_configurations": "2019-07-01",
 156             }
 157         },
 158         AZURE_NETWORK_MGMT_PROFILE_TAG + " osm",
 159     )
 160
 161     def __init__(
 162         self,
 163         uuid,
 164         name,
 165         tenant_id,
 166         tenant_name,
 167         url,
 168         url_admin=None,
 169         user=None,
 170         passwd=None,
 171         log_level=None,
 172         config={},
 173         persistent_info={},
 174     ):
 175         """
 176         Constructor of VIM. Raise an exception is some needed parameter is missing, but it must not do any connectivity
 177         checking against the VIM
 178         Using common constructor parameters.
 179         In this case: config must include the following parameters:
 180         subscription_id: assigned azure subscription identifier
 181         region_name: current region for azure network
 182         resource_group: used for all azure created resources
 183         vnet_name: base vnet for azure, created networks will be subnets from this base network
 184         config may also include the following parameter:
 185         flavors_pattern: pattern that will be used to select a range of vm sizes, for example
 186             "^((?!Standard_B).)*$" will filter out Standard_B range that is cheap but is very overused
 187             "^Standard_B" will select a serie B maybe for test environment
 188         """
 189         vimconn.VimConnector.__init__(
 190             self,
 191             uuid,
 192             name,
 193             tenant_id,
 194             tenant_name,
 195             url,
 196             url_admin,
 197             user,
 198             passwd,
 199             log_level,
 200             config,
 201             persistent_info,
 202         )
 203
 204         # Variable that indicates if client must be reloaded or initialized
 205         self.reload_client = True
 206
 207         self.vnet_address_space = None
 208
 209         # LOGGER
 210         self.logger = logging.getLogger("ro.vim.azure")
 211         if log_level:
 212             self.logger.setLevel(getattr(logging, log_level))
 213
 214         self.tenant = tenant_id or tenant_name
 215
 216         # Store config to create azure subscription later
 217         self._config = {
 218             "user": user,
 219             "passwd": passwd,
 220             "tenant": tenant_id or tenant_name,
 221         }
 222
 223         # SUBSCRIPTION
 224         if "subscription_id" in config:
 225             self._config["subscription_id"] = config.get("subscription_id")
 226             # self.logger.debug("Setting subscription to: %s", self.config["subscription_id"])
 227         else:
 228             raise vimconn.VimConnException("Subscription not specified")
 229
 230         # RESOURCE_GROUP
 231         if "resource_group" in config:
 232             self.resource_group = config.get("resource_group")
 233         else:
 234             raise vimconn.VimConnException(
 235                 "Azure resource_group is not specified at config"
 236             )
 237
 238         # REGION
 239         if "region_name" in config:
 240             self.region = config.get("region_name")
 241         else:
 242             raise vimconn.VimConnException(
 243                 "Azure region_name is not specified at config"
 244             )
 245
 246         # VNET_NAME
 247         if "vnet_name" in config:
 248             self.vnet_name = config["vnet_name"]
 249
 250         # VNET_RESOURCE_GROUP
 251         self.vnet_resource_group = config.get("vnet_resource_group")
 252
 253         # TODO - not used, do anything about it?
 254         # public ssh key
 255         self.pub_key = config.get("pub_key")
 256
 257         # TODO - check default user for azure
 258         # default admin user
 259         self._default_admin_user = "azureuser"
 260
 261         # flavor pattern regex
 262         if "flavors_pattern" in config:
 263             self._config["flavors_pattern"] = config["flavors_pattern"]
 264
 265     def _find_in_capabilities(self, capabilities, name):
 266         cap = find_in_list(capabilities, lambda c: c["name"] == name)
 267         if cap:
 268             return cap.get("value")
 269         else:
 270             return None
 271
 272     def _reload_connection(self):
 273         """
 274         Called before any operation, checks python azure clients
 275         """
 276         if self.reload_client:
 277             self.logger.debug("reloading azure client")
 278
 279             try:
 280                 self.credentials = ClientSecretCredential(
 281                     client_id=self._config["user"],
 282                     client_secret=self._config["passwd"],
 283                     tenant_id=self._config["tenant"],
 284                 )
 285                 self.conn = ResourceManagementClient(
 286                     self.credentials,
 287                     self._config["subscription_id"],
 288                     profile=self.AZURE_RESOURCE_MGMT_PROFILE,
 289                 )
 290                 self.conn_compute = ComputeManagementClient(
 291                     self.credentials,
 292                     self._config["subscription_id"],
 293                     profile=self.AZURE_COMPUTE_MGMT_PROFILE,
 294                 )
 295                 self.conn_vnet = NetworkManagementClient(
 296                     self.credentials,
 297                     self._config["subscription_id"],
 298                     profile=self.AZURE_NETWORK_MGMT_PROFILE,
 299                 )
 300                 self._check_or_create_resource_group()
 301                 self._check_or_create_vnet()
 302
 303                 # Set to client created
 304                 self.reload_client = False
 305             except Exception as e:
 306                 self._format_vimconn_exception(e)
 307
 308     def _get_resource_name_from_resource_id(self, resource_id):
 309         """
 310         Obtains resource_name from the azure complete identifier: resource_name will always be last item
 311         """
 312         try:
 313             resource = str(resource_id.split("/")[-1])
 314
 315             return resource
 316         except Exception as e:
 317             raise vimconn.VimConnException(
 318                 "Unable to get resource name from resource_id '{}' Error: '{}'".format(
 319                     resource_id, e
 320                 )
 321             )
 322
 323     def _get_location_from_resource_group(self, resource_group_name):
 324         try:
 325             location = self.conn.resource_groups.get(resource_group_name).location
 326
 327             return location
 328         except Exception:
 329             raise vimconn.VimConnNotFoundException(
 330                 "Location '{}' not found".format(resource_group_name)
 331             )
 332
 333     def _get_resource_group_name_from_resource_id(self, resource_id):
 334         try:
 335             rg = str(resource_id.split("/")[4])
 336
 337             return rg
 338         except Exception:
 339             raise vimconn.VimConnException(
 340                 "Unable to get resource group from invalid resource_id format '{}'".format(
 341                     resource_id
 342                 )
 343             )
 344
 345     def _get_net_name_from_resource_id(self, resource_id):
 346         try:
 347             net_name = str(resource_id.split("/")[8])
 348
 349             return net_name
 350         except Exception:
 351             raise vimconn.VimConnException(
 352                 "Unable to get azure net_name from invalid resource_id format '{}'".format(
 353                     resource_id
 354                 )
 355             )
 356
 357     def _check_subnets_for_vm(self, net_list):
 358         # All subnets must belong to the same resource group and vnet
 359         # All subnets must belong to the same resource group anded vnet
 360         rg_vnet = set(
 361             self._get_resource_group_name_from_resource_id(net["net_id"])
 362             + self._get_net_name_from_resource_id(net["net_id"])
 363             for net in net_list
 364         )
 365
 366         if len(rg_vnet) != 1:
 367             raise self._format_vimconn_exception(
 368                 "Azure VMs can only attach to subnets in same VNET"
 369             )
 370
 371     def _format_vimconn_exception(self, e):
 372         """
 373         Transforms a generic or azure exception to a vimcommException
 374         """
 375         self.logger.error("Azure plugin error: {}".format(e))
 376         if isinstance(e, vimconn.VimConnException):
 377             raise e
 378         elif isinstance(e, AuthenticationError):
 379             raise vimconn.VimConnAuthException(type(e).__name__ + ": " + str(e))
 380         elif isinstance(e, ConnectionError):
 381             raise vimconn.VimConnConnectionException(type(e).__name__ + ": " + str(e))
 382         else:
 383             # In case of generic error recreate client
 384             self.reload_client = True
 385
 386             raise vimconn.VimConnException(type(e).__name__ + ": " + str(e))
 387
 388     def _check_or_create_resource_group(self):
 389         """
 390         Creates the base resource group if it does not exist
 391         """
 392         try:
 393             rg_exists = self.conn.resource_groups.check_existence(self.resource_group)
 394
 395             if not rg_exists:
 396                 self.logger.debug("create base rgroup: %s", self.resource_group)
 397                 self.conn.resource_groups.create_or_update(
 398                     self.resource_group, {"location": self.region}
 399                 )
 400         except Exception as e:
 401             self._format_vimconn_exception(e)
 402
 403     def _check_or_create_vnet(self):
 404         """
 405         Try to get existent base vnet, in case it does not exist it creates it
 406         """
 407         try:
 408             vnet = self.conn_vnet.virtual_networks.get(
 409                 self.vnet_resource_group or self.resource_group, self.vnet_name
 410             )
 411             self.vnet_address_space = vnet.address_space.address_prefixes[0]
 412             self.vnet_id = vnet.id
 413
 414             return
 415         except CloudError as e:
 416             if e.error.error and "notfound" in e.error.error.lower():
 417                 self.logger.exception("CloudError Exception occured.")
 418                 # continue and create it
 419             else:
 420                 self._format_vimconn_exception(e)
 421
 422         # if it does not exist, create it
 423         try:
 424             vnet_params = {
 425                 "location": self.region,
 426                 "address_space": {"address_prefixes": ["10.0.0.0/8"]},
 427             }
 428             self.vnet_address_space = "10.0.0.0/8"
 429
 430             self.logger.debug("create base vnet: %s", self.vnet_name)
 431             self.conn_vnet.virtual_networks.begin_create_or_update(
 432                 self.vnet_resource_group or self.resource_group,
 433                 self.vnet_name,
 434                 vnet_params,
 435             )
 436             vnet = self.conn_vnet.virtual_networks.get(
 437                 self.vnet_resource_group or self.resource_group, self.vnet_name
 438             )
 439             self.vnet_id = vnet.id
 440         except Exception as e:
 441             self._format_vimconn_exception(e)
 442
 443     def new_network(
 444         self,
 445         net_name,
 446         net_type,
 447         ip_profile=None,
 448         shared=False,
 449         provider_network_profile=None,
 450     ):
 451         """
 452         Adds a tenant network to VIM
 453         :param net_name: name of the network
 454         :param net_type: not used for azure networks
 455         :param ip_profile: is a dict containing the IP parameters of the network (Currently only IPv4 is implemented)
 456                 'ip-version': can be one of ['IPv4','IPv6']
 457                 'subnet-address': ip_prefix_schema, that is X.X.X.X/Y
 458                 'gateway-address': (Optional) ip_schema, that is X.X.X.X, not implemented for azure connector
 459                 'dns-address': (Optional) ip_schema, not implemented for azure connector
 460                 'dhcp': (Optional) dict containing, not implemented for azure connector
 461                     'enabled': {'type': 'boolean'},
 462                     'start-address': ip_schema, first IP to grant
 463                     'count': number of IPs to grant.
 464         :param shared: Not allowed for Azure Connector
 465         :param provider_network_profile: (optional) contains {segmentation-id: vlan, provider-network: vim_netowrk}
 466         :return: a tuple with the network identifier and created_items, or raises an exception on error
 467             created_items can be None or a dictionary where this method can include key-values that will be passed to
 468             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 469             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 470             as not present.
 471         """
 472         return self._new_subnet(net_name, ip_profile)
 473
 474     def _new_subnet(self, net_name, ip_profile):
 475         """
 476         Adds a tenant network to VIM. It creates a new subnet at existing base vnet
 477         :param net_name: subnet name
 478         :param ip_profile:
 479                 subnet-address: if it is not provided a subnet/24 in the default vnet is created,
 480                 otherwise it creates a subnet in the indicated address
 481         :return: a tuple with the network identifier and created_items, or raises an exception on error
 482         """
 483         self.logger.debug("create subnet name %s, ip_profile %s", net_name, ip_profile)
 484         self._reload_connection()
 485
 486         if ip_profile is None:
 487             # get a non used vnet ip range /24 and allocate automatically inside the range self.vnet_address_space
 488             used_subnets = self.get_network_list()
 489             for ip_range in netaddr.IPNetwork(self.vnet_address_space).subnet(24):
 490                 for used_subnet in used_subnets:
 491                     subnet_range = netaddr.IPNetwork(used_subnet["cidr_block"])
 492
 493                     if subnet_range in ip_range or ip_range in subnet_range:
 494                         # this range overlaps with an existing subnet ip range. Breaks and look for another
 495                         break
 496                 else:
 497                     ip_profile = {"subnet_address": str(ip_range)}
 498                     self.logger.debug("dinamically obtained ip_profile: %s", ip_range)
 499                     break
 500             else:
 501                 raise vimconn.VimConnException(
 502                     "Cannot find a non-used subnet range in {}".format(
 503                         self.vnet_address_space
 504                     )
 505                 )
 506         else:
 507             ip_profile = {"subnet_address": ip_profile["subnet_address"]}
 508
 509         try:
 510             # subnet_name = "{}-{}".format(net_name[:24], uuid4())
 511             subnet_params = {"address_prefix": ip_profile["subnet_address"]}
 512             # Assign a not duplicated net name
 513             subnet_name = self._get_unused_subnet_name(net_name)
 514
 515             self.logger.debug("creating subnet_name: {}".format(subnet_name))
 516             async_creation = self.conn_vnet.subnets.begin_create_or_update(
 517                 self.vnet_resource_group or self.resource_group,
 518                 self.vnet_name,
 519                 subnet_name,
 520                 subnet_params,
 521             )
 522             async_creation.wait()
 523             # TODO - do not wait here, check where it is used
 524             self.logger.debug("created subnet_name: {}".format(subnet_name))
 525
 526             return "{}/subnets/{}".format(self.vnet_id, subnet_name), None
 527         except Exception as e:
 528             self._format_vimconn_exception(e)
 529
 530     def _get_unused_subnet_name(self, subnet_name):
 531         """
 532         Adds a prefix to the subnet_name with a number in case the indicated name is repeated
 533         Checks subnets with the indicated name (without suffix) and adds a suffix with a number
 534         """
 535         all_subnets = self.conn_vnet.subnets.list(
 536             self.vnet_resource_group or self.resource_group, self.vnet_name
 537         )
 538         # Filter to subnets starting with the indicated name
 539         subnets = list(
 540             filter(lambda subnet: (subnet.name.startswith(subnet_name)), all_subnets)
 541         )
 542         net_names = [str(subnet.name) for subnet in subnets]
 543
 544         # get the name with the first not used suffix
 545         name_suffix = 0
 546         # name = subnet_name + "-" + str(name_suffix)
 547         name = subnet_name  # first subnet created will have no prefix
 548         while name in net_names:
 549             name_suffix += 1
 550             name = subnet_name + "-" + str(name_suffix)
 551
 552         return name
 553
 554     def _create_nic(self, net, nic_name, region=None, static_ip=None, created_items={}):
 555         self.logger.debug("create nic name %s, net_name %s", nic_name, net)
 556         self._reload_connection()
 557
 558         subnet_id = net["net_id"]
 559         location = self.region or self._get_location_from_resource_group(
 560             self.resource_group
 561         )
 562
 563         try:
 564             net_ifz = {"location": location}
 565             net_ip_config = {
 566                 "name": nic_name + "-ipconfiguration",
 567                 "subnet": {"id": subnet_id},
 568             }
 569
 570             if static_ip:
 571                 net_ip_config["privateIPAddress"] = static_ip
 572                 net_ip_config["privateIPAllocationMethod"] = "Static"
 573
 574             net_ifz["ip_configurations"] = [net_ip_config]
 575             mac_address = net.get("mac_address")
 576
 577             if mac_address:
 578                 net_ifz["mac_address"] = mac_address
 579
 580             async_nic_creation = (
 581                 self.conn_vnet.network_interfaces.begin_create_or_update(
 582                     self.resource_group, nic_name, net_ifz
 583                 )
 584             )
 585             nic_data = async_nic_creation.result()
 586             created_items[nic_data.id] = True
 587             self.logger.debug("created nic name %s", nic_name)
 588
 589             public_ip = net.get("floating_ip")
 590             if public_ip:
 591                 public_ip_address_params = {
 592                     "location": location,
 593                     "public_ip_allocation_method": "Dynamic",
 594                 }
 595                 public_ip_name = nic_name + "-public-ip"
 596                 async_public_ip = (
 597                     self.conn_vnet.public_ip_addresses.begin_create_or_update(
 598                         self.resource_group, public_ip_name, public_ip_address_params
 599                     )
 600                 )
 601                 public_ip = async_public_ip.result()
 602                 self.logger.debug("created public IP: {}".format(public_ip))
 603
 604                 # Associate NIC to Public IP
 605                 nic_data = self.conn_vnet.network_interfaces.get(
 606                     self.resource_group, nic_name
 607                 )
 608
 609                 nic_data.ip_configurations[0].public_ip_address = public_ip
 610                 created_items[public_ip.id] = True
 611
 612                 self.conn_vnet.network_interfaces.begin_create_or_update(
 613                     self.resource_group, nic_name, nic_data
 614                 )
 615
 616         except Exception as e:
 617             self._format_vimconn_exception(e)
 618
 619         return nic_data, created_items
 620
 621     def new_flavor(self, flavor_data):
 622         """
 623         It is not allowed to create new flavors in Azure, must always use an existing one
 624         """
 625         raise vimconn.VimConnAuthException(
 626             "It is not possible to create new flavors in AZURE"
 627         )
 628
 629     def new_tenant(self, tenant_name, tenant_description):
 630         """
 631         It is not allowed to create new tenants in azure
 632         """
 633         raise vimconn.VimConnAuthException(
 634             "It is not possible to create a TENANT in AZURE"
 635         )
 636
 637     def new_image(self, image_dict):
 638         """
 639         It is not allowed to create new images in Azure, must always use an existing one
 640         """
 641         raise vimconn.VimConnAuthException(
 642             "It is not possible to create new images in AZURE"
 643         )
 644
 645     def get_image_id_from_path(self, path):
 646         """Get the image id from image path in the VIM database.
 647         Returns the image_id or raises a vimconnNotFoundException
 648         """
 649         raise vimconn.VimConnAuthException(
 650             "It is not possible to obtain image from path in AZURE"
 651         )
 652
 653     def get_image_list(self, filter_dict={}):
 654         """Obtain tenant images from VIM
 655         Filter_dict can be:
 656             name: image name with the format: publisher:offer:sku:version
 657             If some part of the name is provide ex: publisher:offer it will search all availables skus and version
 658             for the provided publisher and offer
 659             id: image uuid, currently not supported for azure
 660         Returns the image list of dictionaries:
 661             [{<the fields at Filter_dict plus some VIM specific>}, ...]
 662             List can be empty
 663         """
 664         self.logger.debug("get_image_list filter {}".format(filter_dict))
 665
 666         self._reload_connection()
 667         try:
 668             image_list = []
 669             if filter_dict.get("name"):
 670                 # name will have the format "publisher:offer:sku:version"
 671                 # publisher is required, offer sku and version will be searched if not provided
 672                 params = filter_dict["name"].split(":")
 673                 publisher = params[0]
 674                 if publisher:
 675                     # obtain offer list
 676                     offer_list = self._get_offer_list(params, publisher)
 677
 678                     for offer in offer_list:
 679                         # obtain skus
 680                         sku_list = self._get_sku_list(params, publisher, offer)
 681
 682                         for sku in sku_list:
 683                             # if version is defined get directly version, else list images
 684                             if len(params) == 4 and params[3]:
 685                                 version = params[3]
 686                                 if version == "latest":
 687                                     image_list = self._get_sku_image_list(
 688                                         publisher, offer, sku
 689                                     )
 690                                     image_list = [image_list[-1]]
 691                                 else:
 692                                     image_list = self._get_version_image_list(
 693                                         publisher, offer, sku, version
 694                                     )
 695                             else:
 696                                 image_list = self._get_sku_image_list(
 697                                     publisher, offer, sku
 698                                 )
 699                 else:
 700                     raise vimconn.VimConnAuthException(
 701                         "List images in Azure must include name param with at least publisher"
 702                     )
 703             else:
 704                 raise vimconn.VimConnAuthException(
 705                     "List images in Azure must include name param with at"
 706                     " least publisher"
 707                 )
 708
 709             return image_list
 710         except Exception as e:
 711             self._format_vimconn_exception(e)
 712
 713     def _get_offer_list(self, params, publisher):
 714         """
 715         Helper method to obtain offer list for defined publisher
 716         """
 717         if len(params) >= 2 and params[1]:
 718             return [params[1]]
 719         else:
 720             try:
 721                 # get list of offers from azure
 722                 result_offers = self.conn_compute.virtual_machine_images.list_offers(
 723                     self.region, publisher
 724                 )
 725
 726                 return [offer.name for offer in result_offers]
 727             except CloudError as e:
 728                 # azure raises CloudError when not found
 729                 self.logger.info(
 730                     "error listing offers for publisher {}, Error: {}".format(
 731                         publisher, e
 732                     )
 733                 )
 734
 735                 return []
 736
 737     def _get_sku_list(self, params, publisher, offer):
 738         """
 739         Helper method to obtain sku list for defined publisher and offer
 740         """
 741         if len(params) >= 3 and params[2]:
 742             return [params[2]]
 743         else:
 744             try:
 745                 # get list of skus from azure
 746                 result_skus = self.conn_compute.virtual_machine_images.list_skus(
 747                     self.region, publisher, offer
 748                 )
 749
 750                 return [sku.name for sku in result_skus]
 751             except CloudError as e:
 752                 # azure raises CloudError when not found
 753                 self.logger.info(
 754                     "error listing skus for publisher {}, offer {}, Error: {}".format(
 755                         publisher, offer, e
 756                     )
 757                 )
 758
 759                 return []
 760
 761     def _get_sku_image_list(self, publisher, offer, sku):
 762         """
 763         Helper method to obtain image list for publisher, offer and sku
 764         """
 765         image_list = []
 766         try:
 767             result_images = self.conn_compute.virtual_machine_images.list(
 768                 self.region, publisher, offer, sku
 769             )
 770             for result_image in result_images:
 771                 image_list.append(
 772                     {
 773                         "id": str(result_image.id),
 774                         "name": ":".join([publisher, offer, sku, result_image.name]),
 775                     }
 776                 )
 777         except CloudError as e:
 778             self.logger.info(
 779                 "error listing skus for publisher {}, offer {}, Error: {}".format(
 780                     publisher, offer, e
 781                 )
 782             )
 783             image_list = []
 784
 785         return image_list
 786
 787     def _get_version_image_list(self, publisher, offer, sku, version):
 788         image_list = []
 789         try:
 790             result_image = self.conn_compute.virtual_machine_images.get(
 791                 self.region, publisher, offer, sku, version
 792             )
 793
 794             if result_image:
 795                 image_list.append(
 796                     {
 797                         "id": str(result_image.id),
 798                         "name": ":".join([publisher, offer, sku, version]),
 799                     }
 800                 )
 801         except CloudError as e:
 802             # azure gives CloudError when not found
 803             self.logger.info(
 804                 "error listing images for publisher {}, offer {}, sku {}, version {} Error: {}".format(
 805                     publisher, offer, sku, version, e
 806                 )
 807             )
 808             image_list = []
 809
 810         return image_list
 811
 812     def get_network_list(self, filter_dict={}):
 813         """Obtain tenant networks of VIM
 814         Filter_dict can be:
 815             name: network name
 816             id: network id
 817             shared: boolean, not implemented in Azure
 818             tenant_id: tenant, not used in Azure, all networks same tenants
 819             admin_state_up: boolean, not implemented in Azure
 820             status: 'ACTIVE', not implemented in Azure #
 821         Returns the network list of dictionaries
 822         """
 823         # self.logger.debug("getting network list for vim, filter %s", filter_dict)
 824         try:
 825             self._reload_connection()
 826
 827             vnet = self.conn_vnet.virtual_networks.get(
 828                 self.vnet_resource_group or self.resource_group, self.vnet_name
 829             )
 830             subnet_list = []
 831
 832             for subnet in vnet.subnets:
 833                 if filter_dict:
 834                     if filter_dict.get("id") and str(subnet.id) != filter_dict["id"]:
 835                         continue
 836
 837                     if (
 838                         filter_dict.get("name")
 839                         and str(subnet.name) != filter_dict["name"]
 840                     ):
 841                         continue
 842
 843                 name = self._get_resource_name_from_resource_id(subnet.id)
 844
 845                 subnet_list.append(
 846                     {
 847                         "id": str(subnet.id),
 848                         "name": name,
 849                         "status": self.provision_state2osm[subnet.provisioning_state],
 850                         "cidr_block": str(subnet.address_prefix),
 851                         "type": "bridge",
 852                         "shared": False,
 853                     }
 854                 )
 855
 856             return subnet_list
 857         except Exception as e:
 858             self._format_vimconn_exception(e)
 859
 860     def new_vminstance(
 861         self,
 862         name,
 863         description,
 864         start,
 865         image_id,
 866         flavor_id,
 867         affinity_group_list,
 868         net_list,
 869         cloud_config=None,
 870         disk_list=None,
 871         availability_zone_index=None,
 872         availability_zone_list=None,
 873     ):
 874         self.logger.debug(
 875             "new vm instance name: %s, image_id: %s, flavor_id: %s, net_list: %s, cloud_config: %s, "
 876             "disk_list: %s, availability_zone_index: %s, availability_zone_list: %s",
 877             name,
 878             image_id,
 879             flavor_id,
 880             net_list,
 881             cloud_config,
 882             disk_list,
 883             availability_zone_index,
 884             availability_zone_list,
 885         )
 886         self._reload_connection()
 887
 888         # Validate input data is valid
 889         # The virtual machine name must have less or 64 characters and it can not have the following
 890         # characters: (~ ! @ # $ % ^ & * ( ) = + _ [ ] { } \ | ; : ' " , < > / ?.)
 891         vm_name = self._check_vm_name(name)
 892         # Obtain vm unused name
 893         vm_name = self._get_unused_vm_name(vm_name)
 894
 895         # At least one network must be provided
 896         if not net_list:
 897             raise vimconn.VimConnException(
 898                 "At least one net must be provided to create a new VM"
 899             )
 900
 901         # image_id are several fields of the image_id
 902         image_reference = self._get_image_reference(image_id)
 903
 904         try:
 905             virtual_machine = None
 906             created_items = {}
 907
 908             # Create nics for each subnet
 909             self._check_subnets_for_vm(net_list)
 910             vm_nics = []
 911
 912             for idx, net in enumerate(net_list):
 913                 # Fault with subnet_id
 914                 # subnet_id=net["subnet_id"]
 915                 # subnet_id=net["net_id"]
 916                 nic_name = vm_name + "-nic-" + str(idx)
 917                 vm_nic, nic_items = self._create_nic(
 918                     net, nic_name, self.region, net.get("ip_address"), created_items
 919                 )
 920                 vm_nics.append({"id": str(vm_nic.id)})
 921                 net["vim_id"] = vm_nic.id
 922
 923             vm_parameters = {
 924                 "location": self.region,
 925                 "os_profile": self._build_os_profile(vm_name, cloud_config, image_id),
 926                 "hardware_profile": {"vm_size": flavor_id},
 927                 "storage_profile": {"image_reference": image_reference},
 928             }
 929
 930             # If the machine has several networks one must be marked as primary
 931             # As it is not indicated in the interface the first interface will be marked as primary
 932             if len(vm_nics) > 1:
 933                 for idx, vm_nic in enumerate(vm_nics):
 934                     if idx == 0:
 935                         vm_nics[0]["Primary"] = True
 936                     else:
 937                         vm_nics[idx]["Primary"] = False
 938
 939             vm_parameters["network_profile"] = {"network_interfaces": vm_nics}
 940
 941             # Obtain zone information
 942             vm_zone = self._get_vm_zone(availability_zone_index, availability_zone_list)
 943             if vm_zone:
 944                 vm_parameters["zones"] = [vm_zone]
 945
 946             self.logger.debug("create vm name: %s", vm_name)
 947             creation_result = self.conn_compute.virtual_machines.begin_create_or_update(
 948                 self.resource_group, vm_name, vm_parameters, polling=False
 949             )
 950             self.logger.debug("obtained creation result: %s", creation_result)
 951             virtual_machine = creation_result.result()
 952             self.logger.debug("created vm name: %s", vm_name)
 953
 954             return virtual_machine.id, created_items
 955
 956         except Exception as e:
 957             # Rollback vm creacion
 958             vm_id = None
 959
 960             if virtual_machine:
 961                 vm_id = virtual_machine.id
 962
 963             try:
 964                 self.logger.debug("exception creating vm try to rollback")
 965                 self.delete_vminstance(vm_id, created_items)
 966             except Exception as e2:
 967                 self.logger.error("new_vminstance rollback fail {}".format(e2))
 968
 969             self.logger.debug("Exception creating new vminstance: %s", e, exc_info=True)
 970             self._format_vimconn_exception(e)
 971
 972     def _build_os_profile(self, vm_name, cloud_config, image_id):
 973
 974         # initial os_profile
 975         os_profile = {"computer_name": vm_name}
 976
 977         # for azure os_profile admin_username is required
 978         if cloud_config and cloud_config.get("users"):
 979             admin_username = cloud_config.get("users")[0].get(
 980                 "name", self._get_default_admin_user(image_id)
 981             )
 982         else:
 983             admin_username = self._get_default_admin_user(image_id)
 984         os_profile["admin_username"] = admin_username
 985
 986         # if there is a cloud-init load it
 987         if cloud_config:
 988             _, userdata = self._create_user_data(cloud_config)
 989             custom_data = base64.b64encode(userdata.encode("utf-8")).decode("latin-1")
 990             os_profile["custom_data"] = custom_data
 991
 992         # either password of ssh-keys are required
 993         # we will always use ssh-keys, in case it is not available we will generate it
 994         if cloud_config and cloud_config.get("key-pairs"):
 995             key_data = cloud_config.get("key-pairs")[0]
 996         else:
 997             _, key_data = self._generate_keys()
 998
 999         os_profile["linux_configuration"] = {
1000             "ssh": {
1001                 "public_keys": [
1002                     {
1003                         "path": "/home/{}/.ssh/authorized_keys".format(admin_username),
1004                         "key_data": key_data,
1005                     }
1006                 ]
1007             },
1008         }
1009
1010         return os_profile
1011
1012     def _generate_keys(self):
1013         """Method used to generate a pair of private/public keys.
1014         This method is used because to create a vm in Azure we always need a key or a password
1015         In some cases we may have a password in a cloud-init file but it may not be available
1016         """
1017         key = rsa.generate_private_key(
1018             backend=crypto_default_backend(), public_exponent=65537, key_size=2048
1019         )
1020         private_key = key.private_bytes(
1021             crypto_serialization.Encoding.PEM,
1022             crypto_serialization.PrivateFormat.PKCS8,
1023             crypto_serialization.NoEncryption(),
1024         )
1025         public_key = key.public_key().public_bytes(
1026             crypto_serialization.Encoding.OpenSSH,
1027             crypto_serialization.PublicFormat.OpenSSH,
1028         )
1029         private_key = private_key.decode("utf8")
1030         # Change first line because Paramiko needs a explicit start with 'BEGIN RSA PRIVATE KEY'
1031         i = private_key.find("\n")
1032         private_key = "-----BEGIN RSA PRIVATE KEY-----" + private_key[i:]
1033         public_key = public_key.decode("utf8")
1034
1035         return private_key, public_key
1036
1037     def _get_unused_vm_name(self, vm_name):
1038         """
1039         Checks the vm name and in case it is used adds a suffix to the name to allow creation
1040         :return:
1041         """
1042         all_vms = self.conn_compute.virtual_machines.list(self.resource_group)
1043         # Filter to vms starting with the indicated name
1044         vms = list(filter(lambda vm: (vm.name.startswith(vm_name)), all_vms))
1045         vm_names = [str(vm.name) for vm in vms]
1046
1047         # get the name with the first not used suffix
1048         name_suffix = 0
1049         # name = subnet_name + "-" + str(name_suffix)
1050         name = vm_name  # first subnet created will have no prefix
1051
1052         while name in vm_names:
1053             name_suffix += 1
1054             name = vm_name + "-" + str(name_suffix)
1055
1056         return name
1057
1058     def _get_vm_zone(self, availability_zone_index, availability_zone_list):
1059         if availability_zone_index is None:
1060             return None
1061
1062         vim_availability_zones = self._get_azure_availability_zones()
1063         # check if VIM offer enough availability zones describe in the VNFD
1064         if vim_availability_zones and len(availability_zone_list) <= len(
1065             vim_availability_zones
1066         ):
1067             # check if all the names of NFV AV match VIM AV names
1068             match_by_index = False
1069
1070             if not availability_zone_list:
1071                 match_by_index = True
1072             else:
1073                 for av in availability_zone_list:
1074                     if av not in vim_availability_zones:
1075                         match_by_index = True
1076                         break
1077
1078             if match_by_index:
1079                 return vim_availability_zones[availability_zone_index]
1080             else:
1081                 return availability_zone_list[availability_zone_index]
1082         else:
1083             raise vimconn.VimConnConflictException(
1084                 "No enough availability zones at VIM for this deployment"
1085             )
1086
1087     def _get_azure_availability_zones(self):
1088         return self.AZURE_ZONES
1089
1090     def _get_image_reference(self, image_id):
1091         try:
1092             # The data input format example:
1093             # /Subscriptions/ca3d18ab-d373-4afb-a5d6-7c44f098d16a/Providers/Microsoft.Compute/Locations/westeurope/
1094             # Publishers/Canonical/ArtifactTypes/VMImage/
1095             # Offers/UbuntuServer/
1096             # Skus/18.04-LTS/
1097             # Versions/18.04.201809110
1098             publisher = str(image_id.split("/")[8])
1099             offer = str(image_id.split("/")[12])
1100             sku = str(image_id.split("/")[14])
1101             version = str(image_id.split("/")[16])
1102
1103             return {
1104                 "publisher": publisher,
1105                 "offer": offer,
1106                 "sku": sku,
1107                 "version": version,
1108             }
1109         except Exception:
1110             raise vimconn.VimConnException(
1111                 "Unable to get image_reference from invalid image_id format: '{}'".format(
1112                     image_id
1113                 )
1114             )
1115
1116     # Azure VM names can not have some special characters
1117     def _check_vm_name(self, vm_name):
1118         """
1119         Checks vm name, in case the vm has not allowed characters they are removed, not error raised
1120         """
1121         chars_not_allowed_list = "~!@#$%^&*()=+_[]{}|;:<>/?."
1122
1123         # First: the VM name max length is 64 characters
1124         vm_name_aux = vm_name[:64]
1125
1126         # Second: replace not allowed characters
1127         for elem in chars_not_allowed_list:
1128             # Check if string is in the main string
1129             if elem in vm_name_aux:
1130                 # self.logger.debug("Dentro del IF")
1131                 # Replace the string
1132                 vm_name_aux = vm_name_aux.replace(elem, "-")
1133
1134         return vm_name_aux
1135
1136     def get_flavor_id_from_data(self, flavor_dict):
1137         self.logger.debug("getting flavor id from data, flavor_dict: %s", flavor_dict)
1138         filter_dict = flavor_dict or {}
1139
1140         try:
1141             self._reload_connection()
1142             vm_sizes_list = [
1143                 vm_size.as_dict()
1144                 for vm_size in self.conn_compute.resource_skus.list(
1145                     "location eq '{}'".format(self.region)
1146                 )
1147             ]
1148
1149             cpus = filter_dict.get("vcpus") or 0
1150             memMB = filter_dict.get("ram") or 0
1151             numberInterfaces = len(filter_dict.get("interfaces", [])) or 0
1152
1153             # Filter
1154             filtered_sizes = []
1155             for size in vm_sizes_list:
1156                 if size["resource_type"] == "virtualMachines":
1157                     size_cpus = int(
1158                         self._find_in_capabilities(size["capabilities"], "vCPUs")
1159                     )
1160                     size_memory = float(
1161                         self._find_in_capabilities(size["capabilities"], "MemoryGB")
1162                     )
1163                     size_interfaces = self._find_in_capabilities(
1164                         size["capabilities"], "MaxNetworkInterfaces"
1165                     )
1166                     if size_interfaces:
1167                         size_interfaces = int(size_interfaces)
1168                     else:
1169                         self.logger.debug(
1170                             "Flavor with no defined MaxNetworkInterfaces: {}".format(
1171                                 size["name"]
1172                             )
1173                         )
1174                         continue
1175                     if (
1176                         size_cpus >= cpus
1177                         and size_memory >= memMB / 1024
1178                         and size_interfaces >= numberInterfaces
1179                     ):
1180                         if self._config.get("flavors_pattern"):
1181                             if re.search(
1182                                 self._config.get("flavors_pattern"), size["name"]
1183                             ):
1184                                 new_size = {
1185                                     e["name"]: e["value"] for e in size["capabilities"]
1186                                 }
1187                                 new_size["name"] = size["name"]
1188                                 filtered_sizes.append(new_size)
1189                         else:
1190                             new_size = {
1191                                 e["name"]: e["value"] for e in size["capabilities"]
1192                             }
1193                             new_size["name"] = size["name"]
1194                             filtered_sizes.append(new_size)
1195
1196             # Sort
1197             listedFilteredSizes = sorted(
1198                 filtered_sizes,
1199                 key=lambda k: (
1200                     int(k["vCPUs"]),
1201                     float(k["MemoryGB"]),
1202                     int(k["MaxNetworkInterfaces"]),
1203                     int(k["MaxResourceVolumeMB"]),
1204                 ),
1205             )
1206
1207             if listedFilteredSizes:
1208                 return listedFilteredSizes[0]["name"]
1209
1210             raise vimconn.VimConnNotFoundException(
1211                 "Cannot find any flavor matching '{}'".format(str(flavor_dict))
1212             )
1213         except Exception as e:
1214             self._format_vimconn_exception(e)
1215
1216     def _get_flavor_id_from_flavor_name(self, flavor_name):
1217         # self.logger.debug("getting flavor id from flavor name {}".format(flavor_name))
1218         try:
1219             self._reload_connection()
1220             vm_sizes_list = [
1221                 vm_size.as_dict()
1222                 for vm_size in self.conn_compute.resource_skus.list(
1223                     "location eq '{}'".format(self.region)
1224                 )
1225             ]
1226
1227             output_flavor = None
1228             for size in vm_sizes_list:
1229                 if size["name"] == flavor_name:
1230                     output_flavor = size
1231
1232             # None is returned if not found anything
1233             return output_flavor
1234         except Exception as e:
1235             self._format_vimconn_exception(e)
1236
1237     def check_vim_connectivity(self):
1238         try:
1239             self._reload_connection()
1240             return True
1241         except Exception as e:
1242             raise vimconn.VimConnException(
1243                 "Connectivity issue with Azure API: {}".format(e)
1244             )
1245
1246     def get_network(self, net_id):
1247         # self.logger.debug("get network id: {}".format(net_id))
1248         # res_name = self._get_resource_name_from_resource_id(net_id)
1249         self._reload_connection()
1250
1251         filter_dict = {"name": net_id}
1252         network_list = self.get_network_list(filter_dict)
1253
1254         if not network_list:
1255             raise vimconn.VimConnNotFoundException(
1256                 "network '{}' not found".format(net_id)
1257             )
1258         else:
1259             return network_list[0]
1260
1261     def delete_network(self, net_id, created_items=None):
1262         self.logger.debug(
1263             "deleting network {} - {}".format(
1264                 self.vnet_resource_group or self.resource_group, net_id
1265             )
1266         )
1267
1268         self._reload_connection()
1269         res_name = self._get_resource_name_from_resource_id(net_id)
1270
1271         try:
1272             # Obtain subnets ant try to delete nic first
1273             subnet = self.conn_vnet.subnets.get(
1274                 self.vnet_resource_group or self.resource_group,
1275                 self.vnet_name,
1276                 res_name,
1277             )
1278             if not subnet:
1279                 raise vimconn.VimConnNotFoundException(
1280                     "network '{}' not found".format(net_id)
1281                 )
1282
1283             # TODO - for a quick-fix delete nics sequentially but should not wait
1284             # for each in turn
1285             if subnet.ip_configurations:
1286                 for ip_configuration in subnet.ip_configurations:
1287                     # obtain nic_name from ip_configuration
1288                     parsed_id = azure_tools.parse_resource_id(ip_configuration.id)
1289                     nic_name = parsed_id["name"]
1290                     self.delete_inuse_nic(nic_name)
1291
1292             # Subnet API fails (CloudError: Azure Error: ResourceNotFound)
1293             # Put the initial virtual_network API
1294             async_delete = self.conn_vnet.subnets.begin_delete(
1295                 self.vnet_resource_group or self.resource_group,
1296                 self.vnet_name,
1297                 res_name,
1298             )
1299             async_delete.wait()
1300
1301             return net_id
1302
1303         except ResourceNotFoundError:
1304             raise vimconn.VimConnNotFoundException(
1305                 "network '{}' not found".format(net_id)
1306             )
1307         except CloudError as e:
1308             if e.error.error and "notfound" in e.error.error.lower():
1309                 raise vimconn.VimConnNotFoundException(
1310                     "network '{}' not found".format(net_id)
1311                 )
1312             else:
1313                 self._format_vimconn_exception(e)
1314         except Exception as e:
1315             self._format_vimconn_exception(e)
1316
1317     def delete_inuse_nic(self, nic_name):
1318
1319         # Obtain nic data
1320         nic_data = self.conn_vnet.network_interfaces.get(self.resource_group, nic_name)
1321
1322         # Obtain vm associated to nic in case it exists
1323         if nic_data.virtual_machine:
1324             vm_name = azure_tools.parse_resource_id(nic_data.virtual_machine.id)["name"]
1325             self.logger.debug("vm_name: {}".format(vm_name))
1326             virtual_machine = self.conn_compute.virtual_machines.get(
1327                 self.resource_group, vm_name
1328             )
1329             self.logger.debug("obtained vm")
1330
1331             # Deattach nic from vm if it has netwolk machines attached
1332             network_interfaces = virtual_machine.network_profile.network_interfaces
1333             network_interfaces[:] = [
1334                 interface
1335                 for interface in network_interfaces
1336                 if self._get_resource_name_from_resource_id(interface.id) != nic_name
1337             ]
1338
1339             # TODO - check if there is a public ip to delete and delete it
1340             if network_interfaces:
1341
1342                 # Deallocate the vm
1343                 async_vm_deallocate = (
1344                     self.conn_compute.virtual_machines.begin_deallocate(
1345                         self.resource_group, vm_name
1346                     )
1347                 )
1348                 self.logger.debug("deallocating vm")
1349                 async_vm_deallocate.wait()
1350                 self.logger.debug("vm deallocated")
1351
1352                 async_vm_update = (
1353                     self.conn_compute.virtual_machines.begin_create_or_update(
1354                         self.resource_group, vm_name, virtual_machine
1355                     )
1356                 )
1357                 virtual_machine = async_vm_update.result()
1358                 self.logger.debug("nic removed from interface")
1359
1360             else:
1361                 self.logger.debug("There are no interfaces left, delete vm")
1362                 self.delete_vminstance(virtual_machine.id)
1363                 self.logger.debug("Delete vm")
1364
1365         # Delete nic
1366         self.logger.debug("delete NIC name: %s", nic_name)
1367         nic_delete = self.conn_vnet.network_interfaces.begin_delete(
1368             self.resource_group, nic_name
1369         )
1370         nic_delete.wait()
1371         self.logger.debug("deleted NIC name: %s", nic_name)
1372
1373     def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None):
1374         """Deletes a vm instance from the vim."""
1375         self.logger.debug(
1376             "deleting VM instance {} - {}".format(self.resource_group, vm_id)
1377         )
1378         self._reload_connection()
1379
1380         created_items = created_items or {}
1381         try:
1382             # Check vm exists, we can call delete_vm to clean created_items
1383             if vm_id:
1384                 res_name = self._get_resource_name_from_resource_id(vm_id)
1385                 vm = self.conn_compute.virtual_machines.get(
1386                     self.resource_group, res_name
1387                 )
1388
1389                 # Shuts down the virtual machine and releases the compute resources
1390                 # vm_stop = self.conn_compute.virtual_machines.power_off(self.resource_group, resName)
1391                 # vm_stop.wait()
1392
1393                 vm_delete = self.conn_compute.virtual_machines.begin_delete(
1394                     self.resource_group, res_name
1395                 )
1396                 vm_delete.wait()
1397                 self.logger.debug("deleted VM name: %s", res_name)
1398
1399                 # Delete OS Disk, check if exists, in case of error creating
1400                 # it may not be fully created
1401                 if vm.storage_profile.os_disk:
1402                     os_disk_name = vm.storage_profile.os_disk.name
1403                     self.logger.debug("delete OS DISK: %s", os_disk_name)
1404                     async_disk_delete = self.conn_compute.disks.begin_delete(
1405                         self.resource_group, os_disk_name
1406                     )
1407                     async_disk_delete.wait()
1408                     # os disks are created always with the machine
1409                     self.logger.debug("deleted OS DISK name: %s", os_disk_name)
1410
1411                 for data_disk in vm.storage_profile.data_disks:
1412                     self.logger.debug("delete data_disk: %s", data_disk.name)
1413                     async_disk_delete = self.conn_compute.disks.begin_delete(
1414                         self.resource_group, data_disk.name
1415                     )
1416                     async_disk_delete.wait()
1417                     self._markdel_created_item(data_disk.managed_disk.id, created_items)
1418                     self.logger.debug("deleted OS DISK name: %s", data_disk.name)
1419
1420                 # After deleting VM, it is necessary to delete NIC, because if is not deleted delete_network
1421                 # does not work because Azure says that is in use the subnet
1422                 network_interfaces = vm.network_profile.network_interfaces
1423
1424                 for network_interface in network_interfaces:
1425                     nic_name = self._get_resource_name_from_resource_id(
1426                         network_interface.id
1427                     )
1428                     nic_data = self.conn_vnet.network_interfaces.get(
1429                         self.resource_group, nic_name
1430                     )
1431
1432                     public_ip_name = None
1433                     exist_public_ip = nic_data.ip_configurations[0].public_ip_address
1434                     if exist_public_ip:
1435                         public_ip_id = nic_data.ip_configurations[
1436                             0
1437                         ].public_ip_address.id
1438
1439                         # Delete public_ip
1440                         public_ip_name = self._get_resource_name_from_resource_id(
1441                             public_ip_id
1442                         )
1443
1444                         # Public ip must be deleted afterwards of nic that is attached
1445
1446                     self.logger.debug("delete NIC name: %s", nic_name)
1447                     nic_delete = self.conn_vnet.network_interfaces.begin_delete(
1448                         self.resource_group, nic_name
1449                     )
1450                     nic_delete.wait()
1451                     self._markdel_created_item(network_interface.id, created_items)
1452                     self.logger.debug("deleted NIC name: %s", nic_name)
1453
1454                     # Delete list of public ips
1455                     if public_ip_name:
1456                         self.logger.debug("delete PUBLIC IP - " + public_ip_name)
1457                         ip_delete = self.conn_vnet.public_ip_addresses.begin_delete(
1458                             self.resource_group, public_ip_name
1459                         )
1460                         ip_delete.wait()
1461                         self._markdel_created_item(public_ip_id, created_items)
1462
1463             # Delete created items
1464             self._delete_created_items(created_items)
1465
1466         except ResourceNotFoundError:
1467             raise vimconn.VimConnNotFoundException(
1468                 "No vm instance found '{}'".format(vm_id)
1469             )
1470         except CloudError as e:
1471             if e.error.error and "notfound" in e.error.error.lower():
1472                 raise vimconn.VimConnNotFoundException(
1473                     "No vm instance found '{}'".format(vm_id)
1474                 )
1475             else:
1476                 self._format_vimconn_exception(e)
1477         except Exception as e:
1478             self._format_vimconn_exception(e)
1479
1480     def _markdel_created_item(self, item_id, created_items):
1481         if item_id in created_items:
1482             created_items[item_id] = False
1483
1484     def _delete_created_items(self, created_items):
1485         """Delete created_items elements that have not been deleted with the virtual machine
1486         Created_items may not be deleted correctly with the created machine if the
1487         virtual machine fails creating or in other cases of error
1488         """
1489         self.logger.debug("Created items: %s", created_items)
1490         # TODO - optimize - should not wait until it is deleted
1491         # Must delete in order first nics, then public_ips
1492         # As dictionaries don't preserve order, first get items to be deleted then delete them
1493         nics_to_delete = []
1494         publics_ip_to_delete = []
1495         disks_to_delete = []
1496         for item_id, v in created_items.items():
1497             if not v:  # skip already deleted
1498                 continue
1499
1500             # self.logger.debug("Must delete item id: %s", item_id)
1501             # Obtain type, supported nic, disk or public ip
1502             parsed_id = azure_tools.parse_resource_id(item_id)
1503             resource_type = parsed_id.get("resource_type")
1504             name = parsed_id.get("name")
1505
1506             if resource_type == "networkInterfaces":
1507                 nics_to_delete.append(name)
1508             elif resource_type == "publicIPAddresses":
1509                 publics_ip_to_delete.append(name)
1510             elif resource_type == "disks":
1511                 disks_to_delete.append(name)
1512
1513         # Now delete
1514         for item_name in nics_to_delete:
1515             try:
1516                 self.logger.debug("deleting nic name %s:", item_name)
1517                 nic_delete = self.conn_vnet.network_interfaces.begin_delete(
1518                     self.resource_group, item_name
1519                 )
1520                 nic_delete.wait()
1521                 self.logger.debug("deleted nic name %s:", item_name)
1522             except Exception as e:
1523                 self.logger.error(
1524                     "Error deleting item: {}: {}".format(type(e).__name__, e)
1525                 )
1526
1527         for item_name in publics_ip_to_delete:
1528             try:
1529                 self.logger.debug("deleting public ip name %s:", item_name)
1530                 ip_delete = self.conn_vnet.public_ip_addresses.begin_delete(
1531                     self.resource_group, name
1532                 )
1533                 ip_delete.wait()
1534                 self.logger.debug("deleted public ip name %s:", item_name)
1535             except Exception as e:
1536                 self.logger.error(
1537                     "Error deleting item: {}: {}".format(type(e).__name__, e)
1538                 )
1539
1540         for item_name in disks_to_delete:
1541             try:
1542                 self.logger.debug("deleting data disk name %s:", name)
1543                 async_disk_delete = self.conn_compute.disks.begin_delete(
1544                     self.resource_group, item_name
1545                 )
1546                 async_disk_delete.wait()
1547                 self.logger.debug("deleted data disk name %s:", name)
1548             except Exception as e:
1549                 self.logger.error(
1550                     "Error deleting item: {}: {}".format(type(e).__name__, e)
1551                 )
1552
1553     def action_vminstance(self, vm_id, action_dict, created_items={}):
1554         """Send and action over a VM instance from VIM
1555         Returns the vm_id if the action was successfully sent to the VIM
1556         """
1557         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1558
1559         try:
1560             self._reload_connection()
1561             resName = self._get_resource_name_from_resource_id(vm_id)
1562
1563             if "start" in action_dict:
1564                 self.conn_compute.virtual_machines.begin_start(
1565                     self.resource_group, resName
1566                 )
1567             elif (
1568                 "stop" in action_dict
1569                 or "shutdown" in action_dict
1570                 or "shutoff" in action_dict
1571             ):
1572                 self.conn_compute.virtual_machines.begin_power_off(
1573                     self.resource_group, resName
1574                 )
1575             elif "terminate" in action_dict:
1576                 self.conn_compute.virtual_machines.begin_delete(
1577                     self.resource_group, resName
1578                 )
1579             elif "reboot" in action_dict:
1580                 self.conn_compute.virtual_machines.begin_restart(
1581                     self.resource_group, resName
1582                 )
1583
1584             return None
1585         except ResourceNotFoundError:
1586             raise vimconn.VimConnNotFoundException("No vm found '{}'".format(vm_id))
1587         except CloudError as e:
1588             if e.error.error and "notfound" in e.error.error.lower():
1589                 raise vimconn.VimConnNotFoundException("No vm found '{}'".format(vm_id))
1590             else:
1591                 self._format_vimconn_exception(e)
1592         except Exception as e:
1593             self._format_vimconn_exception(e)
1594
1595     def delete_flavor(self, flavor_id):
1596         raise vimconn.VimConnAuthException(
1597             "It is not possible to delete a FLAVOR in AZURE"
1598         )
1599
1600     def delete_tenant(self, tenant_id):
1601         raise vimconn.VimConnAuthException(
1602             "It is not possible to delete a TENANT in AZURE"
1603         )
1604
1605     def delete_image(self, image_id):
1606         raise vimconn.VimConnAuthException(
1607             "It is not possible to delete a IMAGE in AZURE"
1608         )
1609
1610     def get_vminstance(self, vm_id):
1611         """
1612         Obtaing the vm instance data from v_id
1613         """
1614         self.logger.debug("get vm instance: %s", vm_id)
1615         self._reload_connection()
1616         try:
1617             resName = self._get_resource_name_from_resource_id(vm_id)
1618             vm = self.conn_compute.virtual_machines.get(self.resource_group, resName)
1619         except ResourceNotFoundError:
1620             raise vimconn.VimConnNotFoundException(
1621                 "No vminstance found '{}'".format(vm_id)
1622             )
1623         except CloudError as e:
1624             if e.error.error and "notfound" in e.error.error.lower():
1625                 raise vimconn.VimConnNotFoundException(
1626                     "No vminstance found '{}'".format(vm_id)
1627                 )
1628             else:
1629                 self._format_vimconn_exception(e)
1630         except Exception as e:
1631             self._format_vimconn_exception(e)
1632
1633         return vm
1634
1635     def get_flavor(self, flavor_id):
1636         """
1637         Obtains the flavor_data from the flavor_id
1638         """
1639         self._reload_connection()
1640         self.logger.debug("get flavor from id: %s", flavor_id)
1641         flavor_data = self._get_flavor_id_from_flavor_name(flavor_id)
1642
1643         if flavor_data:
1644             flavor = {
1645                 "id": flavor_id,
1646                 "name": flavor_id,
1647                 "ram": flavor_data["memoryInMB"],
1648                 "vcpus": flavor_data["numberOfCores"],
1649                 "disk": flavor_data["resourceDiskSizeInMB"] / 1024,
1650             }
1651
1652             return flavor
1653         else:
1654             raise vimconn.VimConnNotFoundException(
1655                 "flavor '{}' not found".format(flavor_id)
1656             )
1657
1658     def get_tenant_list(self, filter_dict={}):
1659         """Obtains the list of tenants
1660         For the azure connector only the azure tenant will be returned if it is compatible
1661         with filter_dict
1662         """
1663         tenants_azure = [{"name": self.tenant, "id": self.tenant}]
1664         tenant_list = []
1665
1666         self.logger.debug("get tenant list: %s", filter_dict)
1667         for tenant_azure in tenants_azure:
1668             if filter_dict:
1669                 if (
1670                     filter_dict.get("id")
1671                     and str(tenant_azure.get("id")) != filter_dict["id"]
1672                 ):
1673                     continue
1674
1675                 if (
1676                     filter_dict.get("name")
1677                     and str(tenant_azure.get("name")) != filter_dict["name"]
1678                 ):
1679                     continue
1680
1681             tenant_list.append(tenant_azure)
1682
1683         return tenant_list
1684
1685     def refresh_nets_status(self, net_list):
1686         """Get the status of the networks
1687         Params: the list of network identifiers
1688         Returns a dictionary with:
1689             net_id:  #VIM id of this network
1690             status:  #Mandatory. Text with one of:
1691                      #  DELETED (not found at vim)
1692                      #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1693                      #  OTHER (Vim reported other status not understood)
1694                      #  ERROR (VIM indicates an ERROR status)
1695                      #  ACTIVE, INACTIVE, DOWN (admin down),
1696                      #  BUILD (on building process)
1697                      #
1698             error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1699              vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1700         """
1701         out_nets = {}
1702         self._reload_connection()
1703
1704         self.logger.debug("reload nets status net_list: %s", net_list)
1705         for net_id in net_list:
1706             try:
1707                 netName = self._get_net_name_from_resource_id(net_id)
1708                 resName = self._get_resource_name_from_resource_id(net_id)
1709
1710                 net = self.conn_vnet.subnets.get(
1711                     self.vnet_resource_group or self.resource_group, netName, resName
1712                 )
1713
1714                 out_nets[net_id] = {
1715                     "status": self.provision_state2osm[net.provisioning_state],
1716                     "vim_info": str(net),
1717                 }
1718             except CloudError as e:
1719                 if e.error.error and "notfound" in e.error.error.lower():
1720                     self.logger.info(
1721                         "Not found subnet net_name: %s, subnet_name: %s",
1722                         netName,
1723                         resName,
1724                     )
1725                     out_nets[net_id] = {"status": "DELETED", "error_msg": str(e)}
1726                 else:
1727                     self.logger.error(
1728                         "CloudError Exception %s when searching subnet", e
1729                     )
1730                     out_nets[net_id] = {
1731                         "status": "VIM_ERROR",
1732                         "error_msg": str(e),
1733                     }
1734             except vimconn.VimConnNotFoundException as e:
1735                 self.logger.error(
1736                     "VimConnNotFoundException %s when searching subnet", e
1737                 )
1738                 out_nets[net_id] = {
1739                     "status": "DELETED",
1740                     "error_msg": str(e),
1741                 }
1742             except Exception as e:
1743                 self.logger.error(
1744                     "Exception %s when searching subnet", e, exc_info=True
1745                 )
1746                 out_nets[net_id] = {
1747                     "status": "VIM_ERROR",
1748                     "error_msg": str(e),
1749                 }
1750
1751         return out_nets
1752
1753     def refresh_vms_status(self, vm_list):
1754         """Get the status of the virtual machines and their interfaces/ports
1755         Params: the list of VM identifiers
1756         Returns a dictionary with:
1757             vm_id:          # VIM id of this Virtual Machine
1758                 status:     # Mandatory. Text with one of:
1759                             #  DELETED (not found at vim)
1760                             #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1761                             #  OTHER (Vim reported other status not understood)
1762                             #  ERROR (VIM indicates an ERROR status)
1763                             #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1764                             #  BUILD (on building process), ERROR
1765                             #  ACTIVE:NoMgmtIP (Active but none of its interfaces has an IP address
1766                             #     (ACTIVE:NoMgmtIP is not returned for Azure)
1767                             #
1768                 error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1769                 vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1770                 interfaces: list with interface info. Each item a dictionary with:
1771                     vim_interface_id -  The ID of the interface
1772                     mac_address - The MAC address of the interface.
1773                     ip_address - The IP address of the interface within the subnet.
1774         """
1775         out_vms = {}
1776         self._reload_connection()
1777
1778         self.logger.debug("refresh vm status vm_list: %s", vm_list)
1779         search_vm_list = vm_list or {}
1780
1781         for vm_id in search_vm_list:
1782             out_vm = {}
1783             try:
1784                 res_name = self._get_resource_name_from_resource_id(vm_id)
1785
1786                 vm = self.conn_compute.virtual_machines.get(
1787                     self.resource_group, res_name
1788                 )
1789                 out_vm["vim_info"] = str(vm)
1790                 out_vm["status"] = self.provision_state2osm.get(
1791                     vm.provisioning_state, "OTHER"
1792                 )
1793
1794                 if vm.provisioning_state == "Succeeded":
1795                     # check if machine is running or stopped
1796                     instance_view = self.conn_compute.virtual_machines.instance_view(
1797                         self.resource_group, res_name
1798                     )
1799
1800                     for status in instance_view.statuses:
1801                         splitted_status = status.code.split("/")
1802                         if (
1803                             len(splitted_status) == 2
1804                             and splitted_status[0] == "PowerState"
1805                         ):
1806                             out_vm["status"] = self.power_state2osm.get(
1807                                 splitted_status[1], "OTHER"
1808                             )
1809
1810                 network_interfaces = vm.network_profile.network_interfaces
1811                 out_vm["interfaces"] = self._get_vm_interfaces_status(
1812                     vm_id, network_interfaces
1813                 )
1814
1815             except CloudError as e:
1816                 if e.error.error and "notfound" in e.error.error.lower():
1817                     self.logger.debug("Not found vm id: %s", vm_id)
1818                     out_vm["status"] = "DELETED"
1819                     out_vm["error_msg"] = str(e)
1820                     out_vm["vim_info"] = None
1821                 else:
1822                     # maybe connection error or another type of error, return vim error
1823                     self.logger.error("Exception %s refreshing vm_status", e)
1824                     out_vm["status"] = "VIM_ERROR"
1825                     out_vm["error_msg"] = str(e)
1826                     out_vm["vim_info"] = None
1827             except Exception as e:
1828                 self.logger.error("Exception %s refreshing vm_status", e, exc_info=True)
1829                 out_vm["status"] = "VIM_ERROR"
1830                 out_vm["error_msg"] = str(e)
1831                 out_vm["vim_info"] = None
1832
1833             out_vms[vm_id] = out_vm
1834
1835         return out_vms
1836
1837     def _get_vm_interfaces_status(self, vm_id, interfaces):
1838         """
1839         Gets the interfaces detail for a vm
1840         :param interfaces: List of interfaces.
1841         :return: Dictionary with list of interfaces including, vim_interface_id, mac_address and ip_address
1842         """
1843         try:
1844             interface_list = []
1845             for network_interface in interfaces:
1846                 interface_dict = {}
1847                 nic_name = self._get_resource_name_from_resource_id(
1848                     network_interface.id
1849                 )
1850                 interface_dict["vim_interface_id"] = network_interface.id
1851
1852                 nic_data = self.conn_vnet.network_interfaces.get(
1853                     self.resource_group,
1854                     nic_name,
1855                 )
1856
1857                 ips = []
1858                 if nic_data.ip_configurations[0].public_ip_address:
1859                     self.logger.debug("Obtain public ip address")
1860                     public_ip_name = self._get_resource_name_from_resource_id(
1861                         nic_data.ip_configurations[0].public_ip_address.id
1862                     )
1863                     public_ip = self.conn_vnet.public_ip_addresses.get(
1864                         self.resource_group, public_ip_name
1865                     )
1866                     self.logger.debug("Public ip address is: %s", public_ip.ip_address)
1867                     ips.append(public_ip.ip_address)
1868
1869                 private_ip = nic_data.ip_configurations[0].private_ip_address
1870                 ips.append(private_ip)
1871
1872                 interface_dict["mac_address"] = nic_data.mac_address
1873                 interface_dict["ip_address"] = ";".join(ips)
1874                 interface_list.append(interface_dict)
1875
1876             return interface_list
1877         except Exception as e:
1878             self.logger.error(
1879                 "Exception %s obtaining interface data for vm: %s",
1880                 e,
1881                 vm_id,
1882                 exc_info=True,
1883             )
1884             self._format_vimconn_exception(e)
1885
1886     def _get_default_admin_user(self, image_id):
1887         if "ubuntu" in image_id.lower():
1888             return "ubuntu"
1889         else:
1890             return self._default_admin_user
1891
1892     def migrate_instance(self, vm_id, compute_host=None):
1893         """
1894         Migrate a vdu
1895         param:
1896             vm_id: ID of an instance
1897             compute_host: Host to migrate the vdu to
1898         """
1899         # TODO: Add support for migration
1900         raise vimconn.VimConnNotImplemented("Not implemented")
1901
1902     def resize_instance(self, vm_id, flavor_id=None):
1903         """
1904         resize a vdu
1905         param:
1906             vm_id: ID of an instance
1907             flavor_id: flavor id to resize the vdu
1908         """
1909         # TODO: Add support for resize
1910         raise vimconn.VimConnNotImplemented("Not implemented")
1911
1912
1913 if __name__ == "__main__":
1914     # Init logger
1915     log_format = "%(asctime)s %(levelname)s %(name)s %(filename)s:%(lineno)s %(funcName)s(): %(message)s"
1916     log_formatter = logging.Formatter(log_format, datefmt="%Y-%m-%dT%H:%M:%S")
1917     handler = logging.StreamHandler()
1918     handler.setFormatter(log_formatter)
1919     logger = logging.getLogger("ro.vim.azure")
1920     # logger.setLevel(level=logging.ERROR)
1921     # logger.setLevel(level=logging.INFO)
1922     logger.setLevel(level=logging.DEBUG)
1923     logger.addHandler(handler)
1924
1925     # Making some basic test
1926     vim_id = "azure"
1927     vim_name = "azure"
1928     needed_test_params = {
1929         "client_id": "AZURE_CLIENT_ID",
1930         "secret": "AZURE_SECRET",
1931         "tenant": "AZURE_TENANT",
1932         "resource_group": "AZURE_RESOURCE_GROUP",
1933         "subscription_id": "AZURE_SUBSCRIPTION_ID",
1934         "vnet_name": "AZURE_VNET_NAME",
1935     }
1936     test_params = {}
1937
1938     for param, env_var in needed_test_params.items():
1939         value = getenv(env_var)
1940
1941         if not value:
1942             raise Exception("Provide a valid value for env '{}'".format(env_var))
1943
1944         test_params[param] = value
1945
1946     config = {
1947         "region_name": getenv("AZURE_REGION_NAME", "northeurope"),
1948         "resource_group": getenv("AZURE_RESOURCE_GROUP"),
1949         "subscription_id": getenv("AZURE_SUBSCRIPTION_ID"),
1950         "pub_key": getenv("AZURE_PUB_KEY", None),
1951         "vnet_name": getenv("AZURE_VNET_NAME", "osm_vnet"),
1952     }
1953
1954     azure = vimconnector(
1955         vim_id,
1956         vim_name,
1957         tenant_id=test_params["tenant"],
1958         tenant_name=None,
1959         url=None,
1960         url_admin=None,
1961         user=test_params["client_id"],
1962         passwd=test_params["secret"],
1963         log_level=None,
1964         config=config,
1965     )