osm_ro/vimconn_azure.py

   1 # -*- coding: utf-8 -*-
   2
   3 __author__='Sergio Gonzalez'
   4 __date__ ='$18-apr-2019 23:59:59$'
   5
   6 import vimconn
   7 import logging
   8 import netaddr
   9
  10 from os import getenv
  11 from uuid import uuid4
  12
  13 from azure.common.credentials import ServicePrincipalCredentials
  14 from azure.mgmt.resource import ResourceManagementClient
  15 from azure.mgmt.network import NetworkManagementClient
  16 from azure.mgmt.compute import ComputeManagementClient
  17
  18 from msrestazure.azure_exceptions import CloudError
  19
  20 class vimconnector(vimconn.vimconnector):
  21
  22     provision_state2osm = {
  23         "Deleting": "INACTIVE",
  24         "Failed": "ERROR",
  25         "Succeeded": "ACTIVE",
  26         "Updating": "BUILD",
  27     }
  28
  29     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None,
  30                  config={}, persistent_info={}):
  31
  32         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  33                                       config, persistent_info)
  34
  35         self.vnet_address_space = None
  36         # LOGGER
  37         self.logger = logging.getLogger('openmano.vim.azure')
  38         if log_level:
  39             logging.basicConfig()
  40             self.logger.setLevel(getattr(logging, log_level))
  41
  42         # CREDENTIALS
  43         self.credentials = ServicePrincipalCredentials(
  44             client_id=user,
  45             secret=passwd,
  46             tenant=(tenant_id or tenant_name)
  47         )
  48
  49         # SUBSCRIPTION
  50         if 'subscription_id' in config:
  51             self.subscription_id = config.get('subscription_id')
  52             self.logger.debug('Setting subscription '+str(self.subscription_id))
  53         else:
  54             raise vimconn.vimconnException('Subscription not specified')
  55         # REGION
  56         if 'region_name' in config:
  57             self.region = config.get('region_name')
  58         else:
  59             raise vimconn.vimconnException('Azure region_name is not specified at config')
  60         # RESOURCE_GROUP
  61         if 'resource_group' in config:
  62             self.resource_group = config.get('resource_group')
  63         else:
  64             raise vimconn.vimconnException('Azure resource_group is not specified at config')
  65         # VNET_NAME
  66         if 'vnet_name' in config:
  67             self.vnet_name = config["vnet_name"]
  68
  69         # public ssh key
  70         self.pub_key = config.get('pub_key')
  71
  72     def _reload_connection(self):
  73         """
  74         Sets connections to work with Azure service APIs
  75         :return:
  76         """
  77         self.logger.debug('Reloading API Connection')
  78         try:
  79             self.conn = ResourceManagementClient(self.credentials, self.subscription_id)
  80             self.conn_compute = ComputeManagementClient(self.credentials, self.subscription_id)
  81             self.conn_vnet = NetworkManagementClient(self.credentials, self.subscription_id)
  82             self._check_or_create_resource_group()
  83             self._check_or_create_vnet()
  84         except Exception as e:
  85             self.format_vimconn_exception(e)
  86
  87     def _get_resource_name_from_resource_id(self, resource_id):
  88         return str(resource_id.split('/')[-1])
  89
  90     def _get_location_from_resource_group(self, resource_group_name):
  91         return self.conn.resource_groups.get(resource_group_name).location
  92
  93     def _get_resource_group_name_from_resource_id(self, resource_id):
  94         return str(resource_id.split('/')[4])
  95
  96     def _check_subnets_for_vm(self, net_list):
  97         # All subnets must belong to the same resource group and vnet
  98         if len(set(self._get_resource_group_name_from_resource_id(net['id']) +
  99                    self._get_resource_name_from_resource_id(net['id']) for net in net_list)) != 1:
 100             raise self.format_vimconn_exception('Azure VMs can only attach to subnets in same VNET')
 101
 102     def format_vimconn_exception(self, e):
 103         """
 104         Params: an Exception object
 105         :param e:
 106         :return: Raises the proper vimconnException
 107         """
 108         self.conn = None
 109         self.conn_vnet = None
 110         raise vimconn.vimconnConnectionException(type(e).__name__ + ': ' + str(e))
 111
 112     def _check_or_create_resource_group(self):
 113         """
 114         Creates a resource group in indicated region
 115         :return: None
 116         """
 117         self.logger.debug('Creating RG {} in location {}'.format(self.resource_group, self.region))
 118         self.conn.resource_groups.create_or_update(self.resource_group, {'location': self.region})
 119
 120     def _check_or_create_vnet(self):
 121         try:
 122             vnet = self.conn_vnet.virtual_networks.get(self.resource_group, self.vnet_name)
 123             self.vnet_address_space = vnet.address_space.address_prefixes[0]
 124             self.vnet_id = vnet.id
 125             return
 126         except CloudError as e:
 127             if e.error.error == "ResourceNotFound":
 128                 pass
 129             else:
 130                 raise
 131         # if not exist, creates it
 132         try:
 133             vnet_params = {
 134                 'location': self.region,
 135                 'address_space': {
 136                     'address_prefixes': ["10.0.0.0/8"]
 137                 },
 138             }
 139             self.vnet_address_space = "10.0.0.0/8"
 140             self.conn_vnet.virtual_networks.create_or_update(self.resource_group, self.vnet_name, vnet_params)
 141             vnet = self.conn_vnet.virtual_networks.get(self.resource_group, self.vnet_name)
 142             self.vnet_id = vnet.id
 143         except Exception as e:
 144             self.format_vimconn_exception(e)
 145
 146     def new_network(self, net_name, net_type, ip_profile=None, shared=False, vlan=None):
 147         """
 148         Adds a tenant network to VIM
 149         :param net_name: name of the network
 150         :param net_type:
 151         :param ip_profile: is a dict containing the IP parameters of the network (Currently only IPv4 is implemented)
 152                 'ip-version': can be one of ['IPv4','IPv6']
 153                 'subnet-address': ip_prefix_schema, that is X.X.X.X/Y
 154                 'gateway-address': (Optional) ip_schema, that is X.X.X.X
 155                 'dns-address': (Optional) ip_schema,
 156                 'dhcp': (Optional) dict containing
 157                     'enabled': {'type': 'boolean'},
 158                     'start-address': ip_schema, first IP to grant
 159                     'count': number of IPs to grant.
 160         :param shared:
 161         :param vlan:
 162         :return: a tuple with the network identifier and created_items, or raises an exception on error
 163             created_items can be None or a dictionary where this method can include key-values that will be passed to
 164             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 165             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 166             as not present.
 167         """
 168
 169         return self._new_subnet(net_name, ip_profile)
 170
 171     def _new_subnet(self, net_name, ip_profile):
 172         """
 173         Adds a tenant network to VIM. It creates a new VNET with a single subnet
 174         :param net_name:
 175         :param ip_profile:
 176         :return:
 177         """
 178         self.logger.debug('Adding a subnet to VNET '+self.vnet_name)
 179         self._reload_connection()
 180
 181         if ip_profile is None:
 182             # get a non used vnet ip range /24 and allocate automatically inside the range self.vnet_address_space
 183             used_subnets = self.get_network_list()
 184             for ip_range in netaddr.IPNetwork(self.vnet_address_space).subnet(24):
 185                 for used_subnet in used_subnets:
 186                     subnet_range = netaddr.IPNetwork(used_subnet["cidr_block"])
 187                     if subnet_range in ip_range or ip_range in subnet_range:
 188                         # this range overlaps with an existing subnet ip range. Breaks and look for another
 189                         break
 190                 else:
 191                     ip_profile = {"subnet_address": str(ip_range)}
 192                     break
 193             else:
 194                 vimconn.vimconnException("Cannot find a non-used subnet range in {}".format(self.vnet_address_space))
 195         try:
 196             subnet_name = "{}-{}".format(net_name[:24], uuid4())
 197             subnet_params= {
 198                 'address_prefix': ip_profile['subnet_address']
 199             }
 200             self.conn_vnet.subnets.create_or_update(self.resource_group, self.vnet_name, subnet_name, subnet_params)
 201             return "{}/subnet/{}".format(self.vnet_id, subnet_name), None
 202         except Exception as e:
 203             self.format_vimconn_exception(e)
 204
 205     def _create_nic(self, subnet_id, nic_name, static_ip=None):
 206         self._reload_connection()
 207
 208         resource_group_name=self._get_resource_group_name_from_resource_id(subnet_id)
 209         location = self._get_location_from_resource_group(resource_group_name)
 210
 211         if static_ip:
 212             async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
 213                 resource_group_name,
 214                 nic_name,
 215                 {
 216                     'location': location,
 217                     'ip_configurations': [{
 218                         'name': nic_name + 'ipconfiguration',
 219                         'privateIPAddress': static_ip,
 220                         'privateIPAllocationMethod': 'Static',
 221                         'subnet': {
 222                             'id': subnet_id
 223                         }
 224                     }]
 225                 }
 226             )
 227         else:
 228             async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
 229                 resource_group_name,
 230                 nic_name,
 231                 {
 232                     'location': location,
 233                     'ip_configurations': [{
 234                         'name': nic_name + 'ipconfiguration',
 235                         'subnet': {
 236                             'id': subnet_id
 237                         }
 238                     }]
 239                 }
 240             )
 241
 242         return async_nic_creation.result()
 243
 244     def get_image_list(self, filter_dict={}):
 245         """
 246         The urn contains for marketplace  'publisher:offer:sku:version'
 247
 248         :param filter_dict:
 249         :return:
 250         """
 251         image_list = []
 252
 253         self._reload_connection()
 254         if filter_dict.get("name"):
 255             params = filter_dict["name"].split(":")
 256             if len(params) >= 3:
 257                 publisher = params[0]
 258                 offer = params[1]
 259                 sku = params[2]
 260                 version = None
 261                 if len(params) == 4:
 262                     version = params[3]
 263                 images = self.conn_compute.virtual_machine_images.list(self.region, publisher, offer, sku)
 264                 for image in images:
 265                     if version:
 266                         image_version = str(image.id).split("/")[-1]
 267                         if image_version != version:
 268                             continue
 269                     image_list.append({
 270                         'id': str(image.id),
 271                         'name': self._get_resource_name_from_resource_id(image.id)
 272                     })
 273                 return image_list
 274
 275         images = self.conn_compute.virtual_machine_images.list()
 276
 277         for image in images:
 278             # TODO implement filter_dict
 279             if filter_dict:
 280                 if filter_dict.get("id") and str(image.id) != filter_dict["id"]:
 281                     continue
 282                 if filter_dict.get("name") and \
 283                         self._get_resource_name_from_resource_id(image.id) != filter_dict["name"]:
 284                     continue
 285                 # TODO add checksum
 286             image_list.append({
 287                 'id': str(image.id),
 288                 'name': self._get_resource_name_from_resource_id(image.id),
 289             })
 290         return image_list
 291
 292     def get_network_list(self, filter_dict={}):
 293         """Obtain tenant networks of VIM
 294         Filter_dict can be:
 295             name: network name
 296             id: network uuid
 297             shared: boolean
 298             tenant_id: tenant
 299             admin_state_up: boolean
 300             status: 'ACTIVE'
 301         Returns the network list of dictionaries
 302         """
 303         self.logger.debug('Getting all subnets from VIM')
 304         try:
 305             self._reload_connection()
 306             vnet = self.conn_vnet.virtual_networks.get(self.config["resource_group"], self.vnet_name)
 307             subnet_list = []
 308
 309             for subnet in vnet.subnets:
 310                 # TODO implement filter_dict
 311                 if filter_dict:
 312                     if filter_dict.get("id") and str(subnet.id) != filter_dict["id"]:
 313                         continue
 314                     if filter_dict.get("name") and \
 315                             self._get_resource_name_from_resource_id(subnet.id) != filter_dict["name"]:
 316                         continue
 317
 318                 subnet_list.append({
 319                     'id': str(subnet.id),
 320                      'name': self._get_resource_name_from_resource_id(subnet.id),
 321                      'status': str(vnet.provisioning_state),  # TODO Does subnet contains status???
 322                      'cidr_block': str(subnet.address_prefix)
 323                     }
 324                 )
 325             return subnet_list
 326         except Exception as e:
 327             self.format_vimconn_exception(e)
 328
 329     def new_vminstance(self, vm_name, description, start, image_id, flavor_id, net_list, cloud_config=None,
 330                        disk_list=None, availability_zone_index=None, availability_zone_list=None):
 331
 332         return self._new_vminstance(vm_name, image_id, flavor_id, net_list)
 333
 334     def _new_vminstance(self, vm_name, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 335                         availability_zone_index=None, availability_zone_list=None):
 336         #Create NICs
 337         self._check_subnets_for_vm(net_list)
 338         vm_nics = []
 339         for idx, net in enumerate(net_list):
 340             subnet_id=net['subnet_id']
 341             nic_name = vm_name + '-nic-'+str(idx)
 342             vm_nic = self._create_nic(subnet_id, nic_name)
 343             vm_nics.append({ 'id': str(vm_nic.id)})
 344
 345         try:
 346             vm_parameters = {
 347                 'location': self.region,
 348                 'os_profile': {
 349                     'computer_name': vm_name,  # TODO if vm_name cannot be repeated add uuid4() suffix
 350                     'admin_username': 'sergio',  # TODO is it mandatory???
 351                     'linuxConfiguration': {
 352                         'disablePasswordAuthentication': 'true',
 353                         'ssh': {
 354                           'publicKeys': [
 355                             {
 356                               'path': '/home/sergio/.ssh/authorized_keys',
 357                               'keyData': self.pub_key
 358                             }
 359                           ]
 360                         }
 361                     }
 362
 363                 },
 364                 'hardware_profile': {
 365                     'vm_size':flavor_id
 366                 },
 367                 'storage_profile': {
 368                     'image_reference': image_id
 369                 },
 370                 'network_profile': {
 371                     'network_interfaces': [
 372                         vm_nics[0]
 373                     ]
 374                 }
 375             }
 376             creation_result = self.conn_compute.virtual_machines.create_or_update(
 377                 self.resource_group,
 378                 vm_name,
 379                 vm_parameters
 380             )
 381
 382             run_command_parameters = {
 383                 'command_id': 'RunShellScript', # For linux, don't change it
 384                 'script': [
 385                 'date > /home/sergio/test.txt'
 386                 ]
 387             }
 388             poller = self.conn_compute.virtual_machines.run_command(
 389                 self.resource_group,
 390                 vm_name,
 391                 run_command_parameters
 392             )
 393             # TODO return a tuple (vm-ID, None)
 394         except Exception as e:
 395             self.format_vimconn_exception(e)
 396
 397     def get_flavor_id_from_data(self, flavor_dict):
 398         self.logger.debug("Getting flavor id from data")
 399         self._reload_connection()
 400         vm_sizes_list = [vm_size.serialize() for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region)]
 401
 402         cpus = flavor_dict['vcpus']
 403         memMB = flavor_dict['ram']
 404
 405         filteredSizes = [size for size in vm_sizes_list if size['numberOfCores'] > cpus and size['memoryInMB'] > memMB]
 406         listedFilteredSizes = sorted(filteredSizes, key=lambda k: k['numberOfCores'])
 407
 408         return listedFilteredSizes[0]['name']
 409
 410     def check_vim_connectivity(self):
 411         try:
 412             self._reload_connection()
 413             return True
 414         except Exception as e:
 415             raise vimconn.vimconnException("Connectivity issue with Azure API: {}".format(e))
 416
 417     def get_network(self, net_id):
 418         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 419         resName = self._get_resource_name_from_resource_id(net_id)
 420
 421         self._reload_connection()
 422         vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
 423         return vnet
 424
 425     def delete_network(self, net_id):
 426         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 427         resName = self._get_resource_name_from_resource_id(net_id)
 428
 429         self._reload_connection()
 430         self.conn_vnet.virtual_networks.delete(resGroup, resName)
 431
 432     def delete_vminstance(self, vm_id):
 433         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 434         resName = self._get_resource_name_from_resource_id(net_id)
 435
 436         self._reload_connection()
 437         self.conn_compute.virtual_machines.delete(resGroup, resName)
 438
 439     def get_vminstance(self, vm_id):
 440         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 441         resName = self._get_resource_name_from_resource_id(net_id)
 442
 443         self._reload_connection()
 444         vm=self.conn_compute.virtual_machines.get(resGroup, resName)
 445
 446         return vm
 447
 448     def get_flavor(self, flavor_id):
 449         self._reload_connection()
 450         for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region):
 451             if vm_size.name == flavor_id :
 452                 return vm_size
 453
 454     def refresh_nets_status(self, net_list):
 455         out_nets = {}
 456         self._reload_connection()
 457         for net_id in net_list:
 458             try:
 459                 resGroup = self._get_resource_group_name_from_resource_id(net_id)
 460                 resName = self._get_resource_name_from_resource_id(net_id)
 461
 462                 vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
 463                 out_nets[net_id] ={
 464                     "status": self.provision_state2osm[vnet.provisioning_state],
 465                     "vim_info": str(vnet)
 466                 }
 467             except CloudError as e:
 468                 if e.error.error == "ResourceNotFound":
 469                     out_nets[net_id] = {
 470                         "status": "DELETED",
 471                     }
 472                 else:
 473                     raise
 474             except Exception as e:
 475                 # TODO distinguish when it is deleted
 476                 out_nets[net_id] = {
 477                     "status": "VIM_ERROR",
 478                     "vim_info": str(vnet),
 479                     "error_msg": str(e)
 480                 }
 481
 482         return out_nets
 483
 484     def refresh_vms_status(self, vm_list):
 485         out_vms = {}
 486         self._reload_connection()
 487         for vm_id in vm_list:
 488             try:
 489                 resGroup = self._get_resource_group_name_from_resource_id(vm_id)
 490                 resName = self._get_resource_name_from_resource_id(vm_id)
 491
 492                 vm = self.conn_compute.virtual_machines.get(resGroup, resName)
 493                 out_vms[vm_id] ={
 494                     "status": self.provision_state2osm[vm.provisioning_state],
 495                     "vim_info": str(vm)
 496                 }
 497             except CloudError as e:
 498                 if e.error.error == "ResourceNotFound":
 499                     out_vms[vm_id] = {
 500                         "status": "DELETED",
 501                     }
 502                 else:
 503                     raise
 504             except Exception as e:
 505                 # TODO distinguish when it is deleted
 506                 out_vms[vm_id] = {
 507                     "status": "VIM_ERROR",
 508                     "vim_info": str(vm),
 509                     "error_msg": str(e)
 510                 }
 511
 512         return out_vms
 513
 514 # TODO get_vminstance_console  for getting console
 515
 516 if __name__ == "__main__":
 517
 518     # Making some basic test
 519     vim_id='azure'
 520     vim_name='azure'
 521     needed_test_params = {
 522         "client_id": "AZURE_CLIENT_ID",
 523         "secret": "AZURE_SECRET",
 524         "tenant": "AZURE_TENANT",
 525         "resource_group": "AZURE_RESOURCE_GROUP",
 526         "subscription_id": "AZURE_SUBSCRIPTION_ID",
 527         "vnet_name": "AZURE_VNET_NAME",
 528     }
 529     test_params = {}
 530
 531     for param, env_var in needed_test_params.items():
 532         value = getenv(env_var)
 533         if not value:
 534             raise Exception("Provide a valid value for env '{}'".format(env_var))
 535         test_params[param] = value
 536
 537     config = {
 538             'region_name': getenv("AZURE_REGION_NAME", 'westeurope'),
 539             'resource_group': getenv("AZURE_RESOURCE_GROUP"),
 540             'subscription_id': getenv("AZURE_SUBSCRIPTION_ID"),
 541             'pub_key': getenv("AZURE_PUB_KEY", None),
 542             'vnet_name': getenv("AZURE_VNET_NAME", 'myNetwork'),
 543     }
 544
 545     virtualMachine = {
 546         'name': 'sergio',
 547         'description': 'new VM',
 548         'status': 'running',
 549         'image': {
 550             'publisher': 'Canonical',
 551             'offer': 'UbuntuServer',
 552             'sku': '16.04.0-LTS',
 553             'version': 'latest'
 554         },
 555         'hardware_profile': {
 556             'vm_size': 'Standard_DS1_v2'
 557         },
 558         'networks': [
 559             'sergio'
 560         ]
 561     }
 562
 563     vnet_config = {
 564         'subnet_address': '10.1.2.0/24',
 565         #'subnet_name': 'subnet-oam'
 566     }
 567     ###########################
 568
 569     azure = vimconnector(vim_id, vim_name, tenant_id=test_params["tenant"], tenant_name=None, url=None, url_admin=None,
 570                          user=test_params["client_id"], passwd=test_params["secret"], log_level=None, config=config)
 571
 572     # azure.get_flavor_id_from_data("here")
 573     # subnets=azure.get_network_list()
 574     # azure.new_vminstance(virtualMachine['name'], virtualMachine['description'], virtualMachine['status'],
 575     #                      virtualMachine['image'], virtualMachine['hardware_profile']['vm_size'], subnets)
 576
 577     azure.new_network("mynet", None)
 578     net_id = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/Microsoft."\
 579              "Network/virtualNetworks/test"
 580     net_id_not_found = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/"\
 581                        "Microsoft.Network/virtualNetworks/testALF"
 582     azure.refresh_nets_status([net_id, net_id_not_found])