osm_ro/vimconn_azure.py

   1 # -*- coding: utf-8 -*-
   2
   3 __author__='Sergio Gonzalez'
   4 __date__ ='$18-apr-2019 23:59:59$'
   5
   6 import vimconn
   7 import logging
   8
   9 from os import getenv
  10 from uuid import uuid4
  11
  12 from azure.common.credentials import ServicePrincipalCredentials
  13 from azure.mgmt.resource import ResourceManagementClient
  14 from azure.mgmt.network import NetworkManagementClient
  15 from azure.mgmt.compute import ComputeManagementClient
  16
  17 from msrestazure.azure_exceptions import CloudError
  18
  19 class vimconnector(vimconn.vimconnector):
  20
  21     provision_state2osm = {
  22         "Deleting": "INACTIVE",
  23         "Failed": "ERROR",
  24         "Succeeded": "ACTIVE",
  25         "Updating": "BUILD",
  26     }
  27
  28     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None,
  29                  config={}, persistent_info={}):
  30
  31         vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
  32                                       config, persistent_info)
  33
  34         self.vnet_address_space = None
  35         # LOGGER
  36         self.logger = logging.getLogger('openmano.vim.azure')
  37         if log_level:
  38             logging.basicConfig()
  39             self.logger.setLevel(getattr(logging, log_level))
  40
  41         # CREDENTIALS
  42         self.credentials = ServicePrincipalCredentials(
  43             client_id=user,
  44             secret=passwd,
  45             tenant=(tenant_id or tenant_name)
  46         )
  47
  48         # SUBSCRIPTION
  49         if 'subscription_id' in config:
  50             self.subscription_id = config.get('subscription_id')
  51             self.logger.debug('Setting subscription '+str(self.subscription_id))
  52         else:
  53             raise vimconn.vimconnException('Subscription not specified')
  54         # REGION
  55         if 'region_name' in config:
  56             self.region = config.get('region_name')
  57         else:
  58             raise vimconn.vimconnException('Azure region_name is not specified at config')
  59         # RESOURCE_GROUP
  60         if 'resource_group' in config:
  61             self.resource_group = config.get('resource_group')
  62         else:
  63             raise vimconn.vimconnException('Azure resource_group is not specified at config')
  64         # VNET_NAME
  65         if 'vnet_name' in config:
  66             self.vnet_name = config["vnet_name"]
  67
  68         # public ssh key
  69         self.pub_key = config.get('pub_key')
  70
  71     def _reload_connection(self):
  72         """
  73         Sets connections to work with Azure service APIs
  74         :return:
  75         """
  76         self.logger.debug('Reloading API Connection')
  77         try:
  78             self.conn = ResourceManagementClient(self.credentials, self.subscription_id)
  79             self.conn_compute = ComputeManagementClient(self.credentials, self.subscription_id)
  80             self.conn_vnet = NetworkManagementClient(self.credentials, self.subscription_id)
  81             self._check_or_create_resource_group()
  82             self._check_or_create_vnet()
  83         except Exception as e:
  84             self.format_vimconn_exception(e)
  85
  86     def _get_resource_name_from_resource_id(self, resource_id):
  87         return str(resource_id.split('/')[-1])
  88
  89     def _get_location_from_resource_group(self, resource_group_name):
  90         return self.conn.resource_groups.get(resource_group_name).location
  91
  92     def _get_resource_group_name_from_resource_id(self, resource_id):
  93         return str(resource_id.split('/')[4])
  94
  95     def _check_subnets_for_vm(self, net_list):
  96         # All subnets must belong to the same resource group and vnet
  97         if len(set(self._get_resource_group_name_from_resource_id(net['id']) +
  98                    self._get_resource_name_from_resource_id(net['id']) for net in net_list)) != 1:
  99             raise self.format_vimconn_exception('Azure VMs can only attach to subnets in same VNET')
 100
 101     def format_vimconn_exception(self, e):
 102         """
 103         Params: an Exception object
 104         :param e:
 105         :return: Raises the proper vimconnException
 106         """
 107         self.conn = None
 108         self.conn_vnet = None
 109         raise vimconn.vimconnConnectionException(type(e).__name__ + ': ' + str(e))
 110
 111     def _check_or_create_resource_group(self):
 112         """
 113         Creates a resource group in indicated region
 114         :return: None
 115         """
 116         self.logger.debug('Creating RG {} in location {}'.format(self.resource_group, self.region))
 117         self.conn.resource_groups.create_or_update(self.resource_group, {'location': self.region})
 118
 119     def _check_or_create_vnet(self):
 120         try:
 121             vnet = self.conn_vnet.virtual_networks.get(self.resource_group, self.vnet_name)
 122             self.vnet_address_space = vnet.address_space.address_prefixes[0]
 123             return
 124         except CloudError as e:
 125             if e.error.error == "ResourceNotFound":
 126                 pass
 127             else:
 128                 raise
 129         # if not exist, creates it
 130         try:
 131             vnet_params = {
 132                 'location': self.region,
 133                 'address_space': {
 134                     'address_prefixes': ["10.0.0.0/8"]
 135                 },
 136             }
 137             self.vnet_address_space = "10.0.0.0/8"
 138             self.conn_vnet.virtual_networks.create_or_update(self.resource_group, self.vnet_name, vnet_params)
 139         except Exception as e:
 140             self.format_vimconn_exception(e)
 141
 142     def new_network(self, net_name, net_type, ip_profile=None, shared=False, vlan=None):
 143         """
 144         Adds a tenant network to VIM
 145         :param net_name: name of the network
 146         :param net_type:
 147         :param ip_profile: is a dict containing the IP parameters of the network (Currently only IPv4 is implemented)
 148                 'ip-version': can be one of ['IPv4','IPv6']
 149                 'subnet-address': ip_prefix_schema, that is X.X.X.X/Y
 150                 'gateway-address': (Optional) ip_schema, that is X.X.X.X
 151                 'dns-address': (Optional) ip_schema,
 152                 'dhcp': (Optional) dict containing
 153                     'enabled': {'type': 'boolean'},
 154                     'start-address': ip_schema, first IP to grant
 155                     'count': number of IPs to grant.
 156         :param shared:
 157         :param vlan:
 158         :return: a tuple with the network identifier and created_items, or raises an exception on error
 159             created_items can be None or a dictionary where this method can include key-values that will be passed to
 160             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 161             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 162             as not present.
 163         """
 164
 165         return self._new_subnet(net_name, ip_profile)
 166
 167     def _new_subnet(self, net_name, ip_profile):
 168         """
 169         Adds a tenant network to VIM. It creates a new VNET with a single subnet
 170         :param net_name:
 171         :param ip_profile:
 172         :return:
 173         """
 174         self.logger.debug('Adding a subnet to VNET '+self.vnet_name)
 175         self._reload_connection()
 176
 177         if ip_profile is None:
 178             # TODO get a non used vnet ip range /24 and allocate automatically inside the range self.vnet_address_space
 179             # use netaddr library
 180             raise vimconn.vimconnException('Azure cannot create VNET with no CIDR')
 181
 182         try:
 183             vnet_params= {
 184                 'location': self.region,
 185                 'subnets': [
 186                     {
 187                         'name': "{}-{}".format(net_name[:24], uuid4()),
 188                         'address_prefix': ip_profile['subnet_address']
 189                     }
 190                 ]
 191             }
 192             self.conn_vnet.virtual_networks.create_or_update(self.resource_group, self.vnet_name, vnet_params)
 193             # TODO return a tuple (subnet-ID, None)
 194         except Exception as e:
 195             self.format_vimconn_exception(e)
 196
 197     def _create_nic(self, subnet_id, nic_name, static_ip=None):
 198         self._reload_connection()
 199
 200         resource_group_name=self._get_resource_group_name_from_resource_id(subnet_id)
 201         location = self._get_location_from_resource_group(resource_group_name)
 202
 203         if static_ip:
 204             async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
 205                 resource_group_name,
 206                 nic_name,
 207                 {
 208                     'location': location,
 209                     'ip_configurations': [{
 210                         'name': nic_name + 'ipconfiguration',
 211                         'privateIPAddress': static_ip,
 212                         'privateIPAllocationMethod': 'Static',
 213                         'subnet': {
 214                             'id': subnet_id
 215                         }
 216                     }]
 217                 }
 218             )
 219         else:
 220             async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
 221                 resource_group_name,
 222                 nic_name,
 223                 {
 224                     'location': location,
 225                     'ip_configurations': [{
 226                         'name': nic_name + 'ipconfiguration',
 227                         'subnet': {
 228                             'id': subnet_id
 229                         }
 230                     }]
 231                 }
 232             )
 233
 234         return async_nic_creation.result()
 235
 236     def get_image_list(self, filter_dict={}):
 237         """
 238         The urn contains for marketplace  'publisher:offer:sku:version'
 239
 240         :param filter_dict:
 241         :return:
 242         """
 243         image_list = []
 244
 245         self._reload_connection()
 246         if filter_dict.get("name"):
 247             params = filter_dict["name"].split(":")
 248             if len(params) >= 3:
 249                 publisher = params[0]
 250                 offer = params[1]
 251                 sku = params[2]
 252                 version = None
 253                 if len(params) == 4:
 254                     version = params[3]
 255                 images = self.conn_compute.virtual_machine_images.list(self.region, publisher, offer, sku)
 256                 for image in images:
 257                     if version:
 258                         image_version = str(image.id).split("/")[-1]
 259                         if image_version != version:
 260                             continue
 261                     image_list.append({
 262                         'id': str(image.id),
 263                         'name': self._get_resource_name_from_resource_id(image.id)
 264                     })
 265                 return image_list
 266
 267         images = self.conn_compute.virtual_machine_images.list()
 268
 269         for image in images:
 270             # TODO implement filter_dict
 271             if filter_dict:
 272                 if filter_dict.get("id") and str(image.id) != filter_dict["id"]:
 273                     continue
 274                 if filter_dict.get("name") and \
 275                         self._get_resource_name_from_resource_id(image.id) != filter_dict["name"]:
 276                     continue
 277                 # TODO add checksum
 278             image_list.append({
 279                 'id': str(image.id),
 280                 'name': self._get_resource_name_from_resource_id(image.id),
 281             })
 282         return image_list
 283
 284     def get_network_list(self, filter_dict={}):
 285         """Obtain tenant networks of VIM
 286         Filter_dict can be:
 287             name: network name
 288             id: network uuid
 289             shared: boolean
 290             tenant_id: tenant
 291             admin_state_up: boolean
 292             status: 'ACTIVE'
 293         Returns the network list of dictionaries
 294         """
 295         self.logger.debug('Getting all subnets from VIM')
 296         try:
 297             self._reload_connection()
 298             vnet = self.conn_vnet.virtual_networks.get(self.config["resource_group"], self.vnet_name)
 299             subnet_list = []
 300
 301             for subnet in vnet.subnets:
 302                 # TODO implement filter_dict
 303                 if filter_dict:
 304                     if filter_dict.get("id") and str(subnet.id) != filter_dict["id"]:
 305                         continue
 306                     if filter_dict.get("name") and \
 307                             self._get_resource_name_from_resource_id(subnet.id) != filter_dict["name"]:
 308                         continue
 309
 310                 subnet_list.append({
 311                     'id': str(subnet.id),
 312                      'name': self._get_resource_name_from_resource_id(subnet.id),
 313                      'status': str(vnet.provisioning_state),  # TODO Does subnet contains status???
 314                      'cidr_block': str(subnet.address_prefix)
 315                     }
 316                 )
 317             return subnet_list
 318         except Exception as e:
 319             self.format_vimconn_exception(e)
 320
 321     def new_vminstance(self, vm_name, description, start, image_id, flavor_id, net_list, cloud_config=None,
 322                        disk_list=None, availability_zone_index=None, availability_zone_list=None):
 323
 324         return self._new_vminstance(vm_name, image_id, flavor_id, net_list)
 325
 326     def _new_vminstance(self, vm_name, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
 327                         availability_zone_index=None, availability_zone_list=None):
 328         #Create NICs
 329         self._check_subnets_for_vm(net_list)
 330         vm_nics = []
 331         for idx, net in enumerate(net_list):
 332             subnet_id=net['subnet_id']
 333             nic_name = vm_name + '-nic-'+str(idx)
 334             vm_nic = self._create_nic(subnet_id, nic_name)
 335             vm_nics.append({ 'id': str(vm_nic.id)})
 336
 337         try:
 338             vm_parameters = {
 339                 'location': self.region,
 340                 'os_profile': {
 341                     'computer_name': vm_name,  # TODO if vm_name cannot be repeated add uuid4() suffix
 342                     'admin_username': 'sergio',  # TODO is it mandatory???
 343                     'linuxConfiguration': {
 344                         'disablePasswordAuthentication': 'true',
 345                         'ssh': {
 346                           'publicKeys': [
 347                             {
 348                               'path': '/home/sergio/.ssh/authorized_keys',
 349                               'keyData': self.pub_key
 350                             }
 351                           ]
 352                         }
 353                     }
 354
 355                 },
 356                 'hardware_profile': {
 357                     'vm_size':flavor_id
 358                 },
 359                 'storage_profile': {
 360                     'image_reference': image_id
 361                 },
 362                 'network_profile': {
 363                     'network_interfaces': [
 364                         vm_nics[0]
 365                     ]
 366                 }
 367             }
 368             creation_result = self.conn_compute.virtual_machines.create_or_update(
 369                 self.resource_group,
 370                 vm_name,
 371                 vm_parameters
 372             )
 373
 374             run_command_parameters = {
 375                 'command_id': 'RunShellScript', # For linux, don't change it
 376                 'script': [
 377                 'date > /home/sergio/test.txt'
 378                 ]
 379             }
 380             poller = self.conn_compute.virtual_machines.run_command(
 381                 self.resource_group,
 382                 vm_name,
 383                 run_command_parameters
 384             )
 385             # TODO return a tuple (vm-ID, None)
 386         except Exception as e:
 387             self.format_vimconn_exception(e)
 388
 389     def get_flavor_id_from_data(self, flavor_dict):
 390         self.logger.debug("Getting flavor id from data")
 391         self._reload_connection()
 392         vm_sizes_list = [vm_size.serialize() for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region)]
 393
 394         cpus = flavor_dict['vcpus']
 395         memMB = flavor_dict['ram']
 396
 397         filteredSizes = [size for size in vm_sizes_list if size['numberOfCores'] > cpus and size['memoryInMB'] > memMB]
 398         listedFilteredSizes = sorted(filteredSizes, key=lambda k: k['numberOfCores'])
 399
 400         return listedFilteredSizes[0]['name']
 401
 402     def check_vim_connectivity(self):
 403         try:
 404             self._reload_connection()
 405             return True
 406         except Exception as e:
 407             raise vimconn.vimconnException("Connectivity issue with Azure API: {}".format(e))
 408
 409     def get_network(self, net_id):
 410         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 411         resName = self._get_resource_name_from_resource_id(net_id)
 412
 413         self._reload_connection()
 414         vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
 415
 416         return vnet
 417
 418     def delete_network(self, net_id):
 419         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 420         resName = self._get_resource_name_from_resource_id(net_id)
 421
 422         self._reload_connection()
 423         self.conn_vnet.virtual_networks.delete(resGroup, resName)
 424
 425     def delete_vminstance(self, vm_id):
 426         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 427         resName = self._get_resource_name_from_resource_id(net_id)
 428
 429         self._reload_connection()
 430         self.conn_compute.virtual_machines.delete(resGroup, resName)
 431
 432     def get_vminstance(self, vm_id):
 433         resGroup = self._get_resource_group_name_from_resource_id(net_id)
 434         resName = self._get_resource_name_from_resource_id(net_id)
 435
 436         self._reload_connection()
 437         vm=self.conn_compute.virtual_machines.get(resGroup, resName)
 438
 439         return vm
 440
 441     def get_flavor(self, flavor_id):
 442         self._reload_connection()
 443         for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region):
 444             if vm_size.name == flavor_id :
 445                 return vm_size
 446
 447     def refresh_nets_status(self, net_list):
 448         out_nets = {}
 449         self._reload_connection()
 450         for net_id in net_list:
 451             try:
 452                 resGroup = self._get_resource_group_name_from_resource_id(net_id)
 453                 resName = self._get_resource_name_from_resource_id(net_id)
 454
 455                 vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
 456                 out_nets[net_id] ={
 457                     "status": self.provision_state2osm[vnet.provisioning_state],
 458                     "vim_info": str(vnet)
 459                 }
 460             except CloudError as e:
 461                 if e.error.error == "ResourceNotFound":
 462                     out_nets[net_id] = {
 463                         "status": "DELETED",
 464                     }
 465                 else:
 466                     raise
 467             except Exception as e:
 468                 # TODO distinguish when it is deleted
 469                 out_nets[net_id] = {
 470                     "status": "VIM_ERROR",
 471                     "vim_info": str(vnet),
 472                     "error_msg": str(e)
 473                 }
 474
 475         return out_nets
 476
 477     def refresh_vms_status(self, vm_list):
 478         out_vms = {}
 479         self._reload_connection()
 480         for vm_id in vm_list:
 481             try:
 482                 resGroup = self._get_resource_group_name_from_resource_id(vm_id)
 483                 resName = self._get_resource_name_from_resource_id(vm_id)
 484
 485                 vm = self.conn_compute.virtual_machines.get(resGroup, resName)
 486                 out_vms[vm_id] ={
 487                     "status": self.provision_state2osm[vm.provisioning_state],
 488                     "vim_info": str(vm)
 489                 }
 490             except CloudError as e:
 491                 if e.error.error == "ResourceNotFound":
 492                     out_vms[vm_id] = {
 493                         "status": "DELETED",
 494                     }
 495                 else:
 496                     raise
 497             except Exception as e:
 498                 # TODO distinguish when it is deleted
 499                 out_vms[vm_id] = {
 500                     "status": "VIM_ERROR",
 501                     "vim_info": str(vm),
 502                     "error_msg": str(e)
 503                 }
 504
 505         return out_vms
 506
 507 # TODO get_vminstance_console  for getting console
 508
 509 if __name__ == "__main__":
 510
 511     # Making some basic test
 512     vim_id='azure'
 513     vim_name='azure'
 514     needed_test_params = {
 515         "client_id": "AZURE_CLIENT_ID",
 516         "secret": "AZURE_SECRET",
 517         "tenant": "AZURE_TENANT",
 518         "resource_group": "AZURE_RESOURCE_GROUP",
 519         "subscription_id": "AZURE_SUBSCRIPTION_ID",
 520         "vnet_name": "AZURE_VNET_NAME",
 521     }
 522     test_params = {}
 523
 524     for param, env_var in needed_test_params.items():
 525         value = getenv(env_var)
 526         if not value:
 527             raise Exception("Provide a valid value for env '{}'".format(env_var))
 528         test_params[param] = value
 529
 530     config = {
 531             'region_name': getenv("AZURE_REGION_NAME", 'westeurope'),
 532             'resource_group': getenv("AZURE_RESOURCE_GROUP"),
 533             'subscription_id': getenv("AZURE_SUBSCRIPTION_ID"),
 534             'pub_key': getenv("AZURE_PUB_KEY", None),
 535             'vnet_name': getenv("AZURE_VNET_NAME", 'myNetwork'),
 536     }
 537
 538     virtualMachine = {
 539         'name': 'sergio',
 540         'description': 'new VM',
 541         'status': 'running',
 542         'image': {
 543             'publisher': 'Canonical',
 544             'offer': 'UbuntuServer',
 545             'sku': '16.04.0-LTS',
 546             'version': 'latest'
 547         },
 548         'hardware_profile': {
 549             'vm_size': 'Standard_DS1_v2'
 550         },
 551         'networks': [
 552             'sergio'
 553         ]
 554     }
 555
 556     vnet_config = {
 557         'subnet_address': '10.1.2.0/24',
 558         #'subnet_name': 'subnet-oam'
 559     }
 560     ###########################
 561
 562     azure = vimconnector(vim_id, vim_name, tenant_id=test_params["tenant"], tenant_name=None, url=None, url_admin=None,
 563                          user=test_params["client_id"], passwd=test_params["secret"], log_level=None, config=config)
 564
 565     # azure.get_flavor_id_from_data("here")
 566     # subnets=azure.get_network_list()
 567     # azure.new_vminstance(virtualMachine['name'], virtualMachine['description'], virtualMachine['status'],
 568     #                      virtualMachine['image'], virtualMachine['hardware_profile']['vm_size'], subnets)
 569
 570     net_id = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/Microsoft."\
 571              "Network/virtualNetworks/test"
 572     net_id_not_found = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/"\
 573                        "Microsoft.Network/virtualNetworks/testALF"
 574     azure.refresh_nets_status([net_id, net_id_not_found])