osm_ro/vimconn_azure.py

# -*- coding: utf-8 -*-

__author__='Sergio Gonzalez'
__date__ ='$18-apr-2019 23:59:59$'

import vimconn
import logging
import netaddr

from os import getenv
from uuid import uuid4

from azure.common.credentials import ServicePrincipalCredentials
from azure.mgmt.resource import ResourceManagementClient
from azure.mgmt.network import NetworkManagementClient
from azure.mgmt.compute import ComputeManagementClient

from msrestazure.azure_exceptions import CloudError

class vimconnector(vimconn.vimconnector):

    provision_state2osm = {
        "Deleting": "INACTIVE",
        "Failed": "ERROR",
        "Succeeded": "ACTIVE",
        "Updating": "BUILD",
    }

    def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None,
                 config={}, persistent_info={}):

        vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
                                      config, persistent_info)

        self.vnet_address_space = None
        # LOGGER
        self.logger = logging.getLogger('openmano.vim.azure')
        if log_level:
            logging.basicConfig()
            self.logger.setLevel(getattr(logging, log_level))

        # CREDENTIALS 
        self.credentials = ServicePrincipalCredentials(
            client_id=user,
            secret=passwd,
            tenant=(tenant_id or tenant_name)
        )

        # SUBSCRIPTION
        if 'subscription_id' in config:
            self.subscription_id = config.get('subscription_id')
            self.logger.debug('Setting subscription '+str(self.subscription_id))
        else:
            raise vimconn.vimconnException('Subscription not specified')
        # REGION
        if 'region_name' in config:
            self.region = config.get('region_name')
        else:
            raise vimconn.vimconnException('Azure region_name is not specified at config')
        # RESOURCE_GROUP
        if 'resource_group' in config:
            self.resource_group = config.get('resource_group')
        else:
            raise vimconn.vimconnException('Azure resource_group is not specified at config')
        # VNET_NAME
        if 'vnet_name' in config:
            self.vnet_name = config["vnet_name"]
            
        # public ssh key
        self.pub_key = config.get('pub_key')
            
    def _reload_connection(self):
        """
        Sets connections to work with Azure service APIs
        :return:
        """
        self.logger.debug('Reloading API Connection')
        try:
            self.conn = ResourceManagementClient(self.credentials, self.subscription_id)
            self.conn_compute = ComputeManagementClient(self.credentials, self.subscription_id)
            self.conn_vnet = NetworkManagementClient(self.credentials, self.subscription_id)
            self._check_or_create_resource_group()
            self._check_or_create_vnet()
        except Exception as e:
            self.format_vimconn_exception(e)            

    def _get_resource_name_from_resource_id(self, resource_id):
        return str(resource_id.split('/')[-1])

    def _get_location_from_resource_group(self, resource_group_name):
        return self.conn.resource_groups.get(resource_group_name).location
        
    def _get_resource_group_name_from_resource_id(self, resource_id):
        return str(resource_id.split('/')[4])

    def _check_subnets_for_vm(self, net_list):
        # All subnets must belong to the same resource group and vnet
        if len(set(self._get_resource_group_name_from_resource_id(net['id']) +
                   self._get_resource_name_from_resource_id(net['id']) for net in net_list)) != 1:
            raise self.format_vimconn_exception('Azure VMs can only attach to subnets in same VNET')

    def format_vimconn_exception(self, e):
        """
        Params: an Exception object
        :param e:
        :return: Raises the proper vimconnException
        """
        self.conn = None
        self.conn_vnet = None
        raise vimconn.vimconnConnectionException(type(e).__name__ + ': ' + str(e))        

    def _check_or_create_resource_group(self):
        """
        Creates a resource group in indicated region
        :return: None
        """
        self.logger.debug('Creating RG {} in location {}'.format(self.resource_group, self.region))
        self.conn.resource_groups.create_or_update(self.resource_group, {'location': self.region})

    def _check_or_create_vnet(self):
        try:
            vnet = self.conn_vnet.virtual_networks.get(self.resource_group, self.vnet_name)
            self.vnet_address_space = vnet.address_space.address_prefixes[0]
            self.vnet_id = vnet.id
            return
        except CloudError as e:
            if e.error.error == "ResourceNotFound":
                pass
            else:
                raise
        # if not exist, creates it
        try:
            vnet_params = {
                'location': self.region,
                'address_space': {
                    'address_prefixes': ["10.0.0.0/8"]
                },
            }
            self.vnet_address_space = "10.0.0.0/8"
            self.conn_vnet.virtual_networks.create_or_update(self.resource_group, self.vnet_name, vnet_params)
            vnet = self.conn_vnet.virtual_networks.get(self.resource_group, self.vnet_name)
            self.vnet_id = vnet.id
        except Exception as e:
            self.format_vimconn_exception(e)

    def new_network(self, net_name, net_type, ip_profile=None, shared=False, vlan=None):
        """
        Adds a tenant network to VIM
        :param net_name: name of the network
        :param net_type:
        :param ip_profile: is a dict containing the IP parameters of the network (Currently only IPv4 is implemented)
                'ip-version': can be one of ['IPv4','IPv6']
                'subnet-address': ip_prefix_schema, that is X.X.X.X/Y
                'gateway-address': (Optional) ip_schema, that is X.X.X.X
                'dns-address': (Optional) ip_schema,
                'dhcp': (Optional) dict containing
                    'enabled': {'type': 'boolean'},
                    'start-address': ip_schema, first IP to grant
                    'count': number of IPs to grant.
        :param shared:
        :param vlan:
        :return: a tuple with the network identifier and created_items, or raises an exception on error
            created_items can be None or a dictionary where this method can include key-values that will be passed to
            the method delete_network. Can be used to store created segments, created l2gw connections, etc.
            Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
            as not present.
        """

        return self._new_subnet(net_name, ip_profile)

    def _new_subnet(self, net_name, ip_profile):
        """
        Adds a tenant network to VIM. It creates a new VNET with a single subnet
        :param net_name:
        :param ip_profile:
        :return:
        """
        self.logger.debug('Adding a subnet to VNET '+self.vnet_name)
        self._reload_connection()

        if ip_profile is None:
            # get a non used vnet ip range /24 and allocate automatically inside the range self.vnet_address_space
            used_subnets = self.get_network_list()
            for ip_range in netaddr.IPNetwork(self.vnet_address_space).subnet(24):
                for used_subnet in used_subnets:
                    subnet_range = netaddr.IPNetwork(used_subnet["cidr_block"])
                    if subnet_range in ip_range or ip_range in subnet_range:
                        # this range overlaps with an existing subnet ip range. Breaks and look for another
                        break
                else:
                    ip_profile = {"subnet_address": str(ip_range)}
                    break
            else:
                vimconn.vimconnException("Cannot find a non-used subnet range in {}".format(self.vnet_address_space))
        try:
            subnet_name = "{}-{}".format(net_name[:24], uuid4())
            subnet_params= {
                'address_prefix': ip_profile['subnet_address']
            }
            self.conn_vnet.subnets.create_or_update(self.resource_group, self.vnet_name, subnet_name, subnet_params)
            return "{}/subnet/{}".format(self.vnet_id, subnet_name), None
        except Exception as e:
            self.format_vimconn_exception(e)

    def _create_nic(self, subnet_id, nic_name, static_ip=None):
        self._reload_connection()
        
        resource_group_name=self._get_resource_group_name_from_resource_id(subnet_id)
        location = self._get_location_from_resource_group(resource_group_name)
            
        if static_ip:
            async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
                resource_group_name,
                nic_name,
                {
                    'location': location,
                    'ip_configurations': [{
                        'name': nic_name + 'ipconfiguration',
                        'privateIPAddress': static_ip,
                        'privateIPAllocationMethod': 'Static',
                        'subnet': {
                            'id': subnet_id
                        }
                    }]
                }
            )
        else:
            async_nic_creation = self.conn_vnet.network_interfaces.create_or_update(
                resource_group_name,
                nic_name,
                {
                    'location': location,
                    'ip_configurations': [{
                        'name': nic_name + 'ipconfiguration',
                        'subnet': {
                            'id': subnet_id
                        }
                    }]
                }
            )

        return async_nic_creation.result()

    def get_image_list(self, filter_dict={}):
        """
        The urn contains for marketplace  'publisher:offer:sku:version'

        :param filter_dict:
        :return:
        """
        image_list = []

        self._reload_connection()
        if filter_dict.get("name"):
            params = filter_dict["name"].split(":")
            if len(params) >= 3:
                publisher = params[0]
                offer = params[1]
                sku = params[2]
                version = None
                if len(params) == 4:
                    version = params[3]
                images = self.conn_compute.virtual_machine_images.list(self.region, publisher, offer, sku)
                for image in images:
                    if version:
                        image_version = str(image.id).split("/")[-1]
                        if image_version != version:
                            continue
                    image_list.append({
                        'id': str(image.id),
                        'name': self._get_resource_name_from_resource_id(image.id)
                    })
                return image_list

        images = self.conn_compute.virtual_machine_images.list()

        for image in images:
            # TODO implement filter_dict
            if filter_dict:
                if filter_dict.get("id") and str(image.id) != filter_dict["id"]:
                    continue
                if filter_dict.get("name") and \
                        self._get_resource_name_from_resource_id(image.id) != filter_dict["name"]:
                    continue
                # TODO add checksum
            image_list.append({
                'id': str(image.id),
                'name': self._get_resource_name_from_resource_id(image.id),
            })
        return image_list

    def get_network_list(self, filter_dict={}):
        """Obtain tenant networks of VIM
        Filter_dict can be:
            name: network name
            id: network uuid
            shared: boolean
            tenant_id: tenant
            admin_state_up: boolean
            status: 'ACTIVE'
        Returns the network list of dictionaries
        """
        self.logger.debug('Getting all subnets from VIM')
        try:
            self._reload_connection()
            vnet = self.conn_vnet.virtual_networks.get(self.config["resource_group"], self.vnet_name)
            subnet_list = []
            
            for subnet in vnet.subnets:
                # TODO implement filter_dict
                if filter_dict:
                    if filter_dict.get("id") and str(subnet.id) != filter_dict["id"]:
                        continue
                    if filter_dict.get("name") and \
                            self._get_resource_name_from_resource_id(subnet.id) != filter_dict["name"]:
                        continue

                subnet_list.append({
                    'id': str(subnet.id),
                     'name': self._get_resource_name_from_resource_id(subnet.id),
                     'status': str(vnet.provisioning_state),  # TODO Does subnet contains status???
                     'cidr_block': str(subnet.address_prefix)
                    }
                )
            return subnet_list
        except Exception as e:
            self.format_vimconn_exception(e)

    def new_vminstance(self, vm_name, description, start, image_id, flavor_id, net_list, cloud_config=None,
                       disk_list=None, availability_zone_index=None, availability_zone_list=None):

        return self._new_vminstance(vm_name, image_id, flavor_id, net_list)
        
    def _new_vminstance(self, vm_name, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
                        availability_zone_index=None, availability_zone_list=None):
        #Create NICs
        self._check_subnets_for_vm(net_list)
        vm_nics = []
        for idx, net in enumerate(net_list):
            subnet_id=net['subnet_id']
            nic_name = vm_name + '-nic-'+str(idx)
            vm_nic = self._create_nic(subnet_id, nic_name)
            vm_nics.append({ 'id': str(vm_nic.id)})

        try:
            vm_parameters = {
                'location': self.region,
                'os_profile': {
                    'computer_name': vm_name,  # TODO if vm_name cannot be repeated add uuid4() suffix
                    'admin_username': 'sergio',  # TODO is it mandatory???
                    'linuxConfiguration': {
                        'disablePasswordAuthentication': 'true',
                        'ssh': {
                          'publicKeys': [
                            {
                              'path': '/home/sergio/.ssh/authorized_keys',
                              'keyData': self.pub_key
                            }
                          ]
                        }
                    }                    
                    
                },
                'hardware_profile': {
                    'vm_size':flavor_id
                },
                'storage_profile': {
                    'image_reference': image_id
                },
                'network_profile': {
                    'network_interfaces': [
                        vm_nics[0]
                    ]
                }
            }
            creation_result = self.conn_compute.virtual_machines.create_or_update(
                self.resource_group, 
                vm_name, 
                vm_parameters
            )
            
            run_command_parameters = {
                'command_id': 'RunShellScript', # For linux, don't change it
                'script': [
                'date > /home/sergio/test.txt'
                ]
            }
            poller = self.conn_compute.virtual_machines.run_command(
                self.resource_group, 
                vm_name, 
                run_command_parameters
            )
            # TODO return a tuple (vm-ID, None)
        except Exception as e:
            self.format_vimconn_exception(e)

    def get_flavor_id_from_data(self, flavor_dict):
        self.logger.debug("Getting flavor id from data")
        self._reload_connection()
        vm_sizes_list = [vm_size.serialize() for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region)]

        cpus = flavor_dict['vcpus']
        memMB = flavor_dict['ram']

        filteredSizes = [size for size in vm_sizes_list if size['numberOfCores'] > cpus and size['memoryInMB'] > memMB]
        listedFilteredSizes = sorted(filteredSizes, key=lambda k: k['numberOfCores'])

        return listedFilteredSizes[0]['name']

    def check_vim_connectivity(self):
        try:
            self._reload_connection()
            return True
        except Exception as e:
            raise vimconn.vimconnException("Connectivity issue with Azure API: {}".format(e))

    def get_network(self, net_id):
        resGroup = self._get_resource_group_name_from_resource_id(net_id)
        resName = self._get_resource_name_from_resource_id(net_id)
        
        self._reload_connection()
        vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
        return vnet

    def delete_network(self, net_id):
        resGroup = self._get_resource_group_name_from_resource_id(net_id)
        resName = self._get_resource_name_from_resource_id(net_id)
        
        self._reload_connection()
        self.conn_vnet.virtual_networks.delete(resGroup, resName)

    def delete_vminstance(self, vm_id):
        resGroup = self._get_resource_group_name_from_resource_id(net_id)
        resName = self._get_resource_name_from_resource_id(net_id)
        
        self._reload_connection()
        self.conn_compute.virtual_machines.delete(resGroup, resName)

    def get_vminstance(self, vm_id):
        resGroup = self._get_resource_group_name_from_resource_id(net_id)
        resName = self._get_resource_name_from_resource_id(net_id)
        
        self._reload_connection()
        vm=self.conn_compute.virtual_machines.get(resGroup, resName)

        return vm

    def get_flavor(self, flavor_id):
        self._reload_connection()
        for vm_size in self.conn_compute.virtual_machine_sizes.list(self.region):
            if vm_size.name == flavor_id :
                return vm_size

    def refresh_nets_status(self, net_list):
        out_nets = {}
        self._reload_connection()
        for net_id in net_list:
            try:
                resGroup = self._get_resource_group_name_from_resource_id(net_id)
                resName = self._get_resource_name_from_resource_id(net_id)

                vnet = self.conn_vnet.virtual_networks.get(resGroup, resName)
                out_nets[net_id] ={
                    "status": self.provision_state2osm[vnet.provisioning_state],
                    "vim_info": str(vnet)
                }
            except CloudError as e:
                if e.error.error == "ResourceNotFound":
                    out_nets[net_id] = {
                        "status": "DELETED",
                    }
                else:
                    raise
            except Exception as e:
                # TODO distinguish when it is deleted
                out_nets[net_id] = {
                    "status": "VIM_ERROR",
                    "vim_info": str(vnet),
                    "error_msg": str(e)
                }

        return out_nets

    def refresh_vms_status(self, vm_list):
        out_vms = {}
        self._reload_connection()
        for vm_id in vm_list:
            try:
                resGroup = self._get_resource_group_name_from_resource_id(vm_id)
                resName = self._get_resource_name_from_resource_id(vm_id)

                vm = self.conn_compute.virtual_machines.get(resGroup, resName)
                out_vms[vm_id] ={
                    "status": self.provision_state2osm[vm.provisioning_state],
                    "vim_info": str(vm)
                }
            except CloudError as e:
                if e.error.error == "ResourceNotFound":
                    out_vms[vm_id] = {
                        "status": "DELETED",
                    }
                else:
                    raise
            except Exception as e:
                # TODO distinguish when it is deleted
                out_vms[vm_id] = {
                    "status": "VIM_ERROR",
                    "vim_info": str(vm),
                    "error_msg": str(e)
                }

        return out_vms

# TODO get_vminstance_console  for getting console

if __name__ == "__main__":

    # Making some basic test
    vim_id='azure'
    vim_name='azure'
    needed_test_params = {
        "client_id": "AZURE_CLIENT_ID",
        "secret": "AZURE_SECRET",
        "tenant": "AZURE_TENANT",
        "resource_group": "AZURE_RESOURCE_GROUP",
        "subscription_id": "AZURE_SUBSCRIPTION_ID",
        "vnet_name": "AZURE_VNET_NAME",
    }
    test_params = {}

    for param, env_var in needed_test_params.items():
        value = getenv(env_var)
        if not value:
            raise Exception("Provide a valid value for env '{}'".format(env_var))
        test_params[param] = value

    config = {
            'region_name': getenv("AZURE_REGION_NAME", 'westeurope'),
            'resource_group': getenv("AZURE_RESOURCE_GROUP"),
            'subscription_id': getenv("AZURE_SUBSCRIPTION_ID"),
            'pub_key': getenv("AZURE_PUB_KEY", None),
            'vnet_name': getenv("AZURE_VNET_NAME", 'myNetwork'),
    }

    virtualMachine = {
        'name': 'sergio',
        'description': 'new VM',
        'status': 'running',
        'image': {
            'publisher': 'Canonical',
            'offer': 'UbuntuServer',
            'sku': '16.04.0-LTS',
            'version': 'latest'
        },
        'hardware_profile': {
            'vm_size': 'Standard_DS1_v2'
        },
        'networks': [
            'sergio'
        ]
    }

    vnet_config = {
        'subnet_address': '10.1.2.0/24',
        #'subnet_name': 'subnet-oam'
    }
    ###########################

    azure = vimconnector(vim_id, vim_name, tenant_id=test_params["tenant"], tenant_name=None, url=None, url_admin=None,
                         user=test_params["client_id"], passwd=test_params["secret"], log_level=None, config=config)

    # azure.get_flavor_id_from_data("here")
    # subnets=azure.get_network_list()
    # azure.new_vminstance(virtualMachine['name'], virtualMachine['description'], virtualMachine['status'],
    #                      virtualMachine['image'], virtualMachine['hardware_profile']['vm_size'], subnets)

    azure.new_network("mynet", None)
    net_id = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/Microsoft."\
             "Network/virtualNetworks/test"
    net_id_not_found = "/subscriptions/82f80cc1-876b-4591-9911-1fb5788384fd/resourceGroups/osmRG/providers/"\
                       "Microsoft.Network/virtualNetworks/testALF"
    azure.refresh_nets_status([net_id, net_id_not_found])