X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=RO-VIM-aws%2Fosm_rovim_aws%2Fvimconn_aws.py;h=5fc070410839253351399b1e0b497259800e0bb1;hb=7b521f73dd996a279f23b2f512cd89a42c1c79f6;hp=fa2456844bf6b88a95a9e64909de9be1a40e6080;hpb=7277486065c905f91477bb064da86855a8fa269a;p=osm%2FRO.git diff --git a/RO-VIM-aws/osm_rovim_aws/vimconn_aws.py b/RO-VIM-aws/osm_rovim_aws/vimconn_aws.py index fa245684..5fc07041 100644 --- a/RO-VIM-aws/osm_rovim_aws/vimconn_aws.py +++ b/RO-VIM-aws/osm_rovim_aws/vimconn_aws.py @@ -21,59 +21,89 @@ # contact with: saboor.ahmad@xflowresearch.com ## -''' +""" AWS-connector implements all the methods to interact with AWS using the BOTO client -''' - -__author__ = "Saboor Ahmad" -__date__ = "10-Apr-2017" - -from osm_ro_plugin import vimconn -import yaml +""" import logging -import netaddr +import random import time +import traceback import boto import boto.ec2 +from boto.exception import BotoServerError, EC2ResponseError import boto.vpc +from ipconflict import check_conflicts +import netaddr +from osm_ro_plugin import vimconn +import yaml + +__author__ = "Saboor Ahmad" +__date__ = "10-Apr-2017" class vimconnector(vimconn.VimConnector): - def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, - config={}, persistent_info={}): - """ Params: uuid - id asigned to this VIM - name - name assigned to this VIM, can be used for logging - tenant_id - ID to be used for tenant - tenant_name - name of tenant to be used VIM tenant to be used - url_admin - optional, url used for administrative tasks - user - credentials of the VIM user - passwd - credentials of the VIM user - log_level - if must use a different log_level than the general one - config - dictionary with misc VIM information - region_name - name of region to deploy the instances - vpc_cidr_block - default CIDR block for VPC - security_groups - default security group to specify this instance - persistent_info - dict where the class can store information that will be available among class - destroy/creation cycles. This info is unique per VIM/credential. At first call it will contain an - empty dict. Useful to store login/tokens information for speed up communication + def __init__( + self, + uuid, + name, + tenant_id, + tenant_name, + url, + url_admin=None, + user=None, + passwd=None, + log_level=None, + config={}, + persistent_info={}, + ): + """Params: + uuid - id asigned to this VIM + name - name assigned to this VIM, can be used for logging + tenant_id - ID to be used for tenant + tenant_name - name of tenant to be used VIM tenant to be used + url_admin - optional, url used for administrative tasks + user - credentials of the VIM user + passwd - credentials of the VIM user + log_level - if must use a different log_level than the general one + config - dictionary with misc VIM information + region_name - name of region to deploy the instances + vpc_cidr_block - default CIDR block for VPC + security_groups - default security group to specify this instance + persistent_info - dict where the class can store information that will be available among class + destroy/creation cycles. This info is unique per VIM/credential. At first call it will contain an + empty dict. Useful to store login/tokens information for speed up communication """ - - vimconn.VimConnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, - config, persistent_info) + vimconn.VimConnector.__init__( + self, + uuid, + name, + tenant_id, + tenant_name, + url, + url_admin, + user, + passwd, + log_level, + config, + persistent_info, + ) self.persistent_info = persistent_info self.a_creds = {} + if user: - self.a_creds['aws_access_key_id'] = user + self.a_creds["aws_access_key_id"] = user else: raise vimconn.VimConnAuthException("Username is not specified") + if passwd: - self.a_creds['aws_secret_access_key'] = passwd + self.a_creds["aws_secret_access_key"] = passwd else: raise vimconn.VimConnAuthException("Password is not specified") - if 'region_name' in config: - self.region = config.get('region_name') + + if "region_name" in config: + self.region = config.get("region_name") else: raise vimconn.VimConnException("AWS region_name is not specified at config") @@ -82,70 +112,85 @@ class vimconnector(vimconn.VimConnector): self.conn = None self.conn_vpc = None self.account_id = None + self.network_delete_on_termination = [] + self.server_timeout = 180 - self.vpc_id = self.get_tenant_list()[0]['id'] + self.vpc_id = self.get_tenant_list()[0]["id"] # we take VPC CIDR block if specified, otherwise we use the default CIDR # block suggested by AWS while creating instance - self.vpc_cidr_block = '10.0.0.0/24' + self.vpc_cidr_block = "10.0.0.0/24" + + if tenant_name: + self.vpc_id = tenant_name - if tenant_id: - self.vpc_id = tenant_id - if 'vpc_cidr_block' in config: - self.vpc_cidr_block = config['vpc_cidr_block'] + if "vpc_cidr_block" in config: + self.vpc_cidr_block = config["vpc_cidr_block"] self.security_groups = None - if 'security_groups' in config: - self.security_groups = config['security_groups'] + if "security_groups" in config: + self.security_groups = config["security_groups"] self.key_pair = None - if 'key_pair' in config: - self.key_pair = config['key_pair'] + if "key_pair" in config: + self.key_pair = config["key_pair"] self.flavor_info = None - if 'flavor_info' in config: - flavor_data = config.get('flavor_info') + if "flavor_info" in config: + flavor_data = config.get("flavor_info") if isinstance(flavor_data, str): try: if flavor_data[0] == "@": # read from a file - with open(flavor_data[1:], 'r') as stream: - self.flavor_info = yaml.load(stream, Loader=yaml.Loader) + with open(flavor_data[1:], "r") as stream: + self.flavor_info = yaml.safe_load(stream) else: - self.flavor_info = yaml.load(flavor_data, Loader=yaml.Loader) + self.flavor_info = yaml.safe_load(flavor_data) except yaml.YAMLError as e: self.flavor_info = None - raise vimconn.VimConnException("Bad format at file '{}': {}".format(flavor_data[1:], e)) + + raise vimconn.VimConnException( + "Bad format at file '{}': {}".format(flavor_data[1:], e) + ) except IOError as e: - raise vimconn.VimConnException("Error reading file '{}': {}".format(flavor_data[1:], e)) + raise vimconn.VimConnException( + "Error reading file '{}': {}".format(flavor_data[1:], e) + ) elif isinstance(flavor_data, dict): self.flavor_info = flavor_data - self.logger = logging.getLogger('openmano.vim.aws') + self.logger = logging.getLogger("ro.vim.aws") + if log_level: self.logger.setLevel(getattr(logging, log_level)) def __setitem__(self, index, value): - """Params: index - name of value of set - value - value to set + """Params: + index - name of value of set + value - value to set """ - if index == 'user': - self.a_creds['aws_access_key_id'] = value - elif index == 'passwd': - self.a_creds['aws_secret_access_key'] = value - elif index == 'region': + if index == "user": + self.a_creds["aws_access_key_id"] = value + elif index == "passwd": + self.a_creds["aws_secret_access_key"] = value + elif index == "region": self.region = value else: vimconn.VimConnector.__setitem__(self, index, value) def _reload_connection(self): - """Returns: sets boto.EC2 and boto.VPC connection to work with AWS services - """ - + """Returns: sets boto.EC2 and boto.VPC connection to work with AWS services""" try: - self.conn = boto.ec2.connect_to_region(self.region, aws_access_key_id=self.a_creds['aws_access_key_id'], - aws_secret_access_key=self.a_creds['aws_secret_access_key']) - self.conn_vpc = boto.vpc.connect_to_region(self.region, aws_access_key_id=self.a_creds['aws_access_key_id'], - aws_secret_access_key=self.a_creds['aws_secret_access_key']) - # client = boto3.client("sts", aws_access_key_id=self.a_creds['aws_access_key_id'], aws_secret_access_key=self.a_creds['aws_secret_access_key']) + self.conn = boto.ec2.connect_to_region( + self.region, + aws_access_key_id=self.a_creds["aws_access_key_id"], + aws_secret_access_key=self.a_creds["aws_secret_access_key"], + ) + self.conn_vpc = boto.vpc.connect_to_region( + self.region, + aws_access_key_id=self.a_creds["aws_access_key_id"], + aws_secret_access_key=self.a_creds["aws_secret_access_key"], + ) + # client = boto3.client("sts", aws_access_key_id=self.a_creds['aws_access_key_id'], + # aws_secret_access_key=self.a_creds['aws_secret_access_key']) # self.account_id = client.get_caller_identity()["Account"] except Exception as e: self.format_vimconn_exception(e) @@ -154,20 +199,20 @@ class vimconnector(vimconn.VimConnector): """Params: an Exception object Returns: Raises the exception 'e' passed in mehtod parameters """ - self.conn = None self.conn_vpc = None + raise vimconn.VimConnConnectionException(type(e).__name__ + ": " + str(e)) def get_availability_zones_list(self): - """Obtain AvailabilityZones from AWS - """ - + """Obtain AvailabilityZones from AWS""" try: self._reload_connection() az_list = [] + for az in self.conn.get_all_zones(): az_list.append(az.name) + return az_list except Exception as e: self.format_vimconn_exception(e) @@ -181,20 +226,27 @@ class vimconnector(vimconn.VimConnector): Returns the tenant list of dictionaries, and empty list if no tenant match all the filers: [{'name':', 'id':', ...}, ...] """ - try: self._reload_connection() vpc_ids = [] - tfilters = {} + if filter_dict != {}: - if 'id' in filter_dict: - vpc_ids.append(filter_dict['id']) - tfilters['name'] = filter_dict['id'] - tenants = self.conn_vpc.get_all_vpcs(vpc_ids, tfilters) + if "id" in filter_dict: + vpc_ids.append(filter_dict["id"]) + + tenants = self.conn_vpc.get_all_vpcs(vpc_ids, None) tenant_list = [] + for tenant in tenants: - tenant_list.append({'id': str(tenant.id), 'name': str(tenant.id), 'status': str(tenant.state), - 'cidr_block': str(tenant.cidr_block)}) + tenant_list.append( + { + "id": str(tenant.id), + "name": str(tenant.id), + "status": str(tenant.state), + "cidr_block": str(tenant.cidr_block), + } + ) + return tenant_list except Exception as e: self.format_vimconn_exception(e) @@ -205,8 +257,8 @@ class vimconnector(vimconn.VimConnector): "tenant_description": string max length 256 returns the tenant identifier or raise exception """ - self.logger.debug("Adding a new VPC") + try: self._reload_connection() vpc = self.conn_vpc.create_vpc(self.vpc_cidr_block) @@ -216,11 +268,14 @@ class vimconnector(vimconn.VimConnector): gateway = self.conn_vpc.create_internet_gateway() self.conn_vpc.attach_internet_gateway(gateway.id, vpc.id) route_table = self.conn_vpc.create_route_table(vpc.id) - self.conn_vpc.create_route(route_table.id, '0.0.0.0/0', gateway.id) + self.conn_vpc.create_route(route_table.id, "0.0.0.0/0", gateway.id) + + self.vpc_data[vpc.id] = { + "gateway": gateway.id, + "route_table": route_table.id, + "subnets": self.subnet_sizes(self.vpc_cidr_block), + } - self.vpc_data[vpc.id] = {'gateway': gateway.id, 'route_table': route_table.id, - 'subnets': self.subnet_sizes(len(self.get_availability_zones_list()), - self.vpc_cidr_block)} return vpc.id except Exception as e: self.format_vimconn_exception(e) @@ -230,54 +285,54 @@ class vimconnector(vimconn.VimConnector): tenant_id: returned VIM tenant_id on "new_tenant" Returns None on success. Raises and exception of failure. If tenant is not found raises vimconnNotFoundException """ - self.logger.debug("Deleting specified VPC") + try: self._reload_connection() vpc = self.vpc_data.get(tenant_id) - if 'gateway' in vpc and 'route_table' in vpc: - gateway_id, route_table_id = vpc['gateway'], vpc['route_table'] + + if "gateway" in vpc and "route_table" in vpc: + gateway_id, route_table_id = vpc["gateway"], vpc["route_table"] self.conn_vpc.detach_internet_gateway(gateway_id, tenant_id) self.conn_vpc.delete_vpc(tenant_id) - self.conn_vpc.delete_route(route_table_id, '0.0.0.0/0') + self.conn_vpc.delete_route(route_table_id, "0.0.0.0/0") else: self.conn_vpc.delete_vpc(tenant_id) except Exception as e: self.format_vimconn_exception(e) - def subnet_sizes(self, availability_zones, cidr): - """Calcualtes possible subnets given CIDR value of VPC - """ + def subnet_sizes(self, cidr): + """Calculates possible subnets given CIDR value of VPC""" + netmasks = ( + "255.255.0.0", + "255.255.128.0", + "255.255.192.0", + "255.255.224.0", + "255.255.240.0", + "255.255.248.0", + ) - if availability_zones != 2 and availability_zones != 3: - self.logger.debug("Number of AZs should be 2 or 3") - raise vimconn.VimConnNotSupportedException("Number of AZs should be 2 or 3") - - netmasks = ('255.255.252.0', '255.255.254.0', '255.255.255.0', '255.255.255.128') ip = netaddr.IPNetwork(cidr) mask = ip.netmask + pub_split = () - if str(mask) not in netmasks: - self.logger.debug("Netmask " + str(mask) + " not found") - raise vimconn.VimConnNotFoundException("Netmask " + str(mask) + " not found") + for netmask in netmasks: + if str(mask) == netmask: + pub_split = list(ip.subnet(24)) + break - if availability_zones == 2: - for n, netmask in enumerate(netmasks): - if str(mask) == netmask: - subnets = list(ip.subnet(n + 24)) - else: - for n, netmask in enumerate(netmasks): - if str(mask) == netmask: - pub_net = list(ip.subnet(n + 24)) - pri_subs = pub_net[1:] - pub_mask = pub_net[0].netmask - pub_split = list(ip.subnet(26)) if (str(pub_mask) == '255.255.255.0') else list(ip.subnet(27)) - pub_subs = pub_split[:3] - subnets = pub_subs + pri_subs + subnets = pub_split if pub_split else (list(ip.subnet(28))) return map(str, subnets) - def new_network(self, net_name, net_type, ip_profile=None, shared=False, provider_network_profile=None): + def new_network( + self, + net_name, + net_type, + ip_profile=None, + shared=False, + provider_network_profile=None, + ): """Adds a tenant network to VIM Params: 'net_name': name of the network @@ -301,31 +356,79 @@ class vimconnector(vimconn.VimConnector): Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same as not present. """ - self.logger.debug("Adding a subnet to VPC") + try: created_items = {} self._reload_connection() subnet = None vpc_id = self.vpc_id + if self.conn_vpc.get_all_subnets(): + existing_subnet = self.conn_vpc.get_all_subnets()[0] + if not self.availability_zone: + self.availability_zone = str(existing_subnet.availability_zone) + if self.vpc_data.get(vpc_id, None): - cidr_block = list(set(self.vpc_data[vpc_id]['subnets']) - set(self.get_network_details({'tenant_id': vpc_id}, detail='cidr_block')))[0] + cidr_block = list( + set(self.vpc_data[vpc_id]["subnets"]) + - set( + self.get_network_details( + {"tenant_id": vpc_id}, detail="cidr_block" + ) + ) + ) else: - vpc = self.get_tenant_list({'id': vpc_id})[0] - subnet_list = self.subnet_sizes(len(self.get_availability_zones_list()), vpc['cidr_block']) - cidr_block = list(set(subnet_list) - set(self.get_network_details({'tenant_id': vpc['id']}, detail='cidr_block')))[0] - subnet = self.conn_vpc.create_subnet(vpc_id, cidr_block) + vpc = self.get_tenant_list({"id": vpc_id})[0] + subnet_list = self.subnet_sizes(vpc["cidr_block"]) + cidr_block = list( + set(subnet_list) + - set( + self.get_network_details( + {"tenant_id": vpc["id"]}, detail="cidr_block" + ) + ) + ) + + try: + selected_cidr_block = random.choice(cidr_block) + retry = 15 + while retry > 0: + all_subnets = [ + subnet.cidr_block for subnet in self.conn_vpc.get_all_subnets() + ] + all_subnets.append(selected_cidr_block) + conflict = check_conflicts(all_subnets) + if not conflict: + subnet = self.conn_vpc.create_subnet( + vpc_id, selected_cidr_block, self.availability_zone + ) + break + retry -= 1 + selected_cidr_block = random.choice(cidr_block) + else: + raise vimconn.VimConnException( + "Failed to find a proper CIDR which does not overlap" + "with existing subnets", + http_code=vimconn.HTTP_Request_Timeout, + ) + + except (EC2ResponseError, BotoServerError) as error: + self.format_vimconn_exception(error) + + created_items["net:" + str(subnet.id)] = True + return subnet.id, created_items except Exception as e: self.format_vimconn_exception(e) def get_network_details(self, filters, detail): - """Get specified details related to a subnet - """ + """Get specified details related to a subnet""" detail_list = [] subnet_list = self.get_network_list(filters) + for net in subnet_list: detail_list.append(net[detail]) + return detail_list def get_network_list(self, filter_dict={}): @@ -336,7 +439,8 @@ class vimconnector(vimconn.VimConnector): id: string => returns networks with this VIM id, this imply returns one network at most shared: boolean >= returns only networks that are (or are not) shared tenant_id: sting => returns only networks that belong to this tenant/project - ,#(not used yet) admin_state_up: boolean => returns only networks that are (or are not) in admin state active + ,#(not used yet) admin_state_up: boolean => returns only networks that are (or are not) in admin + state active #(not used yet) status: 'ACTIVE','ERROR',... => filter networks that are on this status Returns the network list of dictionaries. each dictionary contains: 'id': (mandatory) VIM network id @@ -347,20 +451,36 @@ class vimconnector(vimconn.VimConnector): List can be empty if no network map the filter_dict. Raise an exception only upon VIM connectivity, authorization, or some other unspecific error """ - self.logger.debug("Getting all subnets from VIM") + try: self._reload_connection() tfilters = {} + if filter_dict != {}: - if 'tenant_id' in filter_dict: - tfilters['vpcId'] = filter_dict['tenant_id'] - subnets = self.conn_vpc.get_all_subnets(subnet_ids=filter_dict.get('name', None), filters=tfilters) + if "tenant_id" in filter_dict: + tfilters["vpcId"] = filter_dict.get("tenant_id") + + subnets = self.conn_vpc.get_all_subnets( + subnet_ids=filter_dict.get("SubnetId", None), filters=tfilters + ) + net_list = [] + for net in subnets: - net_list.append( - {'id': str(net.id), 'name': str(net.id), 'status': str(net.state), 'vpc_id': str(net.vpc_id), - 'cidr_block': str(net.cidr_block), 'type': 'bridge'}) + if net.id == filter_dict.get("name"): + self.availability_zone = str(net.availability_zone) + net_list.append( + { + "id": str(net.id), + "name": str(net.id), + "status": str(net.state), + "vpc_id": str(net.vpc_id), + "cidr_block": str(net.cidr_block), + "type": "bridge", + } + ) + return net_list except Exception as e: self.format_vimconn_exception(e) @@ -375,13 +495,19 @@ class vimconnector(vimconn.VimConnector): other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param Raises an exception upon error or when network is not found """ - self.logger.debug("Getting Subnet from VIM") + try: self._reload_connection() subnet = self.conn_vpc.get_all_subnets(net_id)[0] - return {'id': str(subnet.id), 'name': str(subnet.id), 'status': str(subnet.state), - 'vpc_id': str(subnet.vpc_id), 'cidr_block': str(subnet.cidr_block)} + return { + "id": str(subnet.id), + "name": str(subnet.id), + "status": str(subnet.state), + "vpc_id": str(subnet.vpc_id), + "cidr_block": str(subnet.cidr_block), + "availability_zone": str(subnet.availability_zone), + } except Exception as e: self.format_vimconn_exception(e) @@ -392,15 +518,23 @@ class vimconnector(vimconn.VimConnector): :param created_items: dictionary with extra items to be deleted. provided by method new_network Returns the network identifier or raises an exception upon error or when network is not found """ - self.logger.debug("Deleting subnet from VIM") + try: self._reload_connection() self.logger.debug("DELETING NET_ID: " + str(net_id)) self.conn_vpc.delete_subnet(net_id) + return net_id + except Exception as e: - self.format_vimconn_exception(e) + if isinstance(e, EC2ResponseError): + self.network_delete_on_termination.append(net_id) + self.logger.warning( + f"{net_id} could not be deleted, deletion will retry after dependencies resolved" + ) + else: + self.format_vimconn_exception(e) def refresh_nets_status(self, net_list): """Get the status of the networks @@ -419,31 +553,38 @@ class vimconnector(vimconn.VimConnector): vim_info: #Text with plain information obtained from vim (yaml.safe_dump) 'net_id2': ... """ - self._reload_connection() + try: dict_entry = {} + for net_id in net_list: subnet_dict = {} subnet = None + try: subnet = self.conn_vpc.get_all_subnets(net_id)[0] + if subnet.state == "pending": - subnet_dict['status'] = "BUILD" + subnet_dict["status"] = "BUILD" elif subnet.state == "available": - subnet_dict['status'] = 'ACTIVE' + subnet_dict["status"] = "ACTIVE" else: - subnet_dict['status'] = 'ERROR' - subnet_dict['error_msg'] = '' - except Exception as e: - subnet_dict['status'] = 'DELETED' - subnet_dict['error_msg'] = 'Network not found' + subnet_dict["status"] = "ERROR" + subnet_dict["error_msg"] = "" + except Exception: + subnet_dict["status"] = "DELETED" + subnet_dict["error_msg"] = "Network not found" finally: - try: - subnet_dict['vim_info'] = yaml.safe_dump(subnet, default_flow_style=True, width=256) - except yaml.YAMLError as e: - subnet_dict['vim_info'] = str(subnet) + subnet_dictionary = vars(subnet) + cleared_subnet_dict = { + key: subnet_dictionary[key] + for key in subnet_dictionary + if not isinstance(subnet_dictionary[key], object) + } + subnet_dict["vim_info"] = cleared_subnet_dict dict_entry[net_id] = subnet_dict + return dict_entry except Exception as e: self.format_vimconn_exception(e) @@ -453,13 +594,15 @@ class vimconnector(vimconn.VimConnector): Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } Raises an exception upon error or if not found """ - self.logger.debug("Getting instance type") + try: if flavor_id in self.flavor_info: return self.flavor_info[flavor_id] else: - raise vimconn.VimConnNotFoundException("Cannot find flavor with this flavor ID/Name") + raise vimconn.VimConnNotFoundException( + "Cannot find flavor with this flavor ID/Name" + ) except Exception as e: self.format_vimconn_exception(e) @@ -473,61 +616,83 @@ class vimconnector(vimconn.VimConnector): #todo: complete parameters for EPA Returns the flavor_id or raises a vimconnNotFoundException """ - self.logger.debug("Getting flavor id from data") + try: flavor = None for key, values in self.flavor_info.items(): if (values["ram"], values["cpus"], values["disk"]) == ( - flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"]): + flavor_dict["ram"], + flavor_dict["vcpus"], + flavor_dict["disk"], + ): flavor = (key, values) break elif (values["ram"], values["cpus"], values["disk"]) >= ( - flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"]): + flavor_dict["ram"], + flavor_dict["vcpus"], + flavor_dict["disk"], + ): if not flavor: flavor = (key, values) else: if (flavor[1]["ram"], flavor[1]["cpus"], flavor[1]["disk"]) >= ( - values["ram"], values["cpus"], values["disk"]): + values["ram"], + values["cpus"], + values["disk"], + ): flavor = (key, values) + if flavor: return flavor[0] - raise vimconn.VimConnNotFoundException("Cannot find flavor with this flavor ID/Name") + + raise vimconn.VimConnNotFoundException( + "Cannot find flavor with this flavor ID/Name" + ) except Exception as e: self.format_vimconn_exception(e) def new_image(self, image_dict): - """ Adds a tenant image to VIM + """Adds a tenant image to VIM Params: image_dict - name (string) - The name of the AMI. Valid only for EBS-based images. - description (string) - The description of the AMI. - image_location (string) - Full path to your AMI manifest in Amazon S3 storage. Only used for S3-based AMI’s. - architecture (string) - The architecture of the AMI. Valid choices are: * i386 * x86_64 - kernel_id (string) - The ID of the kernel with which to launch the instances - root_device_name (string) - The root device name (e.g. /dev/sdh) - block_device_map (boto.ec2.blockdevicemapping.BlockDeviceMapping) - A BlockDeviceMapping data structure describing the EBS volumes associated with the Image. - virtualization_type (string) - The virutalization_type of the image. Valid choices are: * paravirtual * hvm - sriov_net_support (string) - Advanced networking support. Valid choices are: * simple - snapshot_id (string) - A snapshot ID for the snapshot to be used as root device for the image. Mutually exclusive with block_device_map, requires root_device_name - delete_root_volume_on_termination (bool) - Whether to delete the root volume of the image after instance termination. Only applies when creating image from snapshot_id. Defaults to False. Note that leaving volumes behind after instance termination is not free + name (string) - The name of the AMI. Valid only for EBS-based images. + description (string) - The description of the AMI. + image_location (string) - Full path to your AMI manifest in Amazon S3 storage. Only used for S3-based AMI’s. + architecture (string) - The architecture of the AMI. Valid choices are: * i386 * x86_64 + kernel_id (string) - The ID of the kernel with which to launch the instances + root_device_name (string) - The root device name (e.g. /dev/sdh) + block_device_map (boto.ec2.blockdevicemapping.BlockDeviceMapping) - A BlockDeviceMapping data structure + describing the EBS volumes associated with the Image. + virtualization_type (string) - The virutalization_type of the image. Valid choices are: * paravirtual * hvm + sriov_net_support (string) - Advanced networking support. Valid choices are: * simple + snapshot_id (string) - A snapshot ID for the snapshot to be used as root device for the image. Mutually + exclusive with block_device_map, requires root_device_name + delete_root_volume_on_termination (bool) - Whether to delete the root volume of the image after instance + termination. Only applies when creating image from snapshot_id. Defaults to False. Note that leaving + volumes behind after instance termination is not free Returns: image_id - image ID of the newly created image """ - try: self._reload_connection() - image_location = image_dict.get('image_location', None) + image_location = image_dict.get("image_location", None) + if image_location: image_location = str(self.account_id) + str(image_location) - image_id = self.conn.register_image(image_dict.get('name', None), image_dict.get('description', None), - image_location, image_dict.get('architecture', None), - image_dict.get('kernel_id', None), - image_dict.get('root_device_name', None), - image_dict.get('block_device_map', None), - image_dict.get('virtualization_type', None), - image_dict.get('sriov_net_support', None), - image_dict.get('snapshot_id', None), - image_dict.get('delete_root_volume_on_termination', None)) + image_id = self.conn.register_image( + image_dict.get("name", None), + image_dict.get("description", None), + image_location, + image_dict.get("architecture", None), + image_dict.get("kernel_id", None), + image_dict.get("root_device_name", None), + image_dict.get("block_device_map", None), + image_dict.get("virtualization_type", None), + image_dict.get("sriov_net_support", None), + image_dict.get("snapshot_id", None), + image_dict.get("delete_root_volume_on_termination", None), + ) + return image_id except Exception as e: self.format_vimconn_exception(e) @@ -539,23 +704,27 @@ class vimconnector(vimconn.VimConnector): try: self._reload_connection() self.conn.deregister_image(image_id) + return image_id except Exception as e: self.format_vimconn_exception(e) def get_image_id_from_path(self, path): - ''' + """ Params: path - location of the image Returns: image_id - ID of the matching image - ''' + """ self._reload_connection() try: filters = {} + if path: - tokens = path.split('/') - filters['owner_id'] = tokens[0] - filters['name'] = '/'.join(tokens[1:]) + tokens = path.split("/") + filters["owner_id"] = tokens[0] + filters["name"] = "/".join(tokens[1:]) + image = self.conn.get_all_images(filters=filters)[0] + return image.id except Exception as e: self.format_vimconn_exception(e) @@ -571,33 +740,59 @@ class vimconnector(vimconn.VimConnector): [{}, ...] List can be empty """ - self.logger.debug("Getting image list from VIM") + try: self._reload_connection() image_id = None filters = {} - if 'id' in filter_dict: - image_id = filter_dict['id'] - if 'name' in filter_dict: - filters['name'] = filter_dict['name'] - if 'location' in filter_dict: - filters['location'] = filter_dict['location'] + + if "id" in filter_dict: + image_id = filter_dict["id"] + + if "name" in filter_dict: + filters["name"] = filter_dict["name"] + + if "location" in filter_dict: + filters["location"] = filter_dict["location"] + # filters['image_type'] = 'machine' # filter_dict['owner_id'] = self.account_id images = self.conn.get_all_images(image_id, filters=filters) image_list = [] + for image in images: - image_list.append({'id': str(image.id), 'name': str(image.name), 'status': str(image.state), - 'owner': str(image.owner_id), 'location': str(image.location), - 'is_public': str(image.is_public), 'architecture': str(image.architecture), - 'platform': str(image.platform)}) + image_list.append( + { + "id": str(image.id), + "name": str(image.name), + "status": str(image.state), + "owner": str(image.owner_id), + "location": str(image.location), + "is_public": str(image.is_public), + "architecture": str(image.architecture), + "platform": str(image.platform), + } + ) + return image_list except Exception as e: self.format_vimconn_exception(e) - def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, - disk_list=None, availability_zone_index=None, availability_zone_list=None): + def new_vminstance( + self, + name, + description, + start, + image_id, + flavor_id, + affinity_group_list, + net_list, + cloud_config=None, + disk_list=None, + availability_zone_index=None, + availability_zone_list=None, + ): """Create a new VM/instance in AWS Params: name decription @@ -607,19 +802,23 @@ class vimconnector(vimconn.VimConnector): net_list name net_id - subnet_id from AWS - vpci - (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities + vpci - (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM + capabilities model: (optional and only have sense for type==virtual) interface model: virtio, e1000, ... mac_address: (optional) mac address to assign to this interface type: (mandatory) can be one of: virtual, in this case always connected to a network of type 'net_type=bridge' - 'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it + 'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a + data/ptp network ot it can created unconnected 'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity. - VFnotShared - (SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs - are allocated on the same physical NIC + VFnotShared - (SRIOV without VLAN tag) same as PF for network connectivity. VF where no other + VFs are allocated on the same physical NIC bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS - port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing or True, it must apply the default VIM behaviour - vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this interface. 'net_list' is modified + port_security': (optional) If False it must avoid any traffic filtering at this interface. + If missing or True, it must apply the default VIM behaviour + vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this + interface. 'net_list' is modified elastic_ip - True/False to define if an elastic_ip is required cloud_config': (optional) dictionary with: key-pairs': (optional) list of strings with the public key to be inserted to the default user @@ -647,11 +846,12 @@ class vimconnector(vimconn.VimConnector): Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same as not present. """ - self.logger.debug("Creating a new VM instance") + try: + created_items = {} self._reload_connection() - instance = None + reservation = None _, userdata = self._create_user_data(cloud_config) if not net_list: @@ -660,67 +860,169 @@ class vimconnector(vimconn.VimConnector): key_name=self.key_pair, instance_type=flavor_id, security_groups=self.security_groups, - user_data=userdata + user_data=userdata, ) + else: for index, subnet in enumerate(net_list): - net_intr = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id=subnet.get('net_id'), - groups=None, - associate_public_ip_address=True) - if subnet.get('elastic_ip'): + net_intr = self.conn_vpc.create_network_interface( + subnet_id=subnet.get("net_id"), + groups=None, + ) + + interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( + network_interface_id=net_intr.id, + device_index=index, + ) + + interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection( + interface + ) + + if subnet.get("elastic_ip"): eip = self.conn.allocate_address() - self.conn.associate_address(allocation_id=eip.allocation_id, network_interface_id=net_intr.id) + self.conn.associate_address( + allocation_id=eip.allocation_id, + network_interface_id=net_intr.id, + ) if index == 0: - reservation = self.conn.run_instances( - image_id, - key_name=self.key_pair, - instance_type=flavor_id, - security_groups=self.security_groups, - network_interfaces=boto.ec2.networkinterface.NetworkInterfaceCollection(net_intr), - user_data=userdata + try: + reservation = self.conn.run_instances( + image_id, + key_name=self.key_pair, + instance_type=flavor_id, + security_groups=self.security_groups, + network_interfaces=interfaces, + user_data=userdata, + ) + except Exception as instance_create_error: + self.logger.debug(traceback.format_exc()) + self.format_vimconn_exception(instance_create_error) + + if index > 0: + try: + if reservation: + instance_id = self.wait_for_instance_id(reservation) + if instance_id and self.wait_for_vm( + instance_id, "running" + ): + self.conn.attach_network_interface( + network_interface_id=net_intr.id, + instance_id=instance_id, + device_index=index, + ) + except Exception as attach_network_error: + self.logger.debug(traceback.format_exc()) + self.format_vimconn_exception(attach_network_error) + + if instance_id := self.wait_for_instance_id(reservation): + time.sleep(30) + instance_status = self.refresh_vms_status(instance_id) + refreshed_instance_status = instance_status.get(instance_id) + instance_interfaces = refreshed_instance_status.get( + "interfaces" ) - else: - while True: - try: - self.conn.attach_network_interface( - network_interface_id=boto.ec2.networkinterface.NetworkInterfaceCollection(net_intr), - instance_id=instance.id, device_index=0) - break - except: - time.sleep(10) - net_list[index]['vim_id'] = reservation.instances[0].interfaces[index].id - - instance = reservation.instances[0] - return instance.id, None + for idx, interface in enumerate(instance_interfaces): + if idx == index: + net_list[index]["vim_id"] = instance_interfaces[ + idx + ].get("vim_interface_id") + + instance_id = self.wait_for_instance_id(reservation) + created_items["vm_id:" + str(instance_id)] = True + + return instance_id, created_items except Exception as e: + self.logger.debug(traceback.format_exc()) self.format_vimconn_exception(e) def get_vminstance(self, vm_id): """Returns the VM instance information from VIM""" - try: self._reload_connection() reservation = self.conn.get_all_instances(vm_id) + return reservation[0].instances[0].__dict__ except Exception as e: self.format_vimconn_exception(e) - def delete_vminstance(self, vm_id, created_items=None): + def delete_vminstance(self, vm_id, created_items=None, volumes_to_hold=None): """Removes a VM instance from VIM Returns the instance identifier""" - try: self._reload_connection() self.logger.debug("DELETING VM_ID: " + str(vm_id)) - self.conn.terminate_instances(vm_id) - return vm_id + reservation = self.conn.get_all_instances(vm_id)[0] + if hasattr(reservation, "instances"): + instance = reservation.instances[0] + + self.conn.terminate_instances(vm_id) + if self.wait_for_vm(vm_id, "terminated"): + for interface in instance.interfaces: + self.conn_vpc.delete_network_interface( + network_interface_id=interface.id, + ) + if self.network_delete_on_termination: + for net in self.network_delete_on_termination: + try: + self.conn_vpc.delete_subnet(net) + except Exception as net_delete_error: + if isinstance(net_delete_error, EC2ResponseError): + self.logger.warning(f"Deleting network {net}: failed") + else: + self.format_vimconn_exception(net_delete_error) + + return vm_id except Exception as e: self.format_vimconn_exception(e) + def wait_for_instance_id(self, reservation): + if not reservation: + return False + + self._reload_connection() + elapsed_time = 0 + while elapsed_time < 30: + if reservation.instances: + instance_id = reservation.instances[0].id + return instance_id + time.sleep(5) + elapsed_time += 5 + else: + raise vimconn.VimConnException( + "Failed to get instance_id for reservation", + reservation, + http_code=vimconn.HTTP_Request_Timeout, + ) + + def wait_for_vm(self, vm_id, status): + """wait until vm is in the desired status and return True. + If the timeout is reached generate an exception""" + + self._reload_connection() + + elapsed_time = 0 + while elapsed_time < self.server_timeout: + if self.conn.get_all_instances(vm_id): + reservation = self.conn.get_all_instances(vm_id)[0] + if hasattr(reservation, "instances"): + instance = reservation.instances[0] + if instance.state == status: + return True + time.sleep(5) + elapsed_time += 5 + + # if we exceeded the timeout + else: + raise vimconn.VimConnException( + "Timeout waiting for instance " + vm_id + " to get " + status, + http_code=vimconn.HTTP_Request_Timeout, + ) + def refresh_vms_status(self, vm_list): - """ Get the status of the virtual machines and their interfaces/ports + """Get the status of the virtual machines and their interfaces/ports Params: the list of VM identifiers Returns a dictionary with: vm_id: #VIM id of this Virtual Machine @@ -742,42 +1044,79 @@ class vimconnector(vimconn.VimConnector): ip_address - The IP address of the interface within the subnet. """ self.logger.debug("Getting VM instance information from VIM") + try: self._reload_connection() - reservation = self.conn.get_all_instances(vm_list)[0] + elapsed_time = 0 + while elapsed_time < self.server_timeout: + reservation = self.conn.get_all_instances(vm_list)[0] + if reservation: + break + time.sleep(5) + elapsed_time += 5 + + # if we exceeded the timeout + else: + raise vimconn.VimConnException( + vm_list + "could not be gathered, refresh vm status failed", + http_code=vimconn.HTTP_Request_Timeout, + ) + instances = {} instance_dict = {} + for instance in reservation.instances: - try: - if instance.state in ("pending"): - instance_dict['status'] = "BUILD" - elif instance.state in ("available", "running", "up"): - instance_dict['status'] = 'ACTIVE' - else: - instance_dict['status'] = 'ERROR' - instance_dict['error_msg'] = "" - instance_dict['interfaces'] = [] - interface_dict = {} - for interface in instance.interfaces: - interface_dict['vim_interface_id'] = interface.id - interface_dict['vim_net_id'] = interface.subnet_id - interface_dict['mac_address'] = interface.mac_address - if hasattr(interface, 'publicIp') and interface.publicIp != None: - interface_dict['ip_address'] = interface.publicIp + ";" + interface.private_ip_address - else: - interface_dict['ip_address'] = interface.private_ip_address - instance_dict['interfaces'].append(interface_dict) - except Exception as e: - self.logger.error("Exception getting vm status: %s", str(e), exc_info=True) - instance_dict['status'] = "DELETED" - instance_dict['error_msg'] = str(e) - finally: + if hasattr(instance, "id"): try: - instance_dict['vim_info'] = yaml.safe_dump(instance, default_flow_style=True, width=256) - except yaml.YAMLError as e: - # self.logger.error("Exception getting vm status: %s", str(e), exc_info=True) - instance_dict['vim_info'] = str(instance) - instances[instance.id] = instance_dict + if instance.state in ("pending"): + instance_dict["status"] = "BUILD" + elif instance.state in ("available", "running", "up"): + instance_dict["status"] = "ACTIVE" + else: + instance_dict["status"] = "ERROR" + + instance_dict["error_msg"] = "" + instance_dict["interfaces"] = [] + + for interface in instance.interfaces: + interface_dict = { + "vim_interface_id": interface.id, + "vim_net_id": interface.subnet_id, + "mac_address": interface.mac_address, + } + + if ( + hasattr(interface, "publicIp") + and interface.publicIp is not None + ): + interface_dict["ip_address"] = ( + interface.publicIp + + ";" + + interface.private_ip_address + ) + else: + interface_dict[ + "ip_address" + ] = interface.private_ip_address + + instance_dict["interfaces"].append(interface_dict) + except Exception as e: + self.logger.error( + "Exception getting vm status: %s", str(e), exc_info=True + ) + instance_dict["status"] = "DELETED" + instance_dict["error_msg"] = str(e) + finally: + instance_dictionary = vars(instance) + cleared_instance_dict = { + key: instance_dictionary[key] + for key in instance_dictionary + if not (isinstance(instance_dictionary[key], object)) + } + instance_dict["vim_info"] = cleared_instance_dict + + instances[instance.id] = instance_dict + return instances except Exception as e: self.logger.error("Exception getting vm status: %s", str(e), exc_info=True) @@ -798,6 +1137,27 @@ class vimconnector(vimconn.VimConnector): self.conn.terminate_instances(vm_id) elif "reboot" in action_dict: self.conn.reboot_instances(vm_id) + return None except Exception as e: self.format_vimconn_exception(e) + + def migrate_instance(self, vm_id, compute_host=None): + """ + Migrate a vdu + param: + vm_id: ID of an instance + compute_host: Host to migrate the vdu to + """ + # TODO: Add support for migration + raise vimconn.VimConnNotImplemented("Not implemented") + + def resize_instance(self, vm_id, flavor_id=None): + """ + resize a vdu + param: + vm_id: ID of an instance + flavor_id: flavor to resize the vdu + """ + # TODO: Add support for resize + raise vimconn.VimConnNotImplemented("Not implemented")