X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=blobdiff_plain;f=osm_ro%2Fvimconn_openstack.py;h=073a752274d0ec5149d8552ecdbe9cd11fbcdf5c;hp=f4cf655ce8c6a85c290b69e5659d244eee221b55;hb=b7aa1bb3eaee2b986d3e192a4e3980c2fb1446b9;hpb=d42cceac9ede14b960d480aecd106592b35f154c diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py index f4cf655c..073a7522 100644 --- a/osm_ro/vimconn_openstack.py +++ b/osm_ro/vimconn_openstack.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ## -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U. # This file is part of openmano # All Rights Reserved. # @@ -22,19 +22,30 @@ ## ''' -osconnector implements all the methods to interact with openstack using the python-client. +osconnector implements all the methods to interact with openstack using the python-neutronclient. + +For the VNF forwarding graph, The OpenStack VIM connector calls the +networking-sfc Neutron extension methods, whose resources are mapped +to the VIM connector's SFC resources as follows: +- Classification (OSM) -> Flow Classifier (Neutron) +- Service Function Instance (OSM) -> Port Pair (Neutron) +- Service Function (OSM) -> Port Pair Group (Neutron) +- Service Function Path (OSM) -> Port Chain (Neutron) ''' -__author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research" -__date__ ="$22-jun-2014 11:19:29$" +__author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa" +__date__ = "$22-sep-2017 23:59:59$" import vimconn -import json -import yaml +# import json import logging import netaddr import time import yaml import random +import re +import copy +from pprint import pformat +from types import StringTypes from novaclient import client as nClient, exceptions as nvExceptions from keystoneauth1.identity import v2, v3 @@ -43,7 +54,6 @@ import keystoneclient.exceptions as ksExceptions import keystoneclient.v3.client as ksClient_v3 import keystoneclient.v2_0.client as ksClient_v2 from glanceclient import client as glClient -import glanceclient.client as gl1Client import glanceclient.exc as gl1Exceptions from cinderclient import client as cClient from httplib import HTTPException @@ -51,7 +61,8 @@ from neutronclient.neutron import client as neClient from neutronclient.common import exceptions as neExceptions from requests.exceptions import ConnectionError -'''contain the openstack virtual machine status to openmano status''' + +"""contain the openstack virtual machine status to openmano status""" vmStatus2manoFormat={'ACTIVE':'ACTIVE', 'PAUSED':'PAUSED', 'SUSPENDED': 'SUSPENDED', @@ -62,9 +73,23 @@ vmStatus2manoFormat={'ACTIVE':'ACTIVE', netStatus2manoFormat={'ACTIVE':'ACTIVE','PAUSED':'PAUSED','INACTIVE':'INACTIVE','BUILD':'BUILD','ERROR':'ERROR','DELETED':'DELETED' } +supportedClassificationTypes = ['legacy_flow_classifier'] + #global var to have a timeout creating and deleting volumes -volume_timeout = 60 -server_timeout = 60 +volume_timeout = 600 +server_timeout = 600 + + +class SafeDumper(yaml.SafeDumper): + def represent_data(self, data): + # Openstack APIs use custom subclasses of dict and YAML safe dumper + # is designed to not handle that (reference issue 142 of pyyaml) + if isinstance(data, dict) and data.__class__ != dict: + # A simple solution is to convert those items back to dicts + data = dict(data.items()) + + return super(SafeDumper, self).represent_data(data) + class vimconnector(vimconn.vimconnector): def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, @@ -77,24 +102,64 @@ class vimconnector(vimconn.vimconnector): if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'): raise vimconn.vimconnException("Invalid value '{}' for config:APIversion. " "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version)) + vim_type = config.get('vim_type') + if vim_type and vim_type not in ('vio', 'VIO'): + raise vimconn.vimconnException("Invalid value '{}' for config:vim_type." + "Allowed values are 'vio' or 'VIO'".format(vim_type)) + + if config.get('dataplane_net_vlan_range') is not None: + #validate vlan ranges provided by user + self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'), 'dataplane_net_vlan_range') + + if config.get('multisegment_vlan_range') is not None: + #validate vlan ranges provided by user + self._validate_vlan_ranges(config.get('multisegment_vlan_range'), 'multisegment_vlan_range') + vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config) - self.insecure = self.config.get("insecure", False) + if self.config.get("insecure") and self.config.get("ca_cert"): + raise vimconn.vimconnException("options insecure and ca_cert are mutually exclusive") + self.verify = True + if self.config.get("insecure"): + self.verify = False + if self.config.get("ca_cert"): + self.verify = self.config.get("ca_cert") + if not url: - raise TypeError, 'url param can not be NoneType' + raise TypeError('url param can not be NoneType') self.persistent_info = persistent_info + self.availability_zone = persistent_info.get('availability_zone', None) self.session = persistent_info.get('session', {'reload_client': True}) + self.my_tenant_id = self.session.get('my_tenant_id') self.nova = self.session.get('nova') self.neutron = self.session.get('neutron') self.cinder = self.session.get('cinder') self.glance = self.session.get('glance') + # self.glancev1 = self.session.get('glancev1') self.keystone = self.session.get('keystone') self.api_version3 = self.session.get('api_version3') + self.vim_type = self.config.get("vim_type") + if self.vim_type: + self.vim_type = self.vim_type.upper() + if self.config.get("use_internal_endpoint"): + self.endpoint_type = "internalURL" + else: + self.endpoint_type = None self.logger = logging.getLogger('openmano.vim.openstack') + + # allow security_groups to be a list or a single string + if isinstance(self.config.get('security_groups'), str): + self.config['security_groups'] = [self.config['security_groups']] + self.security_groups_id = None + + ####### VIO Specific Changes ######### + if self.vim_type == "VIO": + self.logger = logging.getLogger('openmano.vim.vio') + if log_level: - self.logger.setLevel( getattr(logging, log_level) ) + self.logger.setLevel( getattr(logging, log_level)) def __getitem__(self, index): """Get individuals parameters. @@ -104,7 +169,7 @@ class vimconnector(vimconn.vimconnector): elif index == 'user_domain_id': return self.config.get("user_domain_id") else: - vimconn.vimconnector.__getitem__(self, index) + return vimconn.vimconnector.__getitem__(self, index) def __setitem__(self, index, value): """Set individuals parameters and it is marked as dirty so to force connection reload. @@ -117,43 +182,100 @@ class vimconnector(vimconn.vimconnector): vimconn.vimconnector.__setitem__(self, index, value) self.session['reload_client'] = True + def serialize(self, value): + """Serialization of python basic types. + + In the case value is not serializable a message will be logged and a + simple representation of the data that cannot be converted back to + python is returned. + """ + if isinstance(value, StringTypes): + return value + + try: + return yaml.dump(value, Dumper=SafeDumper, + default_flow_style=True, width=256) + except yaml.representer.RepresenterError: + self.logger.debug( + 'The following entity cannot be serialized in YAML:' + '\n\n%s\n\n', pformat(value), exc_info=True) + return str(value) + def _reload_connection(self): '''Called before any operation, it check if credentials has changed Throw keystoneclient.apiclient.exceptions.AuthorizationFailure ''' - #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) + #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) if self.session['reload_client']: if self.config.get('APIversion'): self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3' else: # get from ending auth_url that end with v3 or with v2.0 - self.api_version3 = self.url.split("/")[-1] == "v3" + self.api_version3 = self.url.endswith("/v3") or self.url.endswith("/v3/") self.session['api_version3'] = self.api_version3 if self.api_version3: + if self.config.get('project_domain_id') or self.config.get('project_domain_name'): + project_domain_id_default = None + else: + project_domain_id_default = 'default' + if self.config.get('user_domain_id') or self.config.get('user_domain_name'): + user_domain_id_default = None + else: + user_domain_id_default = 'default' auth = v3.Password(auth_url=self.url, username=self.user, password=self.passwd, project_name=self.tenant_name, project_id=self.tenant_id, - project_domain_id=self.config.get('project_domain_id', 'default'), - user_domain_id=self.config.get('user_domain_id', 'default')) + project_domain_id=self.config.get('project_domain_id', project_domain_id_default), + user_domain_id=self.config.get('user_domain_id', user_domain_id_default), + project_domain_name=self.config.get('project_domain_name'), + user_domain_name=self.config.get('user_domain_name')) else: auth = v2.Password(auth_url=self.url, username=self.user, password=self.passwd, tenant_name=self.tenant_name, tenant_id=self.tenant_id) - sess = session.Session(auth=auth, verify=not self.insecure) + sess = session.Session(auth=auth, verify=self.verify) + # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River Titanium cloud and StarlingX + region_name = self.config.get('region_name') if self.api_version3: - self.keystone = ksClient_v3.Client(session=sess) + self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type, region_name=region_name) else: - self.keystone = ksClient_v2.Client(session=sess) + self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type) self.session['keystone'] = self.keystone - self.nova = self.session['nova'] = nClient.Client("2.1", session=sess) - self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess) - self.cinder = self.session['cinder'] = cClient.Client(2, session=sess) - self.glance = self.session['glance'] = glClient.Client(2, session=sess) + # In order to enable microversion functionality an explicit microversion must be specified in 'config'. + # This implementation approach is due to the warning message in + # https://developer.openstack.org/api-guide/compute/microversions.html + # where it is stated that microversion backwards compatibility is not guaranteed and clients should + # always require an specific microversion. + # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config + version = self.config.get("microversion") + if not version: + version = "2.1" + # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River Titanium cloud and StarlingX + self.nova = self.session['nova'] = nClient.Client(str(version), session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type, region_name=region_name) + try: + self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id() + except Exception as e: + self.logger.error("Cannot get project_id from session", exc_info=True) + if self.endpoint_type == "internalURL": + glance_service_id = self.keystone.services.list(name="glance")[0].id + glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url + else: + glance_endpoint = None + self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint) + # using version 1 of glance client in new_image() + # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess, + # endpoint=glance_endpoint) self.session['reload_client'] = False self.persistent_info['session'] = self.session + # add availablity zone info inside self.persistent_info + self._set_availablity_zones() + self.persistent_info['availability_zone'] = self.availability_zone + self.security_groups_id = None # force to get again security_groups_ids next time they are needed def __net_os2mano(self, net_list_dict): '''Transform the net openstack format to mano format @@ -169,24 +291,162 @@ class vimconnector(vimconn.vimconnector): net['type']='data' else: net['type']='bridge' - - - + + def __classification_os2mano(self, class_list_dict): + """Transform the openstack format (Flow Classifier) to mano format + (Classification) class_list_dict can be a list of dict or a single dict + """ + if isinstance(class_list_dict, dict): + class_list_ = [class_list_dict] + elif isinstance(class_list_dict, list): + class_list_ = class_list_dict + else: + raise TypeError( + "param class_list_dict must be a list or a dictionary") + for classification in class_list_: + id = classification.pop('id') + name = classification.pop('name') + description = classification.pop('description') + project_id = classification.pop('project_id') + tenant_id = classification.pop('tenant_id') + original_classification = copy.deepcopy(classification) + classification.clear() + classification['ctype'] = 'legacy_flow_classifier' + classification['definition'] = original_classification + classification['id'] = id + classification['name'] = name + classification['description'] = description + classification['project_id'] = project_id + classification['tenant_id'] = tenant_id + + def __sfi_os2mano(self, sfi_list_dict): + """Transform the openstack format (Port Pair) to mano format (SFI) + sfi_list_dict can be a list of dict or a single dict + """ + if isinstance(sfi_list_dict, dict): + sfi_list_ = [sfi_list_dict] + elif isinstance(sfi_list_dict, list): + sfi_list_ = sfi_list_dict + else: + raise TypeError( + "param sfi_list_dict must be a list or a dictionary") + for sfi in sfi_list_: + sfi['ingress_ports'] = [] + sfi['egress_ports'] = [] + if sfi.get('ingress'): + sfi['ingress_ports'].append(sfi['ingress']) + if sfi.get('egress'): + sfi['egress_ports'].append(sfi['egress']) + del sfi['ingress'] + del sfi['egress'] + params = sfi.get('service_function_parameters') + sfc_encap = False + if params: + correlation = params.get('correlation') + if correlation: + sfc_encap = True + sfi['sfc_encap'] = sfc_encap + del sfi['service_function_parameters'] + + def __sf_os2mano(self, sf_list_dict): + """Transform the openstack format (Port Pair Group) to mano format (SF) + sf_list_dict can be a list of dict or a single dict + """ + if isinstance(sf_list_dict, dict): + sf_list_ = [sf_list_dict] + elif isinstance(sf_list_dict, list): + sf_list_ = sf_list_dict + else: + raise TypeError( + "param sf_list_dict must be a list or a dictionary") + for sf in sf_list_: + del sf['port_pair_group_parameters'] + sf['sfis'] = sf['port_pairs'] + del sf['port_pairs'] + + def __sfp_os2mano(self, sfp_list_dict): + """Transform the openstack format (Port Chain) to mano format (SFP) + sfp_list_dict can be a list of dict or a single dict + """ + if isinstance(sfp_list_dict, dict): + sfp_list_ = [sfp_list_dict] + elif isinstance(sfp_list_dict, list): + sfp_list_ = sfp_list_dict + else: + raise TypeError( + "param sfp_list_dict must be a list or a dictionary") + for sfp in sfp_list_: + params = sfp.pop('chain_parameters') + sfc_encap = False + if params: + correlation = params.get('correlation') + if correlation: + sfc_encap = True + sfp['sfc_encap'] = sfc_encap + sfp['spi'] = sfp.pop('chain_id') + sfp['classifications'] = sfp.pop('flow_classifiers') + sfp['service_functions'] = sfp.pop('port_pair_groups') + + # placeholder for now; read TODO note below + def _validate_classification(self, type, definition): + # only legacy_flow_classifier Type is supported at this point + return True + # TODO(igordcard): this method should be an abstract method of an + # abstract Classification class to be implemented by the specific + # Types. Also, abstract vimconnector should call the validation + # method before the implemented VIM connectors are called. + def _format_exception(self, exception): '''Transform a keystone, nova, neutron exception into a vimconn exception''' - if isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, - ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed - )): - raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) - elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException, - neExceptions.NeutronException, nvExceptions.BadRequest)): - raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + str(exception)) - elif isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound)): - raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + str(exception)) + + # Fixing bug 665 https://osm.etsi.org/bugzilla/show_bug.cgi?id=665 + # There are some openstack versions that message error are unicode with non English + message_error = exception.message + if isinstance(message_error, unicode): + message_error = message_error.encode("utf") + + if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound, + gl1Exceptions.HTTPNotFound)): + raise vimconn.vimconnNotFoundException(type(exception).__name__ + ": " + message_error) + elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, + ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)): + raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + message_error) + elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)): + raise vimconn.vimconnException(type(exception).__name__ + ": " + message_error) + elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException, + neExceptions.NeutronException)): + raise vimconn.vimconnUnexpectedResponse(type(exception).__name__ + ": " + message_error) elif isinstance(exception, nvExceptions.Conflict): - raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + str(exception)) + raise vimconn.vimconnConflictException(type(exception).__name__ + ": " + message_error) + elif isinstance(exception, vimconn.vimconnException): + raise exception else: # () - raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + str(exception)) + self.logger.error("General Exception " + message_error, exc_info=True) + raise vimconn.vimconnConnectionException(type(exception).__name__ + ": " + message_error) + + def _get_ids_from_name(self): + """ + Obtain ids from name of tenant and security_groups. Store at self .security_groups_id" + :return: None + """ + # get tenant_id if only tenant_name is supplied + self._reload_connection() + if not self.my_tenant_id: + raise vimconn.vimconnConnectionException("Error getting tenant information from name={} id={}". + format(self.tenant_name, self.tenant_id)) + if self.config.get('security_groups') and not self.security_groups_id: + # convert from name to id + neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"] + + self.security_groups_id = [] + for sg in self.config.get('security_groups'): + for neutron_sg in neutron_sg_list: + if sg in (neutron_sg["id"], neutron_sg["name"]): + self.security_groups_id.append(neutron_sg["id"]) + break + else: + self.security_groups_id = None + raise vimconn.vimconnConnectionException("Not found security group {} for this tenant".format(sg)) def get_tenant_list(self, filter_dict={}): '''Obtain tenants of VIM @@ -223,7 +483,7 @@ class vimconnector(vimconn.vimconnector): else: project = self.keystone.tenants.create(tenant_name, tenant_description) return project.id - except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e: + except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError) as e: self._format_exception(e) def delete_tenant(self, tenant_id): @@ -236,52 +496,114 @@ class vimconnector(vimconn.vimconnector): else: self.keystone.tenants.delete(tenant_id) return tenant_id - except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e: + except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError) as e: self._format_exception(e) def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None): - '''Adds a tenant network to VIM. Returns the network identifier''' + """Adds a tenant network to VIM + Params: + 'net_name': name of the network + 'net_type': one of: + 'bridge': overlay isolated network + 'data': underlay E-LAN network for Passthrough and SRIOV interfaces + 'ptp': underlay E-LINE network for Passthrough and SRIOV interfaces. + 'ip_profile': is a dict containing the IP parameters of the network + 'ip_version': can be "IPv4" or "IPv6" (Currently only IPv4 is implemented) + 'subnet_address': ip_prefix_schema, that is X.X.X.X/Y + 'gateway_address': (Optional) ip_schema, that is X.X.X.X + 'dns_address': (Optional) comma separated list of ip_schema, e.g. X.X.X.X[,X,X,X,X] + 'dhcp_enabled': True or False + 'dhcp_start_address': ip_schema, first IP to grant + 'dhcp_count': number of IPs to grant. + 'shared': if this network can be seen/use by other tenants/organization + 'vlan': in case of a data or ptp net_type, the intended vlan tag to be used for the network + Returns a tuple with the network identifier and created_items, or raises an exception on error + created_items can be None or a dictionary where this method can include key-values that will be passed to + the method delete_network. Can be used to store created segments, created l2gw connections, etc. + Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same + as not present. + """ self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type) - #self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile)) + # self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile)) try: new_net = None + created_items = {} self._reload_connection() network_dict = {'name': net_name, 'admin_state_up': True} if net_type=="data" or net_type=="ptp": if self.config.get('dataplane_physical_net') == None: raise vimconn.vimconnConflictException("You must provide a 'dataplane_physical_net' at config value before creating sriov network") - network_dict["provider:physical_network"] = self.config['dataplane_physical_net'] #"physnet_sriov" #TODO physical - network_dict["provider:network_type"] = "vlan" - if vlan!=None: - network_dict["provider:network_type"] = vlan - network_dict["shared"]=shared - new_net=self.neutron.create_network({'network':network_dict}) - #print new_net - #create subnetwork, even if there is no profile + if not self.config.get('multisegment_support'): + network_dict["provider:physical_network"] = self.config[ + 'dataplane_physical_net'] # "physnet_sriov" #TODO physical + network_dict["provider:network_type"] = "vlan" + if vlan!=None: + network_dict["provider:network_type"] = vlan + else: + ###### Multi-segment case ###### + segment_list = [] + segment1_dict = {} + segment1_dict["provider:physical_network"] = '' + segment1_dict["provider:network_type"] = 'vxlan' + segment_list.append(segment1_dict) + segment2_dict = {} + segment2_dict["provider:physical_network"] = self.config['dataplane_physical_net'] + segment2_dict["provider:network_type"] = "vlan" + if self.config.get('multisegment_vlan_range'): + vlanID = self._generate_multisegment_vlanID() + segment2_dict["provider:segmentation_id"] = vlanID + # else + # raise vimconn.vimconnConflictException( + # "You must provide 'multisegment_vlan_range' at config dict before creating a multisegment network") + segment_list.append(segment2_dict) + network_dict["segments"] = segment_list + + ####### VIO Specific Changes ######### + if self.vim_type == "VIO": + if vlan is not None: + network_dict["provider:segmentation_id"] = vlan + else: + if self.config.get('dataplane_net_vlan_range') is None: + raise vimconn.vimconnConflictException("You must provide "\ + "'dataplane_net_vlan_range' in format [start_ID - end_ID]"\ + "at config value before creating sriov network with vlan tag") + + network_dict["provider:segmentation_id"] = self._generate_vlanID() + + network_dict["shared"] = shared + if self.config.get("disable_network_port_security"): + network_dict["port_security_enabled"] = False + new_net = self.neutron.create_network({'network':network_dict}) + # print new_net + # create subnetwork, even if there is no profile if not ip_profile: ip_profile = {} - if 'subnet_address' not in ip_profile: + if not ip_profile.get('subnet_address'): #Fake subnet is required subnet_rand = random.randint(0, 255) ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand) - if 'ip_version' not in ip_profile: + if 'ip_version' not in ip_profile: ip_profile['ip_version'] = "IPv4" - subnet={"name":net_name+"-subnet", + subnet = {"name": net_name+"-subnet", "network_id": new_net["network"]["id"], "ip_version": 4 if ip_profile['ip_version']=="IPv4" else 6, "cidr": ip_profile['subnet_address'] } - if 'gateway_address' in ip_profile: + # Gateway should be set to None if not needed. Otherwise openstack assigns one by default + if ip_profile.get('gateway_address'): subnet['gateway_ip'] = ip_profile['gateway_address'] + else: + subnet['gateway_ip'] = None if ip_profile.get('dns_address'): subnet['dns_nameservers'] = ip_profile['dns_address'].split(";") if 'dhcp_enabled' in ip_profile: - subnet['enable_dhcp'] = False if ip_profile['dhcp_enabled']=="false" else True - if 'dhcp_start_address' in ip_profile: - subnet['allocation_pools']=[] + subnet['enable_dhcp'] = False if \ + ip_profile['dhcp_enabled']=="false" or ip_profile['dhcp_enabled']==False else True + if ip_profile.get('dhcp_start_address'): + subnet['allocation_pools'] = [] subnet['allocation_pools'].append(dict()) subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address'] - if 'dhcp_count' in ip_profile: + if ip_profile.get('dhcp_count'): #parts = ip_profile['dhcp_start_address'].split('.') #ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3]) ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address'])) @@ -290,8 +612,29 @@ class vimconnector(vimconn.vimconnector): subnet['allocation_pools'][0]['end'] = ip_str #self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet)) self.neutron.create_subnet({"subnet": subnet} ) - return new_net["network"]["id"] - except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e: + + if net_type == "data" and self.config.get('multisegment_support'): + if self.config.get('l2gw_support'): + l2gw_list = self.neutron.list_l2_gateways().get("l2_gateways", ()) + for l2gw in l2gw_list: + l2gw_conn = {} + l2gw_conn["l2_gateway_id"] = l2gw["id"] + l2gw_conn["network_id"] = new_net["network"]["id"] + l2gw_conn["segmentation_id"] = str(vlanID) + new_l2gw_conn = self.neutron.create_l2_gateway_connection({"l2_gateway_connection": l2gw_conn}) + created_items["l2gwconn:" + str(new_l2gw_conn["l2_gateway_connection"]["id"])] = True + return new_net["network"]["id"], created_items + except Exception as e: + #delete l2gw connections (if any) before deleting the network + for k, v in created_items.items(): + if not v: # skip already deleted + continue + try: + k_item, _, k_id = k.partition(":") + if k_item == "l2gwconn": + self.neutron.delete_l2_gateway_connection(k_id) + except Exception as e2: + self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e2).__name__, e2)) if new_net: self.neutron.delete_network(new_net['network']['id']) self._format_exception(e) @@ -310,10 +653,11 @@ class vimconnector(vimconn.vimconnector): self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict)) try: self._reload_connection() - if self.api_version3 and "tenant_id" in filter_dict: - filter_dict['project_id'] = filter_dict.pop('tenant_id') #TODO check - net_dict=self.neutron.list_networks(**filter_dict) - net_list=net_dict["networks"] + filter_dict_os = filter_dict.copy() + if self.api_version3 and "tenant_id" in filter_dict_os: + filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id') #T ODO check + net_dict = self.neutron.list_networks(**filter_dict_os) + net_list = net_dict["networks"] self.__net_os2mano(net_list) return net_list except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e: @@ -340,14 +684,33 @@ class vimconnector(vimconn.vimconnector): subnets.append(subnet) net["subnets"] = subnets net["encapsulation"] = net.get('provider:network_type') + net["encapsulation_type"] = net.get('provider:network_type') net["segmentation_id"] = net.get('provider:segmentation_id') + net["encapsulation_id"] = net.get('provider:segmentation_id') return net - def delete_network(self, net_id): - '''Deletes a tenant network from VIM. Returns the old network identifier''' + def delete_network(self, net_id, created_items=None): + """ + Removes a tenant network from VIM and its associated elements + :param net_id: VIM identifier of the network, provided by method new_network + :param created_items: dictionary with extra items to be deleted. provided by method new_network + Returns the network identifier or raises an exception upon error or when network is not found + """ self.logger.debug("Deleting network '%s' from VIM", net_id) + if created_items == None: + created_items = {} try: self._reload_connection() + #delete l2gw connections (if any) before deleting the network + for k, v in created_items.items(): + if not v: # skip already deleted + continue + try: + k_item, _, k_id = k.partition(":") + if k_item == "l2gwconn": + self.neutron.delete_l2_gateway_connection(k_id) + except Exception as e: + self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e).__name__, e)) #delete VM ports attached to this networks before the network ports = self.neutron.list_ports(network_id=net_id) for p in ports['ports']: @@ -368,16 +731,16 @@ class vimconnector(vimconn.vimconnector): net_id: #VIM id of this network status: #Mandatory. Text with one of: # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) + # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) # OTHER (Vim reported other status not understood) # ERROR (VIM indicates an ERROR status) - # ACTIVE, INACTIVE, DOWN (admin down), + # ACTIVE, INACTIVE, DOWN (admin down), # BUILD (on building process) # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR vim_info: #Text with plain information obtained from vim (yaml.safe_dump) - ''' + ''' net_dict={} for net_id in net_list: net = {} @@ -388,13 +751,12 @@ class vimconnector(vimconn.vimconnector): else: net["status"] = "OTHER" net["error_msg"] = "VIM status reported " + net_vim['status'] - + if net['status'] == "ACTIVE" and not net_vim['admin_state_up']: net['status'] = 'DOWN' - try: - net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256) - except yaml.representer.RepresenterError: - net['vim_info'] = str(net_vim) + + net['vim_info'] = self.serialize(net_vim) + if net_vim.get('fault'): #TODO net['error_msg'] = str(net_vim['fault']) except vimconn.vimconnNotFoundException as e: @@ -434,10 +796,10 @@ class vimconnector(vimconn.vimconnector): flavor_candidate_data = (10000, 10000, 10000) flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"]) # numa=None - numas = flavor_dict.get("extended", {}).get("numas") - if numas: + extended = flavor_dict.get("extended", {}) + if extended: #TODO - raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemted") + raise vimconn.vimconnNotFoundException("Flavor with EPA still not implemented") # if len(numas) > 1: # raise vimconn.vimconnNotFoundException("Cannot find any flavor with more than one numa") # numa=numas[0] @@ -459,6 +821,19 @@ class vimconnector(vimconn.vimconnector): except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException, ConnectionError) as e: self._format_exception(e) + def process_resource_quota(self, quota, prefix, extra_specs): + """ + :param prefix: + :param extra_specs: + :return: + """ + if 'limit' in quota: + extra_specs["quota:" + prefix + "_limit"] = quota['limit'] + if 'reserve' in quota: + extra_specs["quota:" + prefix + "_reservation"] = quota['reserve'] + if 'shares' in quota: + extra_specs["quota:" + prefix + "_shares_level"] = "custom" + extra_specs["quota:" + prefix + "_shares_share"] = quota['shares'] def new_flavor(self, flavor_data, change_name_if_used=True): '''Adds a tenant flavor to openstack VIM @@ -469,78 +844,92 @@ class vimconnector(vimconn.vimconnector): retry=0 max_retries=3 name_suffix = 0 - name=flavor_data['name'] - while retry 1: - return -1, "Can not add flavor with more than one numa" - numa_properties = {"hw:numa_nodes":str(numa_nodes)} - numa_properties["hw:mem_page_size"] = "large" - numa_properties["hw:cpu_policy"] = "dedicated" - numa_properties["hw:numa_mempolicy"] = "strict" - for numa in numas: - #overwrite ram and vcpus - ram = numa['memory']*1024 - #See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/implemented/virt-driver-cpu-thread-pinning.html - if 'paired-threads' in numa: - vcpus = numa['paired-threads']*2 - #cpu_thread_policy "require" implies that the compute node must have an STM architecture - numa_properties["hw:cpu_thread_policy"] = "require" - numa_properties["hw:cpu_policy"] = "dedicated" - elif 'cores' in numa: - vcpus = numa['cores'] - # cpu_thread_policy "prefer" implies that the host must not have an SMT architecture, or a non-SMT architecture will be emulated - numa_properties["hw:cpu_thread_policy"] = "isolate" - numa_properties["hw:cpu_policy"] = "dedicated" - elif 'threads' in numa: - vcpus = numa['threads'] - # cpu_thread_policy "prefer" implies that the host may or may not have an SMT architecture - numa_properties["hw:cpu_thread_policy"] = "prefer" - numa_properties["hw:cpu_policy"] = "dedicated" - # for interface in numa.get("interfaces",() ): - # if interface["dedicated"]=="yes": - # raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable) - # #TODO, add the key 'pci_passthrough:alias"="